From 01b9e77d33e2d7942e194d6658c9aeb1ccd5182b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 1 Sep 2026 22:05:51 +0800 Subject: [PATCH 01/25] Implement fully functional auto redirect for Android --- go.mod | 2 +- go.sum | 4 +- nfqueue_linux.go | 95 +-- redirect.go | 40 +- redirect_iptables.go | 842 +++++++++++++++++++++++++-- redirect_linux.go | 532 +++++++++++------ redirect_nftables.go | 54 +- redirect_nftables_rules.go | 74 +-- redirect_route_bypass_android.go | 340 +++++++++++ redirect_route_linux.go | 136 ++++- redirect_route_vpnservice_android.go | 228 ++++++++ redirect_route_vpnservice_stub.go | 28 + redirect_server.go | 87 ++- tun.go | 1 + tun_linux.go | 74 ++- 15 files changed, 2066 insertions(+), 471 deletions(-) create mode 100644 redirect_route_bypass_android.go create mode 100644 redirect_route_vpnservice_android.go create mode 100644 redirect_route_vpnservice_stub.go diff --git a/go.mod b/go.mod index fbba8e2c..87a60420 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/mdlayher/netlink v1.11.2 github.com/sagernet/fswatch v0.1.2 github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 - github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a + github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf github.com/sagernet/nftables v0.3.0-mod.4 github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 0c72db92..6d1e1b40 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/sagernet/fswatch v0.1.2 h1:/TT7k4mkce1qFPxamLO842WjqBgbTBiXP2mlUjp9PF github.com/sagernet/fswatch v0.1.2/go.mod h1:5BpGmpUQVd3Mc5r313HRpvADHRg3/rKn5QbwFteB880= github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 h1:IdQ7yTKkB2wv8txwshxUroPlO4npOYAV71xb7xQ7Lys= github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1/go.mod h1:9O3SQskYuCfdHNvHEsWuEAgoyKEF74PiWp4NsNUia8g= -github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a h1:ObwtHN2VpqE0ZNjr6sGeT00J8uU7JF4cNUdb44/Duis= -github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= +github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf h1:b/3rm+KvxzuYRcnMdLq/hmcqX6/2B8k24l6SDHL2Piw= +github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= github.com/sagernet/nftables v0.3.0-mod.4 h1:vnOtcDYeSXv2e5RoRuGH0lrpttQFJ8iC4ICS2nhlDSo= github.com/sagernet/nftables v0.3.0-mod.4/go.mod h1:8kslHG4VvYNihcco+i6uxIX7qbT8A56T0y5q7U44ZaQ= github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 h1:dyRIj+MZ2rc9JVzJoG04jxu+MpvHrLIZLJr0QjNAMGg= diff --git a/nfqueue_linux.go b/nfqueue_linux.go index 73151c6b..7d789128 100644 --- a/nfqueue_linux.go +++ b/nfqueue_linux.go @@ -17,45 +17,48 @@ import ( ) type nfqueueHandler struct { - ctx context.Context - cancel context.CancelFunc - handler Handler - logger logger.Logger - nfq *nfqueue.Nfqueue - queue uint16 - inputMark uint32 - outputMark uint32 - resetMark uint32 - closed atomic.Bool + ctx context.Context + cancel context.CancelFunc + handler AutoRedirectHandler + logger logger.Logger + nfq *nfqueue.Nfqueue + queue uint16 + inputMark uint32 + outputMark uint32 + resetMark uint32 + tproxyMark uint32 + markMask uint32 + repeatOnAccept bool + closed atomic.Bool } -type nfqueueOptions struct { - Context context.Context - Handler Handler - Logger logger.Logger - Queue uint16 - InputMark uint32 - OutputMark uint32 - ResetMark uint32 -} - -func newNFQueueHandler(options nfqueueOptions) (*nfqueueHandler, error) { - ctx, cancel := context.WithCancel(options.Context) - return &nfqueueHandler{ - ctx: ctx, - cancel: cancel, - handler: options.Handler, - logger: options.Logger, - queue: options.Queue, - inputMark: options.InputMark, - outputMark: options.OutputMark, - resetMark: options.ResetMark, - }, nil +func (h *nfqueueHandler) acceptVerdict(packet preMatchPacket) (verdict int, mark uint32) { + if !h.repeatOnAccept { + return nfqueue.NfAccept, 0 + } + if packet.protocol != uint8(unix.IPPROTO_TCP) { + return nfqueue.NfRepeat, h.inputMark + } + if packet.destination.Addr().Is6() { + if h.tproxyMark != 0 { + return nfqueue.NfRepeat, h.tproxyMark + } + return nfqueue.NfRepeat, h.inputMark + } + return nfqueue.NfAccept, 0 } -func (h *nfqueueHandler) setVerdict(packetID uint32, verdict int, mark uint32) { +func (h *nfqueueHandler) setVerdict(attr nfqueue.Attribute, verdict int, mark uint32) { + packetID := *attr.PacketID var err error if mark != 0 { + if h.markMask != 0 { + var originalMark uint32 + if attr.Mark != nil { + originalMark = *attr.Mark + } + mark = originalMark&^h.markMask | mark + } err = h.nfq.SetVerdictWithOption(packetID, verdict, nfqueue.WithMark(mark)) } else { err = h.nfq.SetVerdict(packetID, verdict) @@ -65,14 +68,17 @@ func (h *nfqueueHandler) setVerdict(packetID uint32, verdict int, mark uint32) { } } +// Without a bypass flag every packet a remaining NFQUEUE rule sends to an unbound queue is +// dropped, so the queue is unbound in Close rather than with the caller's context. func (h *nfqueueHandler) Start() error { + h.ctx, h.cancel = context.WithCancel(context.Background()) config := nfqueue.Config{ NfQueue: h.queue, MaxPacketLen: 0xFFFF, MaxQueueLen: 4096, Copymode: nfqueue.NfQnlCopyPacket, AfFamily: unix.AF_UNSPEC, - Flags: nfqueue.NfQaCfgFlagFailOpen | nfqueue.NfQaCfgFlagGSO, + Flags: nfqueue.NfQaCfgFlagGSO, } nfq, err := nfqueue.Open(&config) @@ -259,12 +265,11 @@ func (h *nfqueueHandler) handlePacket(attr nfqueue.Attribute) int { return 0 } - packetID := *attr.PacketID payload := *attr.Payload packet, loaded := parsePreMatchPacket(payload) if !loaded { - h.setVerdict(packetID, nfqueue.NfAccept, 0) + h.setVerdict(attr, nfqueue.NfAccept, 0) return 0 } @@ -275,27 +280,25 @@ func (h *nfqueueHandler) handlePacket(attr nfqueue.Attribute) int { packet.firstPacket, ) - // nf_reinject: NF_REPEAT re-runs the queueing chain from its first rule, - // while NF_ACCEPT continues at the next hook, skipping the remaining - // rules of the chain. switch verdict.Action { case ActionBypass: - h.setVerdict(packetID, nfqueue.NfRepeat, h.outputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.outputMark) case ActionReject: if packet.protocol == uint8(unix.IPPROTO_TCP) { - h.setVerdict(packetID, nfqueue.NfRepeat, h.resetMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.resetMark) } else { - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) } case ActionDrop: - h.setVerdict(packetID, nfqueue.NfDrop, 0) + h.setVerdict(attr, nfqueue.NfDrop, 0) case ActionFlow, ActionHijackDNS: - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) default: if packet.protocol == uint8(unix.IPPROTO_TCP) { - h.setVerdict(packetID, nfqueue.NfAccept, 0) + acceptVerdict, acceptMark := h.acceptVerdict(packet) + h.setVerdict(attr, acceptVerdict, acceptMark) } else { - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) } } diff --git a/redirect.go b/redirect.go index dcf3e720..17ef6182 100644 --- a/redirect.go +++ b/redirect.go @@ -2,36 +2,48 @@ package tun import ( "context" + "net/netip" "github.com/sagernet/sing/common/control" "github.com/sagernet/sing/common/logger" - - "go4.org/netipx" + N "github.com/sagernet/sing/common/network" ) const ( DefaultAutoRedirectInputMark = 0x2023 DefaultAutoRedirectOutputMark = 0x2024 DefaultAutoRedirectResetMark = 0x2025 + DefaultAutoRedirectTProxyMark = 0x2026 DefaultAutoRedirectNFQueue = 100 + + DefaultAutoRedirectInputMarkAndroid = 0x400000 + DefaultAutoRedirectOutputMarkAndroid = 0x200000 + DefaultAutoRedirectResetMarkAndroid = 0x600000 + DefaultAutoRedirectTProxyMarkAndroid = 0x800000 ) type AutoRedirect interface { Start() error Close() error - UpdateRouteAddressSet() + UpdateRouteAddressSet() error +} + +type AutoRedirectHandler interface { + JudgeFlow(network uint8, source netip.AddrPort, destination netip.AddrPort, firstPacket []byte) FlowVerdict + N.TCPConnectionHandlerEx } type AutoRedirectOptions struct { - TunOptions *Options - Context context.Context - Handler Handler - Logger logger.Logger - NetworkMonitor NetworkUpdateMonitor - InterfaceFinder control.InterfaceFinder - TableName string - DisableNFTables bool - CustomRedirectPort func() int - RouteAddressSet *[]*netipx.IPSet - RouteExcludeAddressSet *[]*netipx.IPSet + TunOptions *Options + Context context.Context + Handler AutoRedirectHandler + Logger logger.Logger + NetworkMonitor NetworkUpdateMonitor + InterfaceFinder control.InterfaceFinder + TableName string + DisableNFTables bool + CustomRedirectPort func() int + CustomRedirectListenerFD func() (int, error) + RouteAddressSet func() (include []netip.Prefix, exclude []netip.Prefix, err error) + AndroidVPNService bool } diff --git a/redirect_iptables.go b/redirect_iptables.go index 1b7948aa..1f228e7b 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -3,61 +3,690 @@ package tun import ( + "net" + "net/netip" + "os" "os/exec" + "runtime" + "slices" + "strconv" "strings" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" F "github.com/sagernet/sing/common/format" + "github.com/sagernet/sing/common/ranges" ) -func (r *autoRedirect) setupIPTables() error { - if r.enableIPv4 { - err := r.setupIPTablesForFamily(r.iptablesPath) - if err != nil { - return E.Cause(err, "setup iptables") +const ( + iptablesTableNAT = "nat" + iptablesTableMangle = "mangle" + iptablesTableFilter = "filter" +) + +type iptablesHook int + +const ( + iptablesHookOutput iptablesHook = iota + iptablesHookPrerouting +) + +type iptablesChainKind int + +const ( + iptablesKindNAT iptablesChainKind = iota + iptablesKindRoute + iptablesKindFilter +) + +type iptablesFamily struct { + path string + isIPv6 bool + tproxy bool + dnsServer netip.Addr + chainCounter int +} + +type iptablesMatch struct { + positive []string + negative []string +} + +type iptablesLocalChain struct { + family *iptablesFamily + table string + name string + next string + hook iptablesHook + kind iptablesChainKind + dnsHijack bool +} + +type iptablesInsert struct { + table string + base string + args []string +} + +type iptablesBuilder struct { + redirect *autoRedirect + family *iptablesFamily + table string + name string + chain string + err error +} + +func iptablesMark(value uint32, mask uint32) string { + return iptablesHex(value) + "/" + iptablesHex(mask) +} + +func iptablesHex(value uint32) string { + return "0x" + strconv.FormatUint(uint64(value), 16) +} + +func iptablesUIDRange(uidRange ranges.Range[uint32]) string { + if uidRange.Start == uidRange.End { + return F.ToString(uidRange.Start) + } + return F.ToString(uidRange.Start, "-", uidRange.End) +} + +func (r *autoRedirect) iptablesChain(family *iptablesFamily, table string, name string) *iptablesBuilder { + builder := &iptablesBuilder{ + redirect: r, + family: family, + table: table, + name: name, + chain: name, + } + builder.err = iptablesRun(family.path, "-t", table, "-N", name) + return builder +} + +func (b *iptablesBuilder) add(args ...string) { + if b.err != nil { + return + } + b.err = iptablesRun(b.family.path, slices.Concat([]string{"-t", b.table, "-A", b.chain}, args)...) +} + +func (b *iptablesBuilder) newChain(prefix string) string { + b.family.chainCounter++ + name := F.ToString(b.redirect.tableName, "-", prefix, "-", b.family.chainCounter) + if b.err == nil { + b.err = iptablesRun(b.family.path, "-t", b.table, "-N", name) + } + return name +} + +func (b *iptablesBuilder) includeOnly(matches []iptablesMatch) { + if len(matches) == 0 { + return + } + if len(matches) == 1 && len(matches[0].negative) > 0 { + b.add(slices.Concat(matches[0].negative, []string{"-j", "RETURN"})...) + return + } + next := b.newChain("include") + for _, match := range matches { + b.add(slices.Concat(match.positive, []string{"-j", next})...) + } + b.chain = next +} + +func (b *iptablesBuilder) branch(prefix string) *iptablesBuilder { + name := b.newChain(prefix) + b.add("-j", name) + return &iptablesBuilder{ + redirect: b.redirect, + family: b.family, + table: b.table, + name: name, + chain: name, + err: b.err, + } +} + +func (b *iptablesBuilder) absorb(child *iptablesBuilder) { + if b.err == nil { + b.err = child.err + } +} + +func (b *iptablesBuilder) localGate(hook iptablesHook, kind iptablesChainKind, dnsHijack bool) { + next := b.newChain("cont") + name := b.newChain("local") + b.add("-j", name) + chain := iptablesLocalChain{ + family: b.family, + table: b.table, + name: name, + next: next, + hook: hook, + kind: kind, + dnsHijack: dnsHijack, + } + if b.err == nil { + b.err = b.redirect.iptablesFillLocalChain(chain, b.redirect.localAddresses) + } + b.redirect.iptablesLocalChains = append(b.redirect.iptablesLocalChains, chain) + b.chain = next +} + +func (b *iptablesBuilder) addVPNServiceGate() { + if b.redirect.androidVPNService { + b.add("!", "-o", b.redirect.tunOptions.Name, "-j", "RETURN") + } +} + +func (b *iptablesBuilder) addRedirect() { + loopbackAddresses := b.redirect.tunOptions.Inet4LoopbackAddress + if b.family.isIPv6 { + loopbackAddresses = b.redirect.tunOptions.Inet6LoopbackAddress + } + redirect := []string{"-p", "tcp", "-j", "REDIRECT", "--to-ports", strconv.Itoa(int(b.redirect.redirectPort()))} + if len(loopbackAddresses) == 0 { + b.add(redirect...) + return + } + name := b.newChain("redirect") + b.add("-j", name) + parent := b.chain + b.chain = name + for _, address := range loopbackAddresses { + b.add("-d", address.String(), "-j", "RETURN") + } + b.add(redirect...) + b.chain = parent +} + +func (r *autoRedirect) findLocalAddresses() ([]netip.Prefix, error) { + err := r.interfaceFinder.Update() + if err != nil { + return nil, E.Cause(err, "update interfaces") + } + return common.Uniq(common.FlatMap(r.interfaceFinder.Interfaces(), func(it control.Interface) []netip.Prefix { + return common.Map(common.Filter(it.Addresses, func(prefix netip.Prefix) bool { + return it.Name == "lo" || prefix.Addr().IsGlobalUnicast() + }), netip.Prefix.Masked) + })), nil +} + +func (r *autoRedirect) iptablesFillLocalChain(chain iptablesLocalChain, addresses []netip.Prefix) error { + prefixes := common.Filter(addresses, func(it netip.Prefix) bool { + if chain.family.isIPv6 { + return it.Addr().Is6() && !it.Addr().Is4In6() + } + return it.Addr().Is4() + }) + if chain.dnsHijack { + action := func(protocol string) []string { + if chain.kind == iptablesKindFilter { + return []string{"-j", "RETURN"} + } + if chain.kind == iptablesKindNAT && !chain.family.isIPv6 && protocol == "udp" { + return []string{"-j", "DNAT", "--to-destination", chain.family.dnsServer.String()} + } + return []string{"-g", chain.next} + } + if chain.hook == iptablesHookPrerouting { + for _, prefix := range prefixes { + for _, protocol := range []string{"udp", "tcp"} { + err := iptablesRun(chain.family.path, slices.Concat([]string{ + "-t", chain.table, "-A", chain.name, + "-s", prefix.String(), "-p", protocol, "--dport", "53", + }, action(protocol))...) + if err != nil { + return err + } + } + } + } else { + for _, protocol := range []string{"udp", "tcp"} { + err := iptablesRun(chain.family.path, slices.Concat([]string{ + "-t", chain.table, "-A", chain.name, + "!", "-o", "lo", "-p", protocol, "--dport", "53", + }, action(protocol))...) + if err != nil { + return err + } + } } } - if r.enableIPv6 { - err := r.setupIPTablesForFamily(r.ip6tablesPath) + for _, prefix := range prefixes { + err := iptablesRun(chain.family.path, "-t", chain.table, "-A", chain.name, "-d", prefix.String(), "-j", "RETURN") if err != nil { - return E.Cause(err, "setup ip6tables") + return err } } - return nil + return iptablesRun(chain.family.path, "-t", chain.table, "-A", chain.name, "-j", chain.next) } -func (r *autoRedirect) setupIPTablesForFamily(iptablesPath string) error { - tableNameOutput := r.tableName + "-output" - redirectPort := r.redirectPort() - // OUTPUT - err := r.runShell(iptablesPath, "-t nat -N", tableNameOutput) +func (r *autoRedirect) updateIPTablesLocalAddresses() error { + localAddresses, err := r.findLocalAddresses() if err != nil { return err } - err = r.runShell(iptablesPath, "-t nat -A", tableNameOutput, - "-p tcp -o", r.tunOptions.Name, - "-j REDIRECT --to-ports", redirectPort) - if err != nil { - return err + if slices.Equal(localAddresses, r.localAddresses) { + return nil + } + r.localAddresses = localAddresses + for _, chain := range r.iptablesLocalChains { + err = iptablesRun(chain.family.path, "-t", chain.table, "-F", chain.name) + if err != nil { + return err + } + err = r.iptablesFillLocalChain(chain, localAddresses) + if err != nil { + return err + } + } + return nil +} + +func (r *autoRedirect) iptablesAddExcludeRules(builder *iptablesBuilder, hook iptablesHook, kind iptablesChainKind, dnsHijack bool) { + options := r.tunOptions + mask := r.effectiveMarkMask() + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + if hook == iptablesHookOutput && kind != iptablesKindFilter { + builder.add("-m", "mark", "--mark", outputMark, "-j", "RETURN") + if kind == iptablesKindRoute { + builder.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + } + } + if kind == iptablesKindNAT { + builder.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + builder.add("-m", "connmark", "--mark", iptablesMark(options.AutoRedirectInputMark, mask), "-j", "RETURN") + } + if hook == iptablesHookPrerouting { + builder.add("-i", options.Name, "-j", "RETURN") + builder.includeOnly(common.Map(options.IncludeInterface, func(it string) iptablesMatch { + return iptablesMatch{positive: []string{"-i", it}, negative: []string{"!", "-i", it}} + })) + for _, name := range options.ExcludeInterface { + builder.add("-i", name, "-j", "RETURN") + } + // xt_mac reports no match on interfaces without an ethernet header + // before applying the invert flag, so an inverted form would keep + // non-ethernet ingress included. + builder.includeOnly(common.Map(options.IncludeMACAddress, func(it net.HardwareAddr) iptablesMatch { + return iptablesMatch{positive: []string{"-m", "mac", "--mac-source", it.String()}} + })) + for _, address := range options.ExcludeMACAddress { + builder.add("-m", "mac", "--mac-source", address.String(), "-j", "RETURN") + } + } else { + builder.includeOnly(common.Map(options.IncludeUID, func(it ranges.Range[uint32]) iptablesMatch { + return iptablesMatch{ + positive: []string{"-m", "owner", "--uid-owner", iptablesUIDRange(it)}, + negative: []string{"-m", "owner", "!", "--uid-owner", iptablesUIDRange(it)}, + } + })) + for _, uidRange := range options.ExcludeUID { + builder.add("-m", "owner", "--uid-owner", iptablesUIDRange(uidRange), "-j", "RETURN") + } + } + routeAddress := options.Inet4RouteAddress + routeExcludeAddress := options.Inet4RouteExcludeAddress + if builder.family.isIPv6 { + routeAddress = options.Inet6RouteAddress + routeExcludeAddress = options.Inet6RouteExcludeAddress + } + builder.includeOnly(common.Map(routeAddress, func(it netip.Prefix) iptablesMatch { + return iptablesMatch{positive: []string{"-d", it.String()}, negative: []string{"!", "-d", it.String()}} + })) + for _, prefix := range routeExcludeAddress { + builder.add("-d", prefix.String(), "-j", "RETURN") + } + builder.localGate(hook, kind, dnsHijack) +} + +func (r *autoRedirect) iptablesAddPreMatchRules(builder *iptablesBuilder, hook iptablesHook, dnsHijack bool) { + options := r.tunOptions + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + resetMark := iptablesMark(r.tunOptions.AutoRedirectResetMark, mask) + if hook == iptablesHookOutput { + if r.androidVPNService { + builder.add("!", "-o", options.Name, "-j", "RETURN") + } + } else { + builder.add("-i", "lo", "-j", "RETURN") + builder.add("-i", options.Name, "-j", "RETURN") + } + builder.add("-m", "conntrack", "--ctdir", "REPLY", "-j", "RETURN") + if builder.family.tproxy { + tproxyMark := iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask) + builder.add("-m", "connmark", "--mark", tproxyMark, "-j", "RETURN") + builder.add("-m", "mark", "--mark", tproxyMark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "mark", "--mark", tproxyMark, "-j", "RETURN") + } + for _, mark := range []string{outputMark, inputMark} { + builder.add("-m", "mark", "--mark", mark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "mark", "--mark", mark, "-j", "RETURN") + builder.add("-m", "connmark", "--mark", mark, "-j", "CONNMARK", "--restore-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "connmark", "--mark", mark, "-j", "RETURN") + } + builder.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + builder.add("-p", "tcp", "!", "--tcp-flags", "SYN,ACK", "SYN", "-j", "RETURN") + r.iptablesAddExcludeRules(builder, hook, iptablesKindFilter, dnsHijack) + queue := []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(int(r.effectiveNFQueue())), "--queue-bypass"} + builder.add(slices.Concat([]string{"-p", "tcp"}, queue)...) + builder.add(slices.Concat([]string{"-p", "udp"}, queue)...) + if builder.family.isIPv6 { + builder.add(slices.Concat([]string{"-p", "icmpv6", "--icmpv6-type", strconv.Itoa(int(header.ICMPv6EchoRequest))}, queue)...) + } else { + builder.add(slices.Concat([]string{"-p", "icmp", "--icmp-type", strconv.Itoa(int(header.ICMPv4Echo))}, queue)...) } - err = r.runShell(iptablesPath, "-t nat -I OUTPUT -j", tableNameOutput) +} + +func (r *autoRedirect) iptablesAddMarkRules(builder *iptablesBuilder) { + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + builder.add("-j", "MARK", "--set-xmark", inputMark) + builder.add("-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) +} + +func (r *autoRedirect) iptablesAddLoopbackReroute(builder *iptablesBuilder, hook iptablesHook) { + loopbackAddresses := r.tunOptions.Inet4LoopbackAddress + if builder.family.isIPv6 { + loopbackAddresses = r.tunOptions.Inet6LoopbackAddress + } + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + for _, address := range loopbackAddresses { + builder.add("-p", "tcp", "-d", address.String(), "-m", "mark", "!", "--mark", inputMark, "-j", "MARK", "--set-xmark", inputMark) + if hook == iptablesHookOutput { + builder.add("-p", "tcp", "-d", address.String(), "-m", "mark", "--mark", inputMark, + "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + } + } +} + +func (r *autoRedirect) setupIPTables() error { + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + localAddresses, err := r.findLocalAddresses() if err != nil { return err } - if r.redirectServer != nil { - tableNameInput := r.tableName + "-input" - err = r.runShell(iptablesPath, "-t filter -N", tableNameInput) + r.localAddresses = localAddresses + var families []*iptablesFamily + if r.enableIPv4 { + families = append(families, &iptablesFamily{path: r.iptablesPath}) + } + if r.enableIPv6 { + families = append(families, &iptablesFamily{path: r.ip6tablesPath, isIPv6: true, tproxy: r.tproxyEnabled()}) + } + for _, family := range families { + err = r.setupIPTablesForFamily(family) if err != nil { return err } - err = r.runShell(iptablesPath, "-t filter -A", tableNameInput, - "-p tcp --dport", redirectPort, - "-m conntrack ! --ctstate DNAT", - "-j REJECT --reject-with tcp-reset") - if err != nil { - return err + } + return r.setupIPTablesUnreachable() +} + +func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { + options := r.tunOptions + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + resetMark := iptablesMark(r.tunOptions.AutoRedirectResetMark, mask) + loopbackAddresses := options.Inet4LoopbackAddress + if family.isIPv6 { + loopbackAddresses = options.Inet6LoopbackAddress + } + dnsHijack := options.DNSModeOrDefault() == DNSModeHijack + if dnsHijack { + var ( + dnsServers []netip.Addr + dnsErr error + ) + if family.isIPv6 { + dnsServers, dnsErr = options.Inet6DNSAddress() + } else { + dnsServers, dnsErr = options.Inet4DNSAddress() + } + if dnsErr != nil { + return dnsErr + } + if len(dnsServers) == 0 { + dnsHijack = false + } else { + family.dnsServer = dnsServers[0] + } + } + markProtocols := iptablesMarkProtocols(family) + var inserts []iptablesInsert + + if len(loopbackAddresses) > 0 { + if !r.shouldSkipOutputChain() { + outputLoopback := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-loopback") + r.iptablesAddLoopbackReroute(outputLoopback, iptablesHookOutput) + if outputLoopback.err != nil { + return outputLoopback.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-j", outputLoopback.name}}) + } + preroutingLoopback := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-loopback") + r.iptablesAddLoopbackReroute(preroutingLoopback, iptablesHookPrerouting) + if preroutingLoopback.err != nil { + return preroutingLoopback.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-j", preroutingLoopback.name}}) + } + + if !r.shouldSkipOutputChain() { + outputPreMatch := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-prematch") + r.iptablesAddPreMatchRules(outputPreMatch, iptablesHookOutput, dnsHijack) + if outputPreMatch.err != nil { + return outputPreMatch.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-j", outputPreMatch.name}}) + } + preroutingPreMatch := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-prematch") + r.iptablesAddPreMatchRules(preroutingPreMatch, iptablesHookPrerouting, dnsHijack) + if preroutingPreMatch.err != nil { + return preroutingPreMatch.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-j", preroutingPreMatch.name}}) + + if !r.shouldSkipOutputChain() { + outputMarkChain := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-mark") + outputMarkChain.addVPNServiceGate() + if family.isIPv6 && !family.tproxy { + outputMarkChain.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + } + r.iptablesAddExcludeRules(outputMarkChain, iptablesHookOutput, iptablesKindRoute, dnsHijack) + r.iptablesAddMarkRules(outputMarkChain) + if outputMarkChain.err != nil { + return outputMarkChain.err + } + for _, protocol := range markProtocols { + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", protocol, "-j", outputMarkChain.name}}) + } + } + preroutingMarkChain := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-mark") + preroutingMarkChain.add("-i", options.Name, "-j", "RETURN") + if family.isIPv6 && !family.tproxy { + preroutingMarkChain.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + } + preroutingMarkChain.add("-m", "connmark", "--mark", inputMark, "-j", "MARK", "--set-xmark", inputMark) + preroutingMarkChain.add("-m", "connmark", "--mark", inputMark, "-j", "RETURN") + preroutingMarkChain.add("-m", "connmark", "--mark", outputMark, "-j", "MARK", "--set-xmark", outputMark) + preroutingMarkChain.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + include := preroutingMarkChain.branch("include") + r.iptablesAddExcludeRules(include, iptablesHookPrerouting, iptablesKindRoute, dnsHijack) + r.iptablesAddMarkRules(include) + preroutingMarkChain.absorb(include) + preroutingMarkChain.add("-m", "mark", "--mark", inputMark, "-j", "RETURN") + preroutingMarkChain.add("-j", "MARK", "--set-xmark", outputMark) + preroutingMarkChain.add("-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + if preroutingMarkChain.err != nil { + return preroutingMarkChain.err + } + for _, protocol := range markProtocols { + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-p", protocol, "-j", preroutingMarkChain.name}}) + } + if r.androidVPNService && !family.isIPv6 { + // A TCP flow addressed to the router itself leaves the pre-match chain + // without a connmark, the accepted socket inherits the unmarked SYN + // through tcp_fwmark_accept, and netd's VPN rule then routes its replies + // into the tun. + preroutingLocal := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-local") + preroutingLocal.add("-i", options.Name, "-j", "RETURN") + preroutingLocal.add("-m", "connmark", "!", "--mark", iptablesMark(0, mask), "-j", "RETURN") + preroutingLocal.add("-j", "MARK", "--set-xmark", outputMark) + localInclude := preroutingLocal.branch("include") + r.iptablesAddExcludeRules(localInclude, iptablesHookPrerouting, iptablesKindRoute, dnsHijack) + localInclude.add("-j", "MARK", "--set-xmark", iptablesMark(0, mask)) + preroutingLocal.absorb(localInclude) + preroutingLocal.add("-m", "mark", "--mark", outputMark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + if preroutingLocal.err != nil { + return preroutingLocal.err + } + inserts = append(inserts, iptablesInsert{ + iptablesTableMangle, "PREROUTING", + []string{"-p", "tcp", "-m", "mark", "--mark", iptablesMark(0, mask), "-j", preroutingLocal.name}, + }) + } + + if !family.isIPv6 { + if !r.shouldSkipOutputChain() { + outputNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-output") + outputNAT.addVPNServiceGate() + r.iptablesAddExcludeRules(outputNAT, iptablesHookOutput, iptablesKindNAT, dnsHijack) + outputNAT.addRedirect() + if outputNAT.err != nil { + return outputNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "OUTPUT", []string{"-j", outputNAT.name}}) + } + preroutingNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-prerouting") + preroutingNAT.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + r.iptablesAddExcludeRules(preroutingNAT, iptablesHookPrerouting, iptablesKindNAT, dnsHijack) + preroutingNAT.addRedirect() + r.iptablesAddMarkRules(preroutingNAT) + if preroutingNAT.err != nil { + return preroutingNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "PREROUTING", []string{"-j", preroutingNAT.name}}) + if r.androidVPNService { + postroutingNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-postrouting") + for _, prefix := range options.Inet4Address { + postroutingNAT.add("-s", prefix.Masked().String(), "!", "-o", options.Name, "-m", "mark", "--mark", outputMark, "-j", "MASQUERADE") + } + if postroutingNAT.err != nil { + return postroutingNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "POSTROUTING", []string{"-j", postroutingNAT.name}}) + } + } + + if r.androidVPNService && (!family.isIPv6 || family.tproxy) { + // The redirect server runs in the app process, and without the + // protectedFromVpn bit netd's VPN rule routes its replies back + // into the tun instead of delivering them over loopback. + outputProtect := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-protect") + if family.isIPv6 { + outputProtect.add("-m", "connmark", "--mark", iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask), + "-m", "conntrack", "--ctdir", "REPLY", + "-j", "MARK", "--set-xmark", iptablesMark(androidSecureVPNMask, androidSecureVPNMask)) + } else { + outputProtect.add("-p", "tcp", "--sport", strconv.Itoa(int(r.redirectPort())), + "-j", "MARK", "--set-xmark", iptablesMark(androidSecureVPNMask, androidSecureVPNMask)) + } + if outputProtect.err != nil { + return outputProtect.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", "tcp", "-j", outputProtect.name}}) + } + + if family.tproxy { + tproxyMark := iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask) + tproxyAction := []string{ + "-p", "tcp", "-j", "TPROXY", + "--on-port", strconv.Itoa(int(r.redirectPort())), "--tproxy-mark", tproxyMark, + } + if !r.shouldSkipOutputChain() { + outputTProxy := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-tproxy") + // Under VpnService the app's own sockets carry the protectedFromVpn bit instead of + // the routing mark, so netd routes them past the tun. + outputTProxy.addVPNServiceGate() + outputTProxy.add("-m", "mark", "--mark", resetMark, "-j", "RETURN") + outputTProxy.add("-m", "connmark", "--mark", tproxyMark, "-m", "conntrack", "--ctdir", "ORIGINAL", + "-j", "MARK", "--set-xmark", tproxyMark) + outputTProxy.add("-m", "mark", "--mark", tproxyMark, "-j", "RETURN") + r.iptablesAddExcludeRules(outputTProxy, iptablesHookOutput, iptablesKindNAT, dnsHijack) + outputTProxy.add("-j", "MARK", "--set-xmark", tproxyMark) + outputTProxy.add("-j", "CONNMARK", "--set-xmark", tproxyMark) + if outputTProxy.err != nil { + return outputTProxy.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", "tcp", "-j", outputTProxy.name}}) + } + preroutingTProxy := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-tproxy") + preroutingTProxy.add(slices.Concat([]string{"-m", "mark", "--mark", tproxyMark}, tproxyAction)...) + preroutingTProxy.add("-m", "mark", "--mark", outputMark, "-j", "RETURN") + preroutingTProxy.add("-m", "mark", "--mark", resetMark, "-j", "RETURN") + preroutingTProxy.add(slices.Concat([]string{ + "-m", "connmark", "--mark", tproxyMark, + "-m", "conntrack", "--ctdir", "ORIGINAL", + }, tproxyAction)...) + r.iptablesAddExcludeRules(preroutingTProxy, iptablesHookPrerouting, iptablesKindNAT, dnsHijack) + preroutingTProxy.add("-j", "CONNMARK", "--set-xmark", tproxyMark) + preroutingTProxy.add(tproxyAction...) + if preroutingTProxy.err != nil { + return preroutingTProxy.err } - err = r.runShell(iptablesPath, "-t filter -I INPUT -j", tableNameInput) + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-p", "tcp", "-j", preroutingTProxy.name}}) + } + + input := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-input") + input.add("-p", "tcp", "--dport", strconv.Itoa(int(r.redirectPort())), + "-m", "conntrack", "!", "--ctstate", "DNAT", + "-j", "REJECT", "--reject-with", "tcp-reset") + if input.err != nil { + return input.err + } + inserts = append(inserts, iptablesInsert{iptablesTableFilter, "INPUT", []string{"-j", input.name}}) + + reset := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-reset") + reset.add("-p", "tcp", "!", "--tcp-flags", "RST", "RST", "-m", "mark", "--mark", resetMark, "-j", "REJECT", "--reject-with", "tcp-reset") + if reset.err != nil { + return reset.err + } + inserts = append(inserts, + iptablesInsert{iptablesTableFilter, "INPUT", []string{"-j", reset.name}}, + iptablesInsert{iptablesTableFilter, "OUTPUT", []string{"-j", reset.name}}, + iptablesInsert{iptablesTableFilter, "FORWARD", []string{"-j", reset.name}}, + ) + + // netd ends its tetherctrl_FORWARD chain with an unconditional DROP and + // only accepts its own downstream/upstream interface pairs, so forwarded + // flows marked into the tun need their own accept. + if runtime.GOOS == "android" { + forward := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-forward") + forward.add("-o", options.Name, "-m", "mark", "--mark", inputMark, "-j", "ACCEPT") + forward.add("-i", options.Name, "-m", "connmark", "--mark", inputMark, "-j", "ACCEPT") + if forward.err != nil { + return forward.err + } + inserts = append(inserts, iptablesInsert{iptablesTableFilter, "FORWARD", []string{"-j", forward.name}}) + } + + for index := len(inserts) - 1; index >= 0; index-- { + insert := inserts[index] + err := iptablesRun(family.path, slices.Concat([]string{"-t", insert.table, "-I", insert.base}, insert.args)...) if err != nil { return err } @@ -65,39 +694,142 @@ func (r *autoRedirect) setupIPTablesForFamily(iptablesPath string) error { return nil } +func iptablesMarkProtocols(family *iptablesFamily) []string { + var protocols []string + if family.isIPv6 { + protocols = []string{"udp", "icmpv6"} + } else { + protocols = []string{"udp", "icmp"} + } + if family.isIPv6 && !family.tproxy { + protocols = append(protocols, "tcp") + } + return protocols +} + +func (r *autoRedirect) setupIPTablesUnreachable() error { + if !r.tunOptions.StrictRoute || r.enableIPv4 == r.enableIPv6 { + return nil + } + path, err := findIPTablesBinary(r.enableIPv4) + if err != nil { + r.logger.Warn("strict route is not available for the disabled address family: ", err) + return nil + } + family := &iptablesFamily{path: path, isIPv6: r.enableIPv4} + outputName := r.tableName + "-unreachable-output" + outputBuilder := r.iptablesChain(family, iptablesTableFilter, outputName) + r.iptablesAddExcludeRules(outputBuilder, iptablesHookOutput, iptablesKindFilter, false) + outputBuilder.add("-j", "REJECT") + if outputBuilder.err != nil { + return outputBuilder.err + } + forwardName := r.tableName + "-unreachable-forward" + forwardBuilder := r.iptablesChain(family, iptablesTableFilter, forwardName) + r.iptablesAddExcludeRules(forwardBuilder, iptablesHookPrerouting, iptablesKindFilter, false) + forwardBuilder.add("-j", "REJECT") + if forwardBuilder.err != nil { + return forwardBuilder.err + } + err = iptablesRun(path, "-t", iptablesTableFilter, "-I", "OUTPUT", "-j", outputName) + if err != nil { + return err + } + return iptablesRun(path, "-t", iptablesTableFilter, "-I", "FORWARD", "-j", forwardName) +} + +func findIPTablesBinary(isIPv6 bool) (string, error) { + name := "iptables" + if isIPv6 { + name = "ip6tables" + } + if runtime.GOOS == "android" { + path := "/system/bin/" + name + _, err := os.Stat(path) + if err != nil { + return "", err + } + return path, nil + } + return exec.LookPath(name) +} + func (r *autoRedirect) cleanupIPTables() { - if r.enableIPv4 { - r.cleanupIPTablesForFamily(r.iptablesPath) + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + r.iptablesLocalChains = nil + var paths []string + for _, isIPv6 := range []bool{false, true} { + path, err := findIPTablesBinary(isIPv6) + if err == nil { + paths = append(paths, path) + } } - if r.enableIPv6 { - r.cleanupIPTablesForFamily(r.ip6tablesPath) + for _, path := range common.Uniq(paths) { + r.cleanupIPTablesForFamily(path) } } func (r *autoRedirect) cleanupIPTablesForFamily(iptablesPath string) { - tableNameOutput := r.tableName + "-output" - tableNameInput := r.tableName + "-input" - - _ = r.runShell(iptablesPath, "-t nat -D OUTPUT -j", tableNameOutput) - _ = r.runShell(iptablesPath, "-t nat -F", tableNameOutput) - _ = r.runShell(iptablesPath, "-t nat -X", tableNameOutput) - _ = r.runShell(iptablesPath, "-t filter -D INPUT -j", tableNameInput) - _ = r.runShell(iptablesPath, "-t filter -F", tableNameInput) - _ = r.runShell(iptablesPath, "-t filter -X", tableNameInput) -} - -func (r *autoRedirect) runShell(commands ...any) error { - commandStr := strings.Join(F.MapToString(commands), " ") - var command *exec.Cmd - if r.androidSu { - command = exec.Command(r.suPath, "-c", commandStr) - } else { - commandArray := strings.Split(commandStr, " ") - command = exec.Command(commandArray[0], commandArray[1:]...) + prefix := r.tableName + "-" + for _, table := range []string{iptablesTableNAT, iptablesTableMangle, iptablesTableFilter} { + output, err := iptablesRunOutput(iptablesPath, "-t", table, "-S") + if err != nil { + continue + } + var chains []string + var references [][]string + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + switch fields[0] { + case "-N": + if strings.HasPrefix(fields[1], prefix) { + chains = append(chains, fields[1]) + } + case "-A": + if strings.HasPrefix(fields[1], prefix) { + continue + } + for index := 2; index < len(fields)-1; index++ { + if fields[index] == "-j" && strings.HasPrefix(fields[index+1], prefix) { + references = append(references, fields[1:]) + break + } + } + } + } + for _, reference := range references { + _ = iptablesRun(iptablesPath, slices.Concat([]string{"-t", table, "-D"}, reference)...) + } + for _, chain := range chains { + _ = iptablesRun(iptablesPath, "-t", table, "-F", chain) + } + for _, chain := range chains { + _ = iptablesRun(iptablesPath, "-t", table, "-X", chain) + } } - combinedOutput, err := command.CombinedOutput() +} + +// netd runs its own iptables commands whenever a network (including the VPN +// interface) appears or disappears, so a run without -w fails with "Can't +// lock /system/etc/xtables.lock" while netd holds the lock. +func iptablesRun(path string, args ...string) error { + args = append([]string{"-w"}, args...) + combinedOutput, err := exec.Command(path, args...).CombinedOutput() if err != nil { - return E.Extend(err, F.ToString(commandStr, ": ", string(combinedOutput))) + return E.Extend(err, F.ToString(path, " ", strings.Join(args, " "), ": ", strings.TrimSpace(string(combinedOutput)))) } return nil } + +func iptablesRunOutput(path string, args ...string) (string, error) { + args = append([]string{"-w"}, args...) + output, err := exec.Command(path, args...).Output() + if err != nil { + return "", E.Extend(err, F.ToString(path, " ", strings.Join(args, " "))) + } + return string(output), nil +} diff --git a/redirect_linux.go b/redirect_linux.go index dd8c61f4..66e7ebc5 100644 --- a/redirect_linux.go +++ b/redirect_linux.go @@ -5,193 +5,127 @@ import ( "net/netip" "os" "os/exec" - "path/filepath" "runtime" - "strings" "sync" + "syscall" + "github.com/sagernet/netlink" "github.com/sagernet/nftables" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/logger" - M "github.com/sagernet/sing/common/metadata" "github.com/sagernet/sing/common/x/list" - "go4.org/netipx" + "golang.org/x/sys/unix" ) type autoRedirect struct { - tunOptions *Options - ctx context.Context - handler Handler - logger logger.Logger - tableName string - networkMonitor NetworkUpdateMonitor - ownedNetworkMonitor bool - networkListener *list.Element[NetworkUpdateCallback] - interfaceFinder control.InterfaceFinder - localAddresses []netip.Prefix - customRedirectPortFunc func() int - customRedirectPort int - redirectServer *redirectServer - enableIPv4 bool - enableIPv6 bool - iptablesPath string - ip6tablesPath string - useNFTables bool - androidSu bool - suPath string - routeAddressSet *[]*netipx.IPSet - routeExcludeAddressSet *[]*netipx.IPSet - nfqueueHandler *nfqueueHandler - nfqueueEnabled bool - redirectRouteTableIndex int - redirectRouteAccess sync.Mutex - redirectRoutesActive bool - dockerFirewallMonitor *nftables.Monitor - dockerFirewallDone chan struct{} + tunOptions *Options + ctx context.Context + handler AutoRedirectHandler + logger logger.Logger + tableName string + networkMonitor NetworkUpdateMonitor + ownedNetworkMonitor bool + networkListener *list.Element[NetworkUpdateCallback] + interfaceListener *list.Element[DefaultInterfaceUpdateCallback] + interfaceFinder control.InterfaceFinder + localAddresses []netip.Prefix + customRedirectPortFunc func() int + customRedirectPort int + customRedirectListenerFD func() (int, error) + customRedirectTransparent bool + routeAddressSet func() ([]netip.Prefix, []netip.Prefix, error) + bypassRouteTableIndex int + bypassRouteAccess sync.Mutex + bypassRouteActive bool + bypassIncludeAddresses []netip.Prefix + bypassExcludeAddresses []netip.Prefix + redirectServer *RedirectServer + enableIPv4 bool + enableIPv6 bool + iptablesPath string + ip6tablesPath string + useNFTables bool + androidVPNService bool + nfqueueHandler *nfqueueHandler + iptablesAccess sync.Mutex + iptablesLocalChains []iptablesLocalChain + redirectRouteTableIndex int + tproxyRouteTableIndex int + redirectRouteAccess sync.Mutex + redirectRoutesActive bool + androidVPNServiceRuleAccess sync.Mutex + androidVPNServiceRules []*netlink.Rule + androidVPNServiceRulesActive bool + dockerFirewallMonitor *nftables.Monitor + dockerFirewallDone chan struct{} } func NewAutoRedirect(options AutoRedirectOptions) (AutoRedirect, error) { - if options.TunOptions.AutoRedirectInputMark == 0 { - options.TunOptions.AutoRedirectInputMark = DefaultAutoRedirectInputMark - } r := &autoRedirect{ - tunOptions: options.TunOptions, - ctx: options.Context, - handler: options.Handler, - logger: options.Logger, - networkMonitor: options.NetworkMonitor, - interfaceFinder: options.InterfaceFinder, - tableName: options.TableName, - useNFTables: runtime.GOOS != "android" && !options.DisableNFTables, - customRedirectPortFunc: options.CustomRedirectPort, - routeAddressSet: options.RouteAddressSet, - routeExcludeAddressSet: options.RouteExcludeAddressSet, + tunOptions: options.TunOptions, + ctx: options.Context, + handler: options.Handler, + logger: options.Logger, + networkMonitor: options.NetworkMonitor, + interfaceFinder: options.InterfaceFinder, + tableName: options.TableName, + useNFTables: runtime.GOOS != "android" && !options.DisableNFTables, + androidVPNService: options.AndroidVPNService, + customRedirectPortFunc: options.CustomRedirectPort, + customRedirectListenerFD: options.CustomRedirectListenerFD, + routeAddressSet: options.RouteAddressSet, } if options.TunOptions.NetNs != "" { r.interfaceFinder = &networkNamespaceInterfaceFinder{control.NewDefaultInterfaceFinder(), options.TunOptions} } - return r, nil -} - -func findAndroidSuPath() (string, error) { - searchPaths := common.Uniq(common.FilterNotDefault(append( - strings.Split(os.Getenv("PATH"), ":"), - "/system/bin", - "/system/xbin", - "/product/bin", - "/odm/bin", - "/vendor/bin", - "/vendor/xbin", - "/apex/com.android.runtime/bin", - "/sbin", - ))) - for _, searchPath := range searchPaths { - suPath, err := exec.LookPath(filepath.Join(searchPath, "su")) - if err == nil { - return suPath, nil + for _, mark := range []struct { + field *uint32 + defaultValue uint32 + androidDefaultValue uint32 + }{ + {&r.tunOptions.AutoRedirectInputMark, DefaultAutoRedirectInputMark, DefaultAutoRedirectInputMarkAndroid}, + {&r.tunOptions.AutoRedirectOutputMark, DefaultAutoRedirectOutputMark, DefaultAutoRedirectOutputMarkAndroid}, + {&r.tunOptions.AutoRedirectResetMark, DefaultAutoRedirectResetMark, DefaultAutoRedirectResetMarkAndroid}, + {&r.tunOptions.AutoRedirectTProxyMark, DefaultAutoRedirectTProxyMark, DefaultAutoRedirectTProxyMarkAndroid}, + } { + value := effectiveMark(*mark.field, mark.defaultValue, mark.androidDefaultValue) + if *mark.field != 0 && *mark.field != value { + r.logger.Warn("configured mark ", iptablesHex(*mark.field), " overlaps the fwmark bits reserved by Android and is replaced by ", iptablesHex(value)) } + *mark.field = value } - return "", E.New("su not found in ", strings.Join(searchPaths, ":")) + return r, nil } func (r *autoRedirect) Start() error { - var err error - if runtime.GOOS == "android" { - r.enableIPv4 = true - r.iptablesPath = "/system/bin/iptables" - userId := os.Getuid() - if userId != 0 { - r.androidSu = true - r.suPath, err = findAndroidSuPath() - if err != nil { - return E.Cause(err, "root permission is required for auto redirect") - } - } - } else { - if r.tunOptions.NetNs != "" && !r.useNFTables { - return E.New("auto_redirect in network namespace requires nftables") - } - if r.useNFTables { - err = runInNetworkNamespace(r.tunOptions.NetNs, r.initializeNFTables) - if err != nil { - return E.Cause(err, "missing nftables support") - } - } - if len(r.tunOptions.Inet4Address) > 0 { - r.enableIPv4 = true - if !r.useNFTables { - r.iptablesPath, err = exec.LookPath("iptables") - if err != nil { - return E.Cause(err, "iptables is required") - } - } - } - if len(r.tunOptions.Inet6Address) > 0 { - r.enableIPv6 = true - if !r.useNFTables { - r.ip6tablesPath, err = exec.LookPath("ip6tables") - if err != nil { - if !r.enableIPv4 { - return E.Cause(err, "ip6tables is required") - } else { - r.enableIPv6 = false - r.logger.Error("device has no ip6tables nat support: ", err) - } - } - } - } + err := r.initializeBackend() + if err != nil { + return err } - if r.customRedirectPortFunc != nil { - r.customRedirectPort = r.customRedirectPortFunc() + err = r.startRedirectServer() + if err != nil { + return err } - if r.customRedirectPort == 0 { - var listenAddr netip.Addr - if runtime.GOOS == "android" { - listenAddr = netip.AddrFrom4([4]byte{127, 0, 0, 1}) - } else if r.enableIPv6 { - listenAddr = netip.IPv6Unspecified() - } else { - listenAddr = netip.IPv4Unspecified() - } - server := newRedirectServer(r.ctx, r.handler, r.logger, listenAddr) - err = runInNetworkNamespace(r.tunOptions.NetNs, server.Start) - if err != nil { - return E.Cause(err, "start redirect server") - } - r.redirectServer = server + err = r.startNFQueue() + if err != nil { + _ = r.Close() + return E.Cause(err, "start nfqueue") } if r.useNFTables { - if r.handler != nil { - var handler *nfqueueHandler - handler, err = newNFQueueHandler(nfqueueOptions{ - Context: r.ctx, - Handler: r.handler, - Logger: r.logger, - Queue: r.effectiveNFQueue(), - InputMark: r.tunOptions.AutoRedirectInputMark, - OutputMark: r.effectiveOutputMark(), - ResetMark: r.effectiveResetMark(), - }) - if err != nil { - r.logger.Warn("nfqueue not available, pre-match disabled (missing nfnetlink_queue and nft_queue kernel module?): ", err) - } else if err = runInNetworkNamespace(r.tunOptions.NetNs, handler.Start); err != nil { - r.logger.Warn("nfqueue start failed, pre-match disabled (missing nfnetlink_queue and nft_queue kernel module?): ", err) - } else { - r.nfqueueHandler = handler - r.nfqueueEnabled = true - } - } if r.tunOptions.NetNs != "" { var monitor NetworkUpdateMonitor monitor, err = NewNetworkUpdateMonitor(r.logger) if err != nil { + _ = r.Close() return E.Cause(err, "create netns network monitor") } err = runInNetworkNamespace(r.tunOptions.NetNs, monitor.Start) if err != nil { + _ = monitor.Close() + _ = r.Close() return E.Cause(err, "start netns network monitor") } r.networkMonitor = monitor @@ -206,29 +140,244 @@ func (r *autoRedirect) Start() error { if r.tunOptions.AutoRedirectMarkMode { setupErr = r.setupRedirectRoutes() if setupErr != nil { - r.cleanupNFTables() return E.Cause(setupErr, "setup redirect routes") } } return nil }) if err != nil { + _ = r.Close() return err } - } else { - r.cleanupIPTables() - err = r.setupIPTables() + return nil + } + r.cleanupIPTables() + err = r.setupRedirectRoutes() + if err != nil { + _ = r.Close() + return E.Cause(err, "setup redirect routes") + } + if r.androidVPNService { + err = r.setupAndroidVPNServiceRules() if err != nil { - return E.Cause(err, "setup iptables") + _ = r.Close() + return E.Cause(err, "setup android vpn service rules") } + r.setupBypassRoute() } + err = r.setupIPTables() + if err != nil { + _ = r.Close() + return E.Cause(err, "setup iptables") + } + r.registerNetworkCallback(func() { + updateErr := r.updateIPTablesNetwork() + if updateErr != nil { + r.logger.Error(updateErr) + } + }) return nil } -func (r *autoRedirect) Close() error { - if r.nfqueueHandler != nil { - r.nfqueueHandler.Close() +func (r *autoRedirect) registerNetworkCallback(callback NetworkUpdateCallback) { + if r.networkMonitor != nil { + r.networkListener = r.networkMonitor.RegisterCallback(callback) + return + } + if r.tunOptions.InterfaceMonitor != nil { + r.interfaceListener = r.tunOptions.InterfaceMonitor.RegisterCallback(func(defaultInterface *control.Interface, flags int) { + callback() + }) + return + } + r.logger.Warn("no network monitor available, address and route updates are disabled") +} + +func (r *autoRedirect) unregisterNetworkCallback() { + if r.networkListener != nil { + r.networkMonitor.UnregisterCallback(r.networkListener) + r.networkListener = nil + } + if r.interfaceListener != nil { + r.tunOptions.InterfaceMonitor.UnregisterCallback(r.interfaceListener) + r.interfaceListener = nil + } +} + +func (r *autoRedirect) initializeBackend() error { + var err error + // iptables-nft refuses DROP in the nat table, where the MPTCP fallback drop belongs. + if r.tunOptions.ExcludeMPTCP && !r.useNFTables { + return E.New("exclude_mptcp requires nftables") + } + if runtime.GOOS == "android" { + if os.Getuid() != 0 { + return E.New("auto_redirect requires root") + } + if len(r.tunOptions.Inet4Address) > 0 { + r.iptablesPath, err = findIPTablesBinary(false) + if err != nil { + return E.Cause(err, "iptables is required") + } + r.enableIPv4 = true + } + if len(r.tunOptions.Inet6Address) > 0 { + r.ip6tablesPath, err = findIPTablesBinary(true) + if err != nil { + return E.Cause(err, "ip6tables is required") + } + r.enableIPv6 = true + } + return nil + } + if r.tunOptions.NetNs != "" && !r.useNFTables { + return E.New("auto_redirect in network namespace requires nftables") } + if r.useNFTables { + err = runInNetworkNamespace(r.tunOptions.NetNs, r.initializeNFTables) + if err != nil { + return E.Cause(err, "missing nftables support") + } + } + if len(r.tunOptions.Inet4Address) > 0 { + r.enableIPv4 = true + if !r.useNFTables { + r.iptablesPath, err = exec.LookPath("iptables") + if err != nil { + return E.Cause(err, "iptables is required") + } + err = exec.Command(r.iptablesPath, "-w", "-t", iptablesTableNAT, "-S").Run() + if err != nil { + return E.Cause(err, "iptables nat table is required") + } + } + } + if len(r.tunOptions.Inet6Address) > 0 { + r.enableIPv6 = true + if !r.useNFTables { + r.ip6tablesPath, err = exec.LookPath("ip6tables") + if err != nil { + return E.Cause(err, "ip6tables is required") + } + } + } + return nil +} + +func (r *autoRedirect) tproxyEnabled() bool { + return !r.useNFTables && r.enableIPv6 && (!r.androidVPNService || r.customRedirectTransparent) +} + +func (r *autoRedirect) prepareCustomTransparentListener() error { + fd, err := r.customRedirectListenerFD() + if err != nil { + return err + } + defer syscall.Close(fd) + err = syscall.SetsockoptInt(fd, syscall.SOL_IPV6, unix.IPV6_TRANSPARENT, 1) + if err != nil { + return E.Cause(err, "set IPV6_TRANSPARENT") + } + err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, int(r.tunOptions.AutoRedirectOutputMark)) + if err != nil { + return E.Cause(err, "set SO_MARK") + } + return nil +} + +func (r *autoRedirect) startRedirectServer() error { + if r.customRedirectPortFunc != nil { + r.customRedirectPort = r.customRedirectPortFunc() + } + if r.customRedirectPort > 0 { + if !r.useNFTables && r.enableIPv6 && r.androidVPNService && r.customRedirectListenerFD != nil { + err := r.prepareCustomTransparentListener() + if err != nil { + r.logger.Warn("IPv6 TCP falls back to the tun stack: prepare transparent listener: ", err) + } else { + r.customRedirectTransparent = true + } + } + return nil + } + var listenAddress netip.Addr + if r.enableIPv6 { + listenAddress = netip.IPv6Unspecified() + } else { + listenAddress = netip.IPv4Unspecified() + } + server := NewRedirectServer(r.ctx, r.handler, r.logger, listenAddress) + if r.tproxyEnabled() { + server.transparent = true + server.listenMark = r.tunOptions.AutoRedirectOutputMark + } + err := runInNetworkNamespace(r.tunOptions.NetNs, server.Start) + if err != nil { + return E.Cause(err, "start redirect server") + } + r.redirectServer = server + return nil +} + +const tcpOptionMultipathTCP = 30 + +func (r *autoRedirect) startNFQueue() error { + var markMask uint32 + if !r.useNFTables { + markMask = r.effectiveMarkMask() + } + var tproxyMark uint32 + if r.tproxyEnabled() { + tproxyMark = r.tunOptions.AutoRedirectTProxyMark + } + handler := &nfqueueHandler{ + handler: r.handler, + logger: r.logger, + queue: r.effectiveNFQueue(), + inputMark: r.tunOptions.AutoRedirectInputMark, + outputMark: r.tunOptions.AutoRedirectOutputMark, + resetMark: r.tunOptions.AutoRedirectResetMark, + tproxyMark: tproxyMark, + markMask: markMask, + // A queue verdict leaves the whole iptables mangle hook, while nftables base chains + // continue on NF_ACCEPT. + repeatOnAccept: !r.useNFTables, + } + err := runInNetworkNamespace(r.tunOptions.NetNs, handler.Start) + if err != nil { + return err + } + r.nfqueueHandler = handler + return nil +} + +func (r *autoRedirect) updateIPTablesNetwork() error { + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + err := r.updateIPTablesLocalAddresses() + if err != nil { + err = E.Cause(err, "update local address chains") + } + routeErr := r.updateRedirectRoutes() + if routeErr != nil { + routeErr = E.Cause(routeErr, "update redirect routes") + } + var vpnServiceErr error + var bypassErr error + if r.androidVPNService { + vpnServiceErr = r.updateAndroidVPNServiceRules() + if vpnServiceErr != nil { + vpnServiceErr = E.Cause(vpnServiceErr, "update android vpn service rules") + } + bypassErr = r.updateBypassRoute() + if bypassErr != nil { + bypassErr = E.Cause(bypassErr, "update bypass route") + } + } + return E.Errors(err, routeErr, vpnServiceErr, bypassErr) +} + +func (r *autoRedirect) Close() error { if r.useNFTables { _ = runInNetworkNamespace(r.tunOptions.NetNs, func() error { r.cleanupNFTables() @@ -237,22 +386,27 @@ func (r *autoRedirect) Close() error { }) if r.ownedNetworkMonitor { _ = r.networkMonitor.Close() + r.ownedNetworkMonitor = false } } else { + r.unregisterNetworkCallback() r.cleanupIPTables() - } - return common.Close( - common.PtrOrNil(r.redirectServer), - ) -} - -func (r *autoRedirect) UpdateRouteAddressSet() { - if r.useNFTables { - err := runInNetworkNamespace(r.tunOptions.NetNs, r.nftablesUpdateRouteAddressSet) - if err != nil { - r.logger.Error("update route address set: ", err) + r.cleanupRedirectRoutes() + if r.androidVPNService { + r.cleanupBypassRoute() + r.cleanupAndroidVPNServiceRules() } } + if r.nfqueueHandler != nil { + r.nfqueueHandler.Close() + r.nfqueueHandler = nil + } + if r.redirectServer != nil { + err := r.redirectServer.Close() + r.redirectServer = nil + return err + } + return nil } func (r *autoRedirect) initializeNFTables() error { @@ -273,21 +427,43 @@ func (r *autoRedirect) redirectPort() uint16 { if r.customRedirectPort > 0 { return uint16(r.customRedirectPort) } - return M.AddrPortFromNet(r.redirectServer.listener.Addr()).Port() + return r.redirectServer.Port() } -func (r *autoRedirect) effectiveOutputMark() uint32 { - if r.tunOptions.AutoRedirectOutputMark != 0 { - return r.tunOptions.AutoRedirectOutputMark - } - return DefaultAutoRedirectOutputMark +// netd's Fwmark owns bits 0-19 (netId, explicitlySelected, protectedFromVpn, +// permission) and bit 20 (uidBillingDone, written by the bw_* chains); bits +// 29-30 are the vendor field and bit 31 is ingress_cpu_wakeup. The mark is also +// mirrored into the connmark, where netd's StrictController owns bits 24-25. +const androidReservedMarkMask = 0xE31FFFFF + +// netd (server/RouteController.cpp) identifies a network's route table as +// ifindex + ROUTE_TABLE_OFFSET_FROM_INDEX, selects VPN traffic with the +// secure (fwmark 0x0/0x20000) and bypassable (fwmark 0x0/0x30000) uid range +// rules, the local network with a rule matching fwmark 0x0/0x10000, and the +// default network with a rule matching fwmark 0x0/0xffff without a uid range. +const ( + androidRouteTableOffsetFromIndex = 1000 + androidExplicitSelectMask = 0x10000 + androidSecureVPNMask = 0x20000 + androidBypassableVPNMask = 0x30000 + androidNetIDMask = 0xFFFF +) + +func (r *autoRedirect) effectiveMarkMask() uint32 { + return r.tunOptions.AutoRedirectInputMark | r.tunOptions.AutoRedirectOutputMark | r.tunOptions.AutoRedirectResetMark | r.tunOptions.AutoRedirectTProxyMark } -func (r *autoRedirect) effectiveResetMark() uint32 { - if r.tunOptions.AutoRedirectResetMark != 0 { - return r.tunOptions.AutoRedirectResetMark +func effectiveMark(value uint32, defaultValue uint32, androidDefaultValue uint32) uint32 { + if runtime.GOOS == "android" { + if value != 0 && value&androidReservedMarkMask == 0 { + return value + } + return androidDefaultValue + } + if value != 0 { + return value } - return DefaultAutoRedirectResetMark + return defaultValue } func (r *autoRedirect) effectiveNFQueue() uint16 { diff --git a/redirect_nftables.go b/redirect_nftables.go index 70e21466..d3e00c90 100644 --- a/redirect_nftables.go +++ b/redirect_nftables.go @@ -30,11 +30,6 @@ func (r *autoRedirect) setupNFTables() error { Family: nftables.TableFamilyINet, }) - err = r.nftablesCreateAddressSets(nft, table, false) - if err != nil { - return E.Cause(err, "create address sets") - } - err = r.interfaceFinder.Update() if err != nil { return E.Cause(err, "update interfaces") @@ -54,18 +49,13 @@ func (r *autoRedirect) setupNFTables() error { return E.Cause(err, "create loopback address sets") } - if r.nfqueueEnabled { - err = r.nftablesCreatePreMatchChains(nft, table) - if err != nil { - return E.Cause(err, "create pre-match chains") - } + err = r.nftablesCreatePreMatchChains(nft, table) + if err != nil { + return E.Cause(err, "create pre-match chains") } if !r.shouldSkipOutputChain() { - outputNATPriority := nftables.ChainPriorityMangle - if r.nfqueueEnabled { - outputNATPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityMangle + 2) - } + outputNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityMangle + 2) chainOutput := nft.AddChain(&nftables.Chain{ Name: "output", Table: table, @@ -135,12 +125,8 @@ func (r *autoRedirect) setupNFTables() error { r.nftablesCreateRedirectPortReject(nft, table, chainInput) } - preroutingNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 1) - preroutingRoutePriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) - if r.nfqueueEnabled { - preroutingNATPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) - preroutingRoutePriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 3) - } + preroutingNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) + preroutingRoutePriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 3) chainPreRouting := nft.AddChain(&nftables.Chain{ Name: "prerouting", Table: table, @@ -313,7 +299,7 @@ func (r *autoRedirect) setupNFTables() error { } } - r.networkListener = r.networkMonitor.RegisterCallback(func() { + r.registerNetworkCallback(func() { updateErr := runInNetworkNamespace(r.tunOptions.NetNs, r.updateNetworkAddresses) if updateErr != nil { r.logger.Error(updateErr) @@ -372,28 +358,8 @@ func (r *autoRedirect) nftablesUpdateLocalAddressSet() error { return nft.Flush() } -func (r *autoRedirect) nftablesUpdateRouteAddressSet() error { - nft, err := nftables.New() - if err != nil { - return E.Cause(err, "create nftables connection") - } - defer nft.CloseLasting() - table, err := nft.ListTableOfFamily(r.tableName, nftables.TableFamilyINet) - if err != nil { - return E.Cause(err, "list nftables table") - } - err = r.nftablesCreateAddressSets(nft, table, true) - if err != nil { - return E.Cause(err, "create address sets") - } - return nft.Flush() -} - func (r *autoRedirect) cleanupNFTables() { - if r.networkListener != nil { - r.networkMonitor.UnregisterCallback(r.networkListener) - r.networkListener = nil - } + r.unregisterNetworkCallback() r.stopDockerFirewallMonitor() nft, err := nftables.New() if err != nil { @@ -468,7 +434,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Verdict{Kind: expr.VerdictReturn}, }, }) - for _, mark := range []uint32{r.effectiveOutputMark(), r.tunOptions.AutoRedirectInputMark} { + for _, mark := range []uint32{r.tunOptions.AutoRedirectOutputMark, r.tunOptions.AutoRedirectInputMark} { nft.AddRule(&nftables.Rule{ Table: table, Chain: chain, @@ -574,7 +540,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, &expr.Meta{Key: expr.MetaKeyMARK, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(r.effectiveResetMark())}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(r.tunOptions.AutoRedirectResetMark)}, &expr.Counter{}, &expr.Reject{Type: unix.NFT_REJECT_TCP_RST}, }, diff --git a/redirect_nftables_rules.go b/redirect_nftables_rules.go index 40c3e0fe..e74c43df 100644 --- a/redirect_nftables_rules.go +++ b/redirect_nftables_rules.go @@ -26,48 +26,6 @@ func init() { allocSetID = 6 } -func (r *autoRedirect) nftablesCreateAddressSets( - nft *nftables.Conn, table *nftables.Table, - update bool, -) error { - routeAddressSet := *r.routeAddressSet - routeExcludeAddressSet := *r.routeExcludeAddressSet - if len(routeAddressSet) == 0 && len(routeExcludeAddressSet) == 0 { - return nil - } - - if len(routeAddressSet) > 0 { - if r.enableIPv4 { - _, err := nftablesCreateIPSet(nft, table, 1, "inet4_route_address_set", nftables.TableFamilyIPv4, routeAddressSet, nil, true, update) - if err != nil { - return E.Cause(err, "create ipv4 route address set") - } - } - if r.enableIPv6 { - _, err := nftablesCreateIPSet(nft, table, 2, "inet6_route_address_set", nftables.TableFamilyIPv6, routeAddressSet, nil, true, update) - if err != nil { - return E.Cause(err, "create ipv6 route address set") - } - } - } - - if len(routeExcludeAddressSet) > 0 { - if r.enableIPv4 { - _, err := nftablesCreateIPSet(nft, table, 3, "inet4_route_exclude_address_set", nftables.TableFamilyIPv4, routeExcludeAddressSet, nil, false, update) - if err != nil { - return E.Cause(err, "create ipv4 route exclude address set") - } - } - if r.enableIPv6 { - _, err := nftablesCreateIPSet(nft, table, 4, "inet6_route_exclude_address_set", nftables.TableFamilyIPv6, routeExcludeAddressSet, nil, false, update) - if err != nil { - return E.Cause(err, "create ipv6 route exclude address set") - } - } - } - return nil -} - func (r *autoRedirect) nftablesCreateLocalAddressSets( nft *nftables.Conn, table *nftables.Table, localAddresses []netip.Prefix, lastAddresses []netip.Prefix, @@ -209,8 +167,8 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }) } } - if r.nfqueueEnabled && chain.Type == nftables.ChainTypeNAT { - for _, mark := range []uint32{r.effectiveOutputMark(), r.tunOptions.AutoRedirectInputMark} { + if chain.Type == nftables.ChainTypeNAT { + for _, mark := range []uint32{r.tunOptions.AutoRedirectOutputMark, r.tunOptions.AutoRedirectInputMark} { nft.AddRule(&nftables.Rule{ Table: table, Chain: chain, @@ -223,7 +181,6 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }) } } - if chain.Hooknum == nftables.ChainHookPrerouting { nft.AddRule(&nftables.Rule{ Table: table, @@ -673,25 +630,6 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft nftablesCreateExcludeDestinationIPSet(nft, table, chain, 6, "inet6_local_address_set", nftables.TableFamilyIPv6, false) } - routeAddressSet := *r.routeAddressSet - routeExcludeAddressSet := *r.routeExcludeAddressSet - - if r.enableIPv4 && len(routeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 1, "inet4_route_address_set", nftables.TableFamilyIPv4, true) - } - - if r.enableIPv6 && len(routeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 2, "inet6_route_address_set", nftables.TableFamilyIPv6, true) - } - - if r.enableIPv4 && len(routeExcludeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 3, "inet4_route_exclude_address_set", nftables.TableFamilyIPv4, false) - } - - if r.enableIPv6 && len(routeExcludeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 4, "inet6_route_exclude_address_set", nftables.TableFamilyIPv6, false) - } - if chain.Type == nftables.ChainTypeNAT || r.tunOptions.ExcludeMPTCP { mptcpVerdict := expr.VerdictDrop if r.tunOptions.ExcludeMPTCP { @@ -712,7 +650,7 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }, &expr.Exthdr{ DestRegister: 1, - Type: 30, + Type: tcpOptionMultipathTCP, Offset: 0, Len: 1, Flags: unix.NFT_EXTHDR_F_PRESENT, @@ -1052,9 +990,9 @@ func (r *autoRedirect) nftablesCreateDNSHijackRulesForFamily( }, } if chain.Hooknum == nftables.ChainHookOutput { - // It looks like we can't hijack DNS requests sent to loopback. - // https://serverfault.com/questions/363899/iptables-dnat-from-loopback - // and tproxy is not available in output + // The DNAT target still runs for loopback-destined packets, but the + // reroute that follows it refuses a loopback source on a non-loopback + // device and drops the packet. exprs = append(exprs, &expr.Meta{ Key: expr.MetaKeyOIFNAME, diff --git a/redirect_route_bypass_android.go b/redirect_route_bypass_android.go new file mode 100644 index 00000000..67868673 --- /dev/null +++ b/redirect_route_bypass_android.go @@ -0,0 +1,340 @@ +package tun + +import ( + "net" + "net/netip" + "slices" + + "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" + + "go4.org/netipx" + "golang.org/x/sys/unix" +) + +func (r *autoRedirect) setupBypassRoute() { + if r.routeAddressSet == nil { + return + } + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + r.cleanupStaleRouteRules(r.androidVPNServiceRulePriority(), 0) + r.bypassRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.redirectRouteTableIndex, r.tproxyRouteTableIndex) + err := r.refreshBypassAddressesLocked() + if err == nil { + err = r.rebuildBypassRouteLocked() + } + if err != nil { + r.cleanupBypassRouteLocked() + r.logger.Warn("address set destinations keep their pre-match bypass: setup bypass route: ", err) + return + } + r.bypassRouteActive = true +} + +func (r *autoRedirect) updateBypassRoute() error { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + if !r.bypassRouteActive { + return nil + } + return r.rebuildBypassRouteLocked() +} + +func (r *autoRedirect) UpdateRouteAddressSet() error { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + if !r.bypassRouteActive { + return nil + } + err := r.refreshBypassAddressesLocked() + if err != nil { + return err + } + return r.rebuildBypassRouteLocked() +} + +func (r *autoRedirect) refreshBypassAddressesLocked() error { + include, exclude, err := r.routeAddressSet() + if err != nil { + return E.Cause(err, "fetch route address set") + } + r.bypassIncludeAddresses = include + r.bypassExcludeAddresses = exclude + return nil +} + +func (r *autoRedirect) cleanupBypassRoute() { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + r.bypassRouteActive = false + r.cleanupBypassRouteLocked() +} + +func (r *autoRedirect) cleanupBypassRouteLocked() { + if r.bypassRouteTableIndex == 0 { + return + } + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + _ = netlink.RuleDel(r.bypassRouteRule(family)) + } + flushRouteTable(r.bypassRouteTableIndex) +} + +func (r *autoRedirect) rebuildBypassRouteLocked() error { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + if family == unix.AF_INET && !r.enableIPv4 || family == unix.AF_INET6 && !r.enableIPv6 { + continue + } + err := r.rebuildBypassRouteFamilyLocked(family) + if err != nil { + return err + } + } + return nil +} + +func (r *autoRedirect) rebuildBypassRouteFamilyLocked(family int) error { + tunPrefixes := r.tunOptions.Inet4Address + if family == unix.AF_INET6 { + tunPrefixes = r.tunOptions.Inet6Address + } + set, err := bypassAddressSet(family, r.bypassIncludeAddresses, r.bypassExcludeAddresses, tunPrefixes) + if err != nil { + return E.Cause(err, "build bypass address set") + } + rule := r.bypassRouteRule(family) + if len(set.Ranges()) == 0 { + _ = netlink.RuleDel(rule) + flushRouteTableFamily(r.bypassRouteTableIndex, family) + return nil + } + ruleList, err := netlink.RuleList(family) + if err != nil { + return E.Cause(err, "list rules") + } + sources := r.bypassRouteSources(family, ruleList) + // netd rebuilds its rules and tables asynchronously after a network + // change, so an update can observe them half-built; the repopulated + // tables generate route events that schedule another rebuild. + if len(sources) == 0 { + r.logger.Debug("bypass route family ", family, ": no source tables, keeping current routes") + return nil + } + routes, err := buildBypassRoutes(family, set, sources, r.bypassRouteTableIndex) + if err != nil { + return E.Cause(err, "build bypass routes") + } + if len(routes) == 0 { + r.logger.Debug("bypass route family ", family, ": no routes built from ", len(sources), " sources, keeping current routes") + return nil + } + changed, err := r.reconcileBypassRoutesLocked(family, routes) + if err != nil { + return err + } + if changed > 0 { + r.logger.Debug("bypass route family ", family, ": ", changed, " of ", len(routes), " routes changed, from ", len(sources), " sources") + } + if !slices.ContainsFunc(ruleList, func(it netlink.Rule) bool { + return it.Table == r.bypassRouteTableIndex + }) { + err = netlink.RuleAdd(rule) + if err != nil { + return E.Cause(err, "add bypass rule") + } + } + return nil +} + +func (r *autoRedirect) reconcileBypassRoutesLocked(family int, desired []netlink.Route) (int, error) { + current, err := netlink.RouteListFiltered(family, &netlink.Route{Table: r.bypassRouteTableIndex}, netlink.RT_FILTER_TABLE) + if err != nil { + return 0, E.Cause(err, "list bypass routes") + } + desiredByDestination := make(map[string]*netlink.Route, len(desired)) + for index := range desired { + desiredByDestination[desired[index].Dst.String()] = &desired[index] + } + matched := make(map[string]bool, len(desired)) + var changed int + for index := range current { + currentRoute := ¤t[index] + if currentRoute.Dst == nil { + currentRoute.Dst = defaultDestination(family) + } + destination := currentRoute.Dst.String() + desiredRoute, exists := desiredByDestination[destination] + if exists && bypassRouteEquals(currentRoute, desiredRoute) { + matched[destination] = true + continue + } + // RouteReplace only overwrites a route with the same metric; a + // current route with another metric would survive alongside it. + if !exists || currentRoute.Priority != desiredRoute.Priority { + _ = netlink.RouteDel(currentRoute) + changed++ + } + } + for index := range desired { + desiredRoute := &desired[index] + if matched[desiredRoute.Dst.String()] { + continue + } + err = netlink.RouteReplace(desiredRoute) + if err != nil { + return changed, E.Cause(err, "add bypass route ", desiredRoute.Dst) + } + changed++ + } + return changed, nil +} + +func bypassRouteEquals(left *netlink.Route, right *netlink.Route) bool { + return left.LinkIndex == right.LinkIndex && + left.Scope == right.Scope && + left.Priority == right.Priority && + left.Gw.Equal(right.Gw) && + routeSourceEquals(left.Src, right.Src) +} + +func routeSourceEquals(left *net.IPNet, right *net.IPNet) bool { + if left == nil || right == nil { + return left == right + } + return left.IP.Equal(right.IP) && slices.Equal(left.Mask, right.Mask) +} + +func (r *autoRedirect) bypassRouteRule(family int) *netlink.Rule { + rule := netlink.NewRule() + rule.Priority = r.androidVPNServiceRulePriority() + rule.Family = family + rule.Table = r.bypassRouteTableIndex + if r.tunOptions.DNSModeOrDefault() == DNSModeHijack { + rule.Invert = true + rule.Dport = netlink.NewRulePortRange(53, 53) + } + return rule +} + +func (r *autoRedirect) bypassRouteSources(family int, ruleList []netlink.Rule) []netlink.Route { + tables := discoverAndroidVPNServiceTables(ruleList, r.tunOptions.Name) + var sourceTables []int + if tables.localNetwork > 0 { + sourceTables = append(sourceTables, tables.localNetwork) + } + sourceTables = append(sourceTables, tables.physical...) + var specificRoutes []netlink.Route + var defaultRoutes []netlink.Route + for _, sourceTable := range sourceTables { + routes, listErr := netlink.RouteListFiltered(family, &netlink.Route{Table: sourceTable}, netlink.RT_FILTER_TABLE) + if listErr != nil { + continue + } + for _, route := range routes { + if route.Type != 0 && route.Type != unix.RTN_UNICAST { + continue + } + if routeDestination(family, route.Dst).Bits() == 0 { + defaultRoutes = append(defaultRoutes, route) + } else { + specificRoutes = append(specificRoutes, route) + } + } + } + slices.SortStableFunc(specificRoutes, func(left netlink.Route, right netlink.Route) int { + return routeDestination(family, right.Dst).Bits() - routeDestination(family, left.Dst).Bits() + }) + return append(specificRoutes, defaultRoutes...) +} + +func buildBypassRoutes(family int, set *netipx.IPSet, sources []netlink.Route, tableIndex int) ([]netlink.Route, error) { + remaining := set + var routes []netlink.Route + for _, source := range sources { + if len(remaining.Ranges()) == 0 { + break + } + destination := routeDestination(family, source.Dst) + if !destination.IsValid() { + continue + } + part, err := intersectPrefix(remaining, destination) + if err != nil { + return nil, err + } + if len(part.Ranges()) == 0 { + continue + } + for _, prefix := range part.Prefixes() { + route := source + route.Dst = prefixToIPNet(prefix) + route.Table = tableIndex + routes = append(routes, route) + } + var remainingBuilder netipx.IPSetBuilder + remainingBuilder.AddSet(remaining) + remainingBuilder.RemoveSet(part) + remaining, err = remainingBuilder.IPSet() + if err != nil { + return nil, err + } + } + return routes, nil +} + +func bypassAddressSet(family int, include []netip.Prefix, exclude []netip.Prefix, tunPrefixes []netip.Prefix) (*netipx.IPSet, error) { + var builder netipx.IPSetBuilder + for _, prefix := range exclude { + if prefixMatchesFamily(family, prefix) { + builder.AddPrefix(prefix) + } + } + if len(include) > 0 { + builder.AddPrefix(unspecifiedPrefix(family)) + for _, prefix := range include { + if prefixMatchesFamily(family, prefix) { + builder.RemovePrefix(prefix) + } + } + } + for _, prefix := range tunPrefixes { + builder.RemovePrefix(prefix.Masked()) + } + return builder.IPSet() +} + +func prefixMatchesFamily(family int, prefix netip.Prefix) bool { + return (family == unix.AF_INET) == prefix.Addr().Is4() +} + +func unspecifiedPrefix(family int) netip.Prefix { + if family == unix.AF_INET { + return netip.PrefixFrom(netip.IPv4Unspecified(), 0) + } + return netip.PrefixFrom(netip.IPv6Unspecified(), 0) +} + +func intersectPrefix(set *netipx.IPSet, prefix netip.Prefix) (*netipx.IPSet, error) { + var prefixBuilder netipx.IPSetBuilder + prefixBuilder.AddPrefix(prefix) + prefixSet, err := prefixBuilder.IPSet() + if err != nil { + return nil, err + } + var builder netipx.IPSetBuilder + builder.AddSet(set) + builder.Intersect(prefixSet) + return builder.IPSet() +} + +func routeDestination(family int, destination *net.IPNet) netip.Prefix { + if destination == nil { + return unspecifiedPrefix(family) + } + prefix, valid := netipx.FromStdIPNet(destination) + if !valid { + return netip.Prefix{} + } + return prefix +} diff --git a/redirect_route_linux.go b/redirect_route_linux.go index f73df11a..a55f0a71 100644 --- a/redirect_route_linux.go +++ b/redirect_route_linux.go @@ -6,6 +6,7 @@ import ( "errors" "math/rand" "net" + "slices" "github.com/sagernet/netlink" "github.com/sagernet/sing/common" @@ -18,18 +19,6 @@ import ( const redirectRouteRulePriority = 1 func (r *autoRedirect) setupRedirectRoutes() error { - for { - r.redirectRouteTableIndex = int(rand.Uint32()) - if r.redirectRouteTableIndex == r.tunOptions.IPRoute2TableIndex { - continue - } - routeList, fErr := netlink.RouteListFiltered(netlink.FAMILY_ALL, - &netlink.Route{Table: r.redirectRouteTableIndex}, - netlink.RT_FILTER_TABLE) - if len(routeList) == 0 || fErr != nil { - break - } - } err := r.interfaceFinder.Update() if err != nil { return E.Cause(err, "update interfaces") @@ -39,6 +28,11 @@ func (r *autoRedirect) setupRedirectRoutes() error { defer r.redirectRouteAccess.Unlock() r.redirectRoutesActive = false r.cleanupRedirectRoutesLocked() + r.cleanupStaleRouteRules(redirectRouteRulePriority, 0) + if r.tproxyEnabled() { + r.cleanupStaleRouteRules(redirectRouteRulePriority, r.tunOptions.AutoRedirectTProxyMark) + } + r.redirectRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.tproxyRouteTableIndex) defer func() { if err != nil { r.cleanupRedirectRoutesLocked() @@ -68,10 +62,52 @@ func (r *autoRedirect) setupRedirectRoutes() error { return E.Cause(err, "add ipv6 redirect rule") } } + if r.tproxyEnabled() { + err = r.setupTProxyRouteLocked() + if err != nil { + return err + } + } r.redirectRoutesActive = true return nil } +func (r *autoRedirect) setupTProxyRouteLocked() error { + r.tproxyRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.redirectRouteTableIndex) + loopbackLink, err := netlink.LinkByName("lo") + if err != nil { + return E.Cause(err, "find loopback interface") + } + route := netlink.Route{ + LinkIndex: loopbackLink.Attrs().Index, + Dst: defaultDestination(unix.AF_INET6), + Table: r.tproxyRouteTableIndex, + Type: unix.RTN_LOCAL, + Scope: netlink.SCOPE_HOST, + } + err = netlink.RouteAdd(&route) + if err != nil { + return E.Cause(err, "add tproxy route") + } + rule := r.tproxyRouteRule() + err = netlink.RuleAdd(rule) + if err != nil { + return E.Cause(err, "add tproxy rule") + } + return nil +} + +func (r *autoRedirect) tproxyRouteRule() *netlink.Rule { + rule := netlink.NewRule() + rule.Priority = redirectRouteRulePriority + rule.Family = unix.AF_INET6 + rule.Mark = r.tunOptions.AutoRedirectTProxyMark + rule.MarkSet = true + rule.Mask = int(r.effectiveMarkMask()) + rule.Table = r.tproxyRouteTableIndex + return rule +} + func (r *autoRedirect) currentRedirectInterfaces() []control.Interface { tunName := r.tunOptions.Name return common.Filter(r.interfaceFinder.Interfaces(), func(it control.Interface) bool { @@ -136,6 +172,10 @@ func (r *autoRedirect) cleanupRedirectRoutesLocked() { rule.Family = unix.AF_INET6 _ = netlink.RuleDel(rule) } + if r.tproxyRouteTableIndex != 0 { + flushRouteTable(r.tproxyRouteTableIndex) + _ = netlink.RuleDel(r.tproxyRouteRule()) + } } type redirectRouteKey struct { @@ -144,9 +184,6 @@ type redirectRouteKey struct { } func (r *autoRedirect) reconcileRedirectRoutesLocked(redirectInterfaces []control.Interface) error { - // Interface snapshots are not sufficient here: network managers can flush a - // route while a fast reconnect leaves the interface index and address families - // unchanged. Reconcile against the kernel's route table on every update. currentRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: r.redirectRouteTableIndex}, netlink.RT_FILTER_TABLE) @@ -242,3 +279,72 @@ func redirectRouteDestinationMatches(destination *net.IPNet, address net.IP, pre ones, bits := destination.Mask.Size() return ones == prefixBits && bits == prefixBits } + +func chooseRouteTableIndex(excludedIndexes ...int) int { + for { + tableIndex := int(rand.Uint32() & 0x7fffffff) + if tableIndex <= 255 || slices.Contains(excludedIndexes, tableIndex) { + continue + } + if !routeTableOccupied(tableIndex) { + return tableIndex + } + } +} + +func routeTableOccupied(tableIndex int) bool { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + routes, err := netlink.RouteListFiltered(family, + &netlink.Route{Table: tableIndex}, + netlink.RT_FILTER_TABLE) + if err == nil && len(routes) > 0 { + return true + } + } + return false +} + +func flushRouteTable(tableIndex int) { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + flushRouteTableFamily(tableIndex, family) + } +} + +func flushRouteTableFamily(tableIndex int, family int) { + routes, _ := netlink.RouteListFiltered(family, + &netlink.Route{Table: tableIndex}, + netlink.RT_FILTER_TABLE) + for index := range routes { + route := &routes[index] + if route.Dst == nil { + route.Dst = defaultDestination(family) + } + _ = netlink.RouteDel(route) + } +} + +func (r *autoRedirect) cleanupStaleRouteRules(priority int, mark uint32) { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + rules, err := netlink.RuleListFiltered(family, + &netlink.Rule{Priority: priority}, + netlink.RT_FILTER_PRIORITY) + if err != nil { + continue + } + for index := range rules { + staleRule := &rules[index] + if staleRule.Mark != mark || staleRule.Table <= 255 { + continue + } + flushRouteTable(staleRule.Table) + _ = netlink.RuleDel(staleRule) + } + } +} + +func defaultDestination(family int) *net.IPNet { + if family == unix.AF_INET6 { + return &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)} + } + return &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)} +} diff --git a/redirect_route_vpnservice_android.go b/redirect_route_vpnservice_android.go new file mode 100644 index 00000000..952882cf --- /dev/null +++ b/redirect_route_vpnservice_android.go @@ -0,0 +1,228 @@ +package tun + +import ( + "errors" + "os" + "slices" + "strconv" + "strings" + + "github.com/sagernet/netlink" + "github.com/sagernet/sing/common" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +func (r *autoRedirect) androidVPNServiceRulePriority() int { + if r.tunOptions.IPRoute2RuleIndex != 0 { + return r.tunOptions.IPRoute2RuleIndex + } + return DefaultIPRoute2RuleIndex +} + +type androidVPNServiceTables struct { + physical []int + localNetwork int +} + +// netd adds one default network rule per interface of the default network +// (with 464xlat: the base interface and the clat interface, only the latter +// holding the IPv4 default route), so every table is kept in dump order and +// the kernel falls through on a lookup miss. netd reaches the local network +// table through the rule matching the VPN as the incoming interface; the +// explicit select mask instead matches the legacy system and legacy network +// tables first, which stay empty. +func discoverAndroidVPNServiceTables(ruleList []netlink.Rule, tunInterfaceName string) androidVPNServiceTables { + var tables androidVPNServiceTables + for _, rule := range ruleList { + if rule.Table > 0 && rule.UIDRange == nil && rule.Mark == 0 && rule.Mask == androidNetIDMask && !slices.Contains(tables.physical, rule.Table) { + tables.physical = append(tables.physical, rule.Table) + } + if rule.Table > 0 && rule.Mark == 0 && rule.UIDRange == nil && rule.IifName == tunInterfaceName && rule.OifName == "" && tables.localNetwork == 0 { + tables.localNetwork = rule.Table + } + } + return tables +} + +func (r *autoRedirect) discoverAndroidVPNServiceRules() ([]*netlink.Rule, error) { + priority := r.androidVPNServiceRulePriority() + mask := int(r.effectiveMarkMask()) + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return nil, E.Cause(err, "find tun interface") + } + vpnTable := tunLink.Attrs().Index + androidRouteTableOffsetFromIndex + var ( + rules []*netlink.Rule + ruleList []netlink.Rule + ) + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + familyName := "ipv4" + if family == unix.AF_INET6 { + if !r.enableIPv6 { + continue + } + familyName = "ipv6" + } else if !r.enableIPv4 { + continue + } + ruleList, err = netlink.RuleList(family) + if err != nil { + return nil, E.Cause(err, "list ", familyName, " rules") + } + tables := discoverAndroidVPNServiceTables(ruleList, r.tunOptions.Name) + physicalTables := tables.physical + localNetworkTable := tables.localNetwork + r.logger.Debug(familyName, " vpn service rules: vpn table ", vpnTable, ", default network tables ", strings.Join(common.Map(physicalTables, strconv.Itoa), " "), ", local network table ", localNetworkTable) + inputRule := netlink.NewRule() + inputRule.Priority = priority + inputRule.Family = family + inputRule.Mark = r.tunOptions.AutoRedirectInputMark + inputRule.MarkSet = true + inputRule.Mask = mask + inputRule.Table = vpnTable + rules = append(rules, inputRule) + // A reset generated by the REJECT chain keeps the reset mark of the + // packet that triggered it and has to reach the client on the served + // network instead of the netd VPN rule that catches unmarked flows. + if localNetworkTable > 0 { + resetRule := netlink.NewRule() + resetRule.Priority = priority + resetRule.Family = family + resetRule.Mark = r.tunOptions.AutoRedirectResetMark + resetRule.MarkSet = true + resetRule.Mask = mask + resetRule.Table = localNetworkTable + rules = append(rules, resetRule) + } + // Bypassed local flows carry the tun address as their source and + // Android ships no ip6tables nat table to rewrite it. + if family == unix.AF_INET6 { + continue + } + if localNetworkTable > 0 { + localOutputRule := netlink.NewRule() + localOutputRule.Priority = priority + localOutputRule.Family = family + localOutputRule.Mark = r.tunOptions.AutoRedirectOutputMark + localOutputRule.MarkSet = true + localOutputRule.Mask = mask + localOutputRule.IifName = "lo" + localOutputRule.Table = localNetworkTable + rules = append(rules, localOutputRule) + } + for _, physicalTable := range physicalTables { + outputRule := netlink.NewRule() + outputRule.Priority = priority + outputRule.Family = family + outputRule.Mark = r.tunOptions.AutoRedirectOutputMark + outputRule.MarkSet = true + outputRule.Mask = mask + outputRule.IifName = "lo" + outputRule.Table = physicalTable + rules = append(rules, outputRule) + } + } + return rules, nil +} + +func androidVPNServiceRuleEquals(left *netlink.Rule, right *netlink.Rule) bool { + return left.Family == right.Family && + left.Priority == right.Priority && + left.Mark == right.Mark && + left.Mask == right.Mask && + left.IifName == right.IifName && + left.Table == right.Table +} + +func (r *autoRedirect) setupAndroidVPNServiceRules() error { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + priority := r.androidVPNServiceRulePriority() + inputMark := r.tunOptions.AutoRedirectInputMark + outputMark := r.tunOptions.AutoRedirectOutputMark + resetMark := r.tunOptions.AutoRedirectResetMark + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + staleRules, err := netlink.RuleListFiltered(family, &netlink.Rule{Priority: priority}, netlink.RT_FILTER_PRIORITY) + if err != nil { + continue + } + for _, staleRule := range staleRules { + if staleRule.Mark != inputMark && staleRule.Mark != outputMark && staleRule.Mark != resetMark { + continue + } + deleteRule := netlink.NewRule() + deleteRule.Family = family + deleteRule.Priority = priority + deleteRule.Mark = staleRule.Mark + deleteRule.MarkSet = true + deleteRule.Mask = staleRule.Mask + _ = netlink.RuleDel(deleteRule) + } + } + rules, err := r.discoverAndroidVPNServiceRules() + if err != nil { + return err + } + r.androidVPNServiceRulesActive = true + for _, rule := range rules { + err = netlink.RuleAdd(rule) + if err != nil && !E.IsMulti(err, unix.EEXIST, os.ErrExist) { + return E.Cause(err, "add rule ", rule) + } + r.androidVPNServiceRules = append(r.androidVPNServiceRules, rule) + } + return nil +} + +func (r *autoRedirect) updateAndroidVPNServiceRules() error { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + if !r.androidVPNServiceRulesActive { + return nil + } + rules, err := r.discoverAndroidVPNServiceRules() + if err != nil { + var linkNotFound netlink.LinkNotFoundError + if errors.As(err, &linkNotFound) { + return nil + } + return err + } + var errorList []error + var installedRules []*netlink.Rule + for _, installedRule := range r.androidVPNServiceRules { + if slices.ContainsFunc(rules, func(it *netlink.Rule) bool { + return androidVPNServiceRuleEquals(it, installedRule) + }) { + continue + } + deleteErr := netlink.RuleDel(installedRule) + if deleteErr != nil && !E.IsMulti(deleteErr, unix.ENOENT, os.ErrNotExist) { + errorList = append(errorList, E.Cause(deleteErr, "delete rule ", installedRule)) + installedRules = append(installedRules, installedRule) + } + } + for _, rule := range rules { + addErr := netlink.RuleAdd(rule) + if addErr != nil && !E.IsMulti(addErr, unix.EEXIST, os.ErrExist) { + errorList = append(errorList, E.Cause(addErr, "add rule ", rule)) + continue + } + installedRules = append(installedRules, rule) + } + r.androidVPNServiceRules = installedRules + return E.Errors(errorList...) +} + +func (r *autoRedirect) cleanupAndroidVPNServiceRules() { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + r.androidVPNServiceRulesActive = false + for _, rule := range r.androidVPNServiceRules { + _ = netlink.RuleDel(rule) + } + r.androidVPNServiceRules = nil +} diff --git a/redirect_route_vpnservice_stub.go b/redirect_route_vpnservice_stub.go new file mode 100644 index 00000000..45fdc608 --- /dev/null +++ b/redirect_route_vpnservice_stub.go @@ -0,0 +1,28 @@ +//go:build linux && !android + +package tun + +func (r *autoRedirect) setupAndroidVPNServiceRules() error { + return nil +} + +func (r *autoRedirect) updateAndroidVPNServiceRules() error { + return nil +} + +func (r *autoRedirect) cleanupAndroidVPNServiceRules() { +} + +func (r *autoRedirect) setupBypassRoute() { +} + +func (r *autoRedirect) updateBypassRoute() error { + return nil +} + +func (r *autoRedirect) cleanupBypassRoute() { +} + +func (r *autoRedirect) UpdateRouteAddressSet() error { + return nil +} diff --git a/redirect_server.go b/redirect_server.go index 1f138ec4..e11dffbd 100644 --- a/redirect_server.go +++ b/redirect_server.go @@ -8,6 +8,7 @@ import ( "net" "net/netip" "sync/atomic" + "syscall" "time" "github.com/sagernet/sing/common/control" @@ -15,31 +16,58 @@ import ( "github.com/sagernet/sing/common/logger" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" ) -type redirectServer struct { - ctx context.Context - handler N.TCPConnectionHandlerEx - logger logger.Logger - listenAddr netip.Addr - listener *net.TCPListener - inShutdown atomic.Bool +type RedirectServer struct { + ctx context.Context + handler N.TCPConnectionHandlerEx + logger logger.Logger + listenAddress netip.Addr + transparent bool + externalTransparent bool + listenMark uint32 + listener *net.TCPListener + inShutdown atomic.Bool +} + +func (s *RedirectServer) SetExternalTransparent() { + s.transparent = true + s.externalTransparent = true } -func newRedirectServer(ctx context.Context, handler N.TCPConnectionHandlerEx, logger logger.Logger, listenAddr netip.Addr) *redirectServer { - return &redirectServer{ - ctx: ctx, - handler: handler, - logger: logger, - listenAddr: listenAddr, +func (s *RedirectServer) ListenerFileDescriptor() (int, error) { + return control.Conn0(s.listener, func(fd uintptr) (int, error) { + return unix.Dup(int(fd)) + }) +} + +func NewRedirectServer(ctx context.Context, handler N.TCPConnectionHandlerEx, logger logger.Logger, listenAddress netip.Addr) *RedirectServer { + return &RedirectServer{ + ctx: ctx, + handler: handler, + logger: logger, + listenAddress: listenAddress, } } -func (s *redirectServer) Start() error { +func (s *RedirectServer) Start() error { var listenConfig net.ListenConfig // listenConfig.KeepAlive = C.TCPKeepAliveInitial listenConfig.KeepAlive = 10 * time.Minute - listener, err := listenConfig.Listen(s.ctx, M.NetworkFromNetAddr("tcp", s.listenAddr), M.SocksaddrFrom(s.listenAddr, 0).String()) + if s.transparent && !s.externalTransparent { + listenConfig.Control = control.Append(func(network string, address string, conn syscall.RawConn) error { + return control.Raw(conn, func(fd uintptr) error { + err := syscall.SetsockoptInt(int(fd), syscall.SOL_IPV6, unix.IPV6_TRANSPARENT, 1) + if err != nil { + return E.Cause(err, "set IPV6_TRANSPARENT") + } + return nil + }) + }, control.RoutingMark(s.listenMark)) + } + listener, err := listenConfig.Listen(s.ctx, M.NetworkFromNetAddr("tcp", s.listenAddress), M.SocksaddrFrom(s.listenAddress, 0).String()) if err != nil { return err } @@ -48,12 +76,16 @@ func (s *redirectServer) Start() error { return nil } -func (s *redirectServer) Close() error { +func (s *RedirectServer) Port() uint16 { + return M.AddrPortFromNet(s.listener.Addr()).Port() +} + +func (s *RedirectServer) Close() error { s.inShutdown.Store(true) return s.listener.Close() } -func (s *redirectServer) loopIn() { +func (s *RedirectServer) loopIn() { var retryDelay time.Duration for { conn, err := s.listener.AcceptTCP() @@ -82,13 +114,20 @@ func (s *redirectServer) loopIn() { } retryDelay = 0 source := M.SocksaddrFromNet(conn.RemoteAddr()).Unwrap() - destination, err := control.GetOriginalDestination(conn) - if err != nil { - _ = conn.SetLinger(0) - _ = conn.Close() - s.logger.Error("process redirect connection from ", source, ": invalid connection: ", err) - continue + var destination M.Socksaddr + if s.transparent && !source.IsIPv4() { + destination = M.SocksaddrFromNet(conn.LocalAddr()).Unwrap() + } else { + var originalDestination netip.AddrPort + originalDestination, err = control.GetOriginalDestination(conn) + if err != nil { + _ = conn.SetLinger(0) + _ = conn.Close() + s.logger.Error("process redirect connection from ", source, ": invalid connection: ", err) + continue + } + destination = M.SocksaddrFromNetIP(originalDestination).Unwrap() } - go s.handler.NewConnectionEx(s.ctx, conn, source, M.SocksaddrFromNetIP(destination).Unwrap(), nil) + go s.handler.NewConnectionEx(s.ctx, conn, source, destination, nil) } } diff --git a/tun.go b/tun.go index e7701222..528bd22c 100644 --- a/tun.go +++ b/tun.go @@ -86,6 +86,7 @@ type Options struct { AutoRedirectInputMark uint32 AutoRedirectOutputMark uint32 AutoRedirectResetMark uint32 + AutoRedirectTProxyMark uint32 AutoRedirectNFQueue uint16 ExcludeMPTCP bool Inet4LoopbackAddress []netip.Addr diff --git a/tun_linux.go b/tun_linux.go index 15515ea9..82b98d8e 100644 --- a/tun_linux.go +++ b/tun_linux.go @@ -3,12 +3,13 @@ package tun import ( "errors" "fmt" - "math/rand" "net" "net/netip" "os" "os/exec" "runtime" + "strconv" + "strings" "sync" "syscall" "unsafe" @@ -329,16 +330,31 @@ func (t *NativeTun) start() error { return nil } - _ = os.WriteFile("/proc/sys/net/ipv4/conf/"+t.options.Name+"/rp_filter", []byte("2"), 0o644) + // The kernel uses max(all, interface) as the effective rp_filter value, so + // writing loose mode here relaxes an inherited strict mode but tightens a + // disabled one, where loose mode still drops replies whose source has no + // route on any interface. + rpFilter := 0 + for _, procPath := range []string{ + "/proc/sys/net/ipv4/conf/all/rp_filter", + "/proc/sys/net/ipv4/conf/" + t.options.Name + "/rp_filter", + } { + content, readErr := os.ReadFile(procPath) + if readErr != nil { + continue + } + value, parseErr := strconv.Atoi(strings.TrimSpace(string(content))) + if parseErr != nil { + continue + } + rpFilter = max(rpFilter, value) + } + if rpFilter == 1 { + _ = os.WriteFile("/proc/sys/net/ipv4/conf/"+t.options.Name+"/rp_filter", []byte("2"), 0o644) + } if t.options.IPRoute2TableIndex == 0 { - for { - t.options.IPRoute2TableIndex = int(rand.Uint32()) - routeList, fErr := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: t.options.IPRoute2TableIndex}, netlink.RT_FILTER_TABLE) - if len(routeList) == 0 || fErr != nil { - break - } - } + t.options.IPRoute2TableIndex = chooseRouteTableIndex() } err = t.setRoute(tunLink) @@ -720,11 +736,17 @@ func (t *NativeTun) rules() []*netlink.Rule { priority6 := priority if t.options.AutoRedirectMarkMode { + inputMark := effectiveMark(t.options.AutoRedirectInputMark, DefaultAutoRedirectInputMark, DefaultAutoRedirectInputMarkAndroid) + outputMark := effectiveMark(t.options.AutoRedirectOutputMark, DefaultAutoRedirectOutputMark, DefaultAutoRedirectOutputMarkAndroid) + resetMark := effectiveMark(t.options.AutoRedirectResetMark, DefaultAutoRedirectResetMark, DefaultAutoRedirectResetMarkAndroid) + tproxyMark := effectiveMark(t.options.AutoRedirectTProxyMark, DefaultAutoRedirectTProxyMark, DefaultAutoRedirectTProxyMarkAndroid) + markMask := int(inputMark | outputMark | resetMark | tproxyMark) if p4 { it = netlink.NewRule() it.Priority = priority - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Goto = priority + 2 it.Family = unix.AF_INET rules = append(rules, it) @@ -732,8 +754,9 @@ func (t *NativeTun) rules() []*netlink.Rule { it = netlink.NewRule() it.Priority = priority - it.Mark = t.options.AutoRedirectInputMark + it.Mark = inputMark it.MarkSet = true + it.Mask = markMask it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET rules = append(rules, it) @@ -747,8 +770,9 @@ func (t *NativeTun) rules() []*netlink.Rule { if p6 { it = netlink.NewRule() it.Priority = priority6 - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Goto = priority6 + 2 it.Family = unix.AF_INET6 rules = append(rules, it) @@ -756,8 +780,9 @@ func (t *NativeTun) rules() []*netlink.Rule { it = netlink.NewRule() it.Priority = priority6 - it.Mark = t.options.AutoRedirectInputMark + it.Mark = inputMark it.MarkSet = true + it.Mask = markMask it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET6 rules = append(rules, it) @@ -768,13 +793,12 @@ func (t *NativeTun) rules() []*netlink.Rule { it.Family = unix.AF_INET6 rules = append(rules, it) } - // Fallback rules after system default rules (32766: main, 32767: default) - // Only reached when main and default tables have no route if p4 { it = netlink.NewRule() it.Priority = t.options.IPRoute2AutoRedirectFallbackRuleIndex - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Invert = true it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET @@ -783,8 +807,9 @@ func (t *NativeTun) rules() []*netlink.Rule { if p6 { it = netlink.NewRule() it.Priority = t.options.IPRoute2AutoRedirectFallbackRuleIndex - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Invert = true it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET6 @@ -1039,6 +1064,14 @@ func (t *NativeTun) rules() []*netlink.Rule { // priority++ } if p6 { + it = netlink.NewRule() + it.Priority = priority6 + it.IifName = t.options.Name + it.Goto = nopPriority + it.Family = unix.AF_INET6 + rules = append(rules, it) + priority6++ + for _, address := range t.options.Inet6Address { it = netlink.NewRule() it.Priority = priority6 @@ -1050,13 +1083,6 @@ func (t *NativeTun) rules() []*netlink.Rule { } priority6++ - it = netlink.NewRule() - it.Priority = priority6 - it.IifName = t.options.Name - it.Goto = nopPriority - it.Family = unix.AF_INET6 - rules = append(rules, it) - it = netlink.NewRule() it.Priority = priority6 it.IifName = "lo" From 266a388988358e869ecae4b4da1c1713b1e7346e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 1 Sep 2026 22:45:20 +0800 Subject: [PATCH 02/25] Fix lint errors --- Makefile | 8 +------- internal/fdbased_darwin/endpoint.go | 2 +- internal/fdbased_darwin/packet_dispatchers.go | 2 +- internal/wintun/memmod/memmod_windows.go | 4 ++-- internal/wintun/memmod/syscall_windows.go | 2 +- ping/port.go | 2 +- redirect_iptables.go | 5 ++--- 7 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index ed36c792..6c5581aa 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,7 @@ build: GOOS=windows GOARCH=amd64 go build -v -tags with_gvisor . fmt: - @gofumpt -l -w . - @gofmt -s -w . - @gci write --custom-order -s standard -s "prefix(github.com/sagernet/)" -s "default" . - -fmt_install: - go install -v mvdan.cc/gofumpt@latest - go install -v github.com/daixiang0/gci@latest + @golangci-lint fmt lint: GOOS=linux golangci-lint --max-same-issues=0 --max-issues-per-linter=0 run ./... diff --git a/internal/fdbased_darwin/endpoint.go b/internal/fdbased_darwin/endpoint.go index b3292a2e..a99b5871 100644 --- a/internal/fdbased_darwin/endpoint.go +++ b/internal/fdbased_darwin/endpoint.go @@ -261,7 +261,7 @@ func New(opts *Options) (stack.LinkEndpoint, error) { } var batchSize int if opts.MultiPendingPackets { - batchSize = int((512*1024)/(opts.MTU)) + 1 + batchSize = int((512*1024)/opts.MTU) + 1 } else { batchSize = 1 } diff --git a/internal/fdbased_darwin/packet_dispatchers.go b/internal/fdbased_darwin/packet_dispatchers.go index afe362f6..4bad9b27 100644 --- a/internal/fdbased_darwin/packet_dispatchers.go +++ b/internal/fdbased_darwin/packet_dispatchers.go @@ -121,7 +121,7 @@ func newRecvMMsgDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, } var batchSize int if opts.MultiPendingPackets { - batchSize = int((512*1024)/(opts.MTU)) + 1 + batchSize = int((512*1024)/opts.MTU) + 1 } else { batchSize = 1 } diff --git a/internal/wintun/memmod/memmod_windows.go b/internal/wintun/memmod/memmod_windows.go index 0b345ba6..37e25560 100644 --- a/internal/wintun/memmod/memmod_windows.go +++ b/internal/wintun/memmod/memmod_windows.go @@ -67,7 +67,7 @@ func (module *Module) copySections(address, size uintptr, oldHeaders *IMAGE_NT_H // Always use position from file to support alignments smaller than page size (allocation above will align to page size). dest := module.codeBase + uintptr(sections[i].VirtualAddress) // NOTE: On 64bit systems we truncate to 32bit here but expand again later when "PhysicalAddress" is used. - sections[i].SetPhysicalAddress((uint32)(dest & 0xffffffff)) + sections[i].SetPhysicalAddress(uint32(dest & 0xffffffff)) dst := unsafe.Slice((*byte)(a2p(dest)), sectionSize) for j := range dst { dst[j] = 0 @@ -94,7 +94,7 @@ func (module *Module) copySections(address, size uintptr, oldHeaders *IMAGE_NT_H address+uintptr(sections[i].PointerToRawData), uintptr(sections[i].SizeOfRawData)) // NOTE: On 64bit systems we truncate to 32bit here but expand again later when "PhysicalAddress" is used. - sections[i].SetPhysicalAddress((uint32)(dest & 0xffffffff)) + sections[i].SetPhysicalAddress(uint32(dest & 0xffffffff)) } return nil diff --git a/internal/wintun/memmod/syscall_windows.go b/internal/wintun/memmod/syscall_windows.go index b79be69e..e1d25d2d 100644 --- a/internal/wintun/memmod/syscall_windows.go +++ b/internal/wintun/memmod/syscall_windows.go @@ -118,7 +118,7 @@ type IMAGE_NT_HEADERS struct { func (ntheader *IMAGE_NT_HEADERS) Sections() []IMAGE_SECTION_HEADER { return (*[0xffff]IMAGE_SECTION_HEADER)(unsafe.Pointer( - (uintptr)(unsafe.Pointer(ntheader)) + + uintptr(unsafe.Pointer(ntheader)) + unsafe.Offsetof(ntheader.OptionalHeader) + uintptr(ntheader.FileHeader.SizeOfOptionalHeader)))[:ntheader.FileHeader.NumberOfSections] } diff --git a/ping/port.go b/ping/port.go index e0cc2a1d..b0b8afc7 100644 --- a/ping/port.go +++ b/ping/port.go @@ -64,7 +64,7 @@ func (p *Port) AttachReturn(returnPath tun.Return) error { if slices.Contains(p.returnPaths, returnPath) { return nil } - p.returnPaths = append(p.returnPaths[:len(p.returnPaths):len(p.returnPaths)], returnPath) + p.returnPaths = append(slices.Clip(p.returnPaths), returnPath) return nil } diff --git a/redirect_iptables.go b/redirect_iptables.go index 1f228e7b..df985065 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -684,8 +684,7 @@ func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { inserts = append(inserts, iptablesInsert{iptablesTableFilter, "FORWARD", []string{"-j", forward.name}}) } - for index := len(inserts) - 1; index >= 0; index-- { - insert := inserts[index] + for _, insert := range slices.Backward(inserts) { err := iptablesRun(family.path, slices.Concat([]string{"-t", insert.table, "-I", insert.base}, insert.args)...) if err != nil { return err @@ -779,7 +778,7 @@ func (r *autoRedirect) cleanupIPTablesForFamily(iptablesPath string) { } var chains []string var references [][]string - for _, line := range strings.Split(output, "\n") { + for line := range strings.SplitSeq(output, "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue From e97d28edb80ac1292fa6e4a0e9169ac57e2c5e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Sep 2026 18:51:31 +0800 Subject: [PATCH 03/25] Add NEON checksum for arm64 Apple M2, internal/checksum_test, median of 3 runs: length generic NEON 64 B 5.47 ns 3.76 ns 128 B 6.63 ns 4.96 ns 512 B 14.8 ns 8.46 ns 1460 B 52.4 ns 16.7 ns 8500 B 341 ns 85.4 ns 65535 B 2661 ns 609 ns --- gtcpip/checksum/checksum_default.go | 2 +- gtcpip/checksum/checksum_ts.go | 2 +- internal/checksum_test/sum_bench_test.go | 25 +++++ internal/checksum_test/sum_test.go | 40 +++++++ internal/tschecksum/checksum_arm64.go | 8 ++ internal/tschecksum/checksum_arm64.s | 133 +++++++++++++++++++++++ internal/tschecksum/checksum_generic.go | 2 +- 7 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 internal/checksum_test/sum_test.go create mode 100644 internal/tschecksum/checksum_arm64.go create mode 100644 internal/tschecksum/checksum_arm64.s diff --git a/gtcpip/checksum/checksum_default.go b/gtcpip/checksum/checksum_default.go index 99a2d753..932481f0 100644 --- a/gtcpip/checksum/checksum_default.go +++ b/gtcpip/checksum/checksum_default.go @@ -1,4 +1,4 @@ -//go:build !amd64 +//go:build !amd64 && !arm64 package checksum diff --git a/gtcpip/checksum/checksum_ts.go b/gtcpip/checksum/checksum_ts.go index f6766d3f..14aa3a47 100644 --- a/gtcpip/checksum/checksum_ts.go +++ b/gtcpip/checksum/checksum_ts.go @@ -1,4 +1,4 @@ -//go:build amd64 +//go:build amd64 || arm64 package checksum diff --git a/internal/checksum_test/sum_bench_test.go b/internal/checksum_test/sum_bench_test.go index fb4aaa4c..9b6ff266 100644 --- a/internal/checksum_test/sum_bench_test.go +++ b/internal/checksum_test/sum_bench_test.go @@ -2,12 +2,37 @@ package checksum_test import ( "crypto/rand" + "strconv" "testing" "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/internal/tschecksum" ) +var benchSizes = []int{64, 128, 512, 1460, 8500, 65535} + +func benchChecksum(b *testing.B, size int, fn func([]byte, uint16) uint16) { + data := make([]byte, size) + rand.Read(data) + b.SetBytes(int64(size)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + fn(data, 0) + } +} + +func BenchmarkTsChecksumSizes(b *testing.B) { + for _, size := range benchSizes { + b.Run(strconv.Itoa(size), func(b *testing.B) { benchChecksum(b, size, tschecksum.Checksum) }) + } +} + +func BenchmarkGChecksumSizes(b *testing.B) { + for _, size := range benchSizes { + b.Run(strconv.Itoa(size), func(b *testing.B) { benchChecksum(b, size, checksum.ChecksumDefault) }) + } +} + func BenchmarkTsChecksum(b *testing.B) { packet := make([][]byte, 1000) for i := range 1000 { diff --git a/internal/checksum_test/sum_test.go b/internal/checksum_test/sum_test.go new file mode 100644 index 00000000..f6fb8d7e --- /dev/null +++ b/internal/checksum_test/sum_test.go @@ -0,0 +1,40 @@ +package checksum_test + +import ( + "crypto/rand" + mathrand "math/rand" + "testing" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/internal/tschecksum" +) + +func TestChecksumDifferential(t *testing.T) { + lengths := make([]int, 0, 2054) + for length := 0; length <= 2049; length++ { + lengths = append(lengths, length) + } + lengths = append(lengths, 4096, 8500, 9000, 65535) + backing := make([]byte, 65535+16) + for _, pattern := range []string{"random", "ones"} { + if pattern == "random" { + rand.Read(backing) + } else { + for i := range backing { + backing[i] = 0xff + } + } + for _, length := range lengths { + for offset := range 16 { + data := backing[offset : offset+length] + initial := uint16(mathrand.Uint32()) + expected := checksum.ChecksumDefault(data, initial) + actual := tschecksum.Checksum(data, initial) + if actual != expected { + t.Fatalf("mismatch: pattern=%s length=%d offset=%d initial=%#04x got %#04x want %#04x", + pattern, length, offset, initial, actual, expected) + } + } + } + } +} diff --git a/internal/tschecksum/checksum_arm64.go b/internal/tschecksum/checksum_arm64.go new file mode 100644 index 00000000..f1964b80 --- /dev/null +++ b/internal/tschecksum/checksum_arm64.go @@ -0,0 +1,8 @@ +package tschecksum + +//go:noescape +func checksumNEON(b []byte, initial uint16) uint16 + +func Checksum(data []byte, initial uint16) uint16 { + return checksumNEON(data, initial) +} diff --git a/internal/tschecksum/checksum_arm64.s b/internal/tschecksum/checksum_arm64.s new file mode 100644 index 00000000..35c64cf2 --- /dev/null +++ b/internal/tschecksum/checksum_arm64.s @@ -0,0 +1,133 @@ +#include "textflag.h" + +// func checksumNEON(b []byte, initial uint16) uint16 +TEXT ·checksumNEON(SB), NOSPLIT|NOFRAME, $0-34 + MOVD b_base+0(FP), R0 + MOVD b_len+8(FP), R1 + MOVHU initial+24(FP), R2 + REV16W R2, R2 + TBZ $0, R1, neonEvenLength + SUB $1, R1 + MOVBU (R0)(R1), R3 + ADD R3, R2 + +neonEvenLength: + CMP $128, R1 + BGE neonChunk + CMP $64, R1 + BLT neonTail32 + LDP (R0), (R4, R5) + LDP 16(R0), (R6, R7) + LDP 32(R0), (R8, R9) + LDP 48(R0), (R10, R11) + ADDS R4, R2 + ADCS R5, R2 + ADCS R6, R2 + ADCS R7, R2 + ADCS R8, R2 + ADCS R9, R2 + ADCS R10, R2 + ADCS R11, R2 + ADC $0, R2 + ADD $64, R0 + SUB $64, R1 + B neonTail32 + +neonChunk: + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + + AND $-64, R1, R3 + MOVD $1048576, R4 + CMP R4, R3 + CSEL LT, R3, R4, R3 + SUB R3, R1 + ADD R3, R0, R5 + +neonInner: + VLD1.P 64(R0), [V16.B16, V17.B16, V18.B16, V19.B16] + VUADDW V16.H4, V0.S4, V0.S4 + VUADDW2 V16.H8, V1.S4, V1.S4 + VUADDW V17.H4, V2.S4, V2.S4 + VUADDW2 V17.H8, V3.S4, V3.S4 + VUADDW V18.H4, V4.S4, V4.S4 + VUADDW2 V18.H8, V5.S4, V5.S4 + VUADDW V19.H4, V6.S4, V6.S4 + VUADDW2 V19.H8, V7.S4, V7.S4 + CMP R5, R0 + BNE neonInner + + VADD V1.S4, V0.S4, V0.S4 + VADD V3.S4, V2.S4, V2.S4 + VADD V5.S4, V4.S4, V4.S4 + VADD V7.S4, V6.S4, V6.S4 + VADD V2.S4, V0.S4, V0.S4 + VADD V6.S4, V4.S4, V4.S4 + VUADDLV V0.S4, V16 + VUADDLV V4.S4, V17 + VMOV V16.D[0], R6 + VMOV V17.D[0], R7 + ADDS R6, R2 + ADCS R7, R2 + ADC $0, R2 + + CMP $64, R1 + BGE neonChunk + +neonTail32: + TBZ $5, R1, neonTail16 + LDP (R0), (R4, R5) + LDP 16(R0), (R6, R7) + ADDS R4, R2 + ADCS R5, R2 + ADCS R6, R2 + ADCS R7, R2 + ADC $0, R2 + ADD $32, R0 + +neonTail16: + TBZ $4, R1, neonTail8 + LDP (R0), (R4, R5) + ADDS R4, R2 + ADCS R5, R2 + ADC $0, R2 + ADD $16, R0 + +neonTail8: + TBZ $3, R1, neonTail4 + MOVD (R0), R4 + ADDS R4, R2 + ADC $0, R2 + ADD $8, R0 + +neonTail4: + TBZ $2, R1, neonTail2 + MOVWU (R0), R4 + ADDS R4, R2 + ADC $0, R2 + ADD $4, R0 + +neonTail2: + TBZ $1, R1, neonFold + MOVHU (R0), R4 + ADDS R4, R2 + ADC $0, R2 + +neonFold: + AND $0xffffffff, R2, R3 + ADD R2>>32, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + REV16W R2, R2 + MOVH R2, ret+32(FP) + RET diff --git a/internal/tschecksum/checksum_generic.go b/internal/tschecksum/checksum_generic.go index 2d6c1348..81e7cdbb 100644 --- a/internal/tschecksum/checksum_generic.go +++ b/internal/tschecksum/checksum_generic.go @@ -1,7 +1,7 @@ // This file contains IP checksum algorithms that are not specific to any // architecture and don't use hardware acceleration. -//go:build !amd64 +//go:build !amd64 && !arm64 package tschecksum From 046f9543bc65630d814426433aed72b331fd7120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Sep 2026 18:52:16 +0800 Subject: [PATCH 04/25] Keep the forward packet network header as a byte slice --- flow_dispatch.go | 11 ++++++----- flow_mtu.go | 2 +- flow_parse.go | 13 ++++++++++--- flow_reject.go | 30 +++++++++++------------------- flow_rewrite.go | 45 ++++++++++++++++++++++++++------------------- 5 files changed, 54 insertions(+), 47 deletions(-) diff --git a/flow_dispatch.go b/flow_dispatch.go index c8562a61..74d49645 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -509,7 +509,7 @@ func (d *ForwardDispatcher) forwardToPort(flow *forwardFlow, packet *forwardPack return } if packet.ipVersion == 4 { - ipHdr := packet.network.(header.IPv4) + ipHdr := header.IPv4(packet.network) if ipHdr.Flags()&header.IPv4FlagDontFragment == 0 { if flow.tracker != nil { flow.tracker.CountForward(len(raw)) @@ -529,7 +529,7 @@ func (d *ForwardDispatcher) forwardToPort(flow *forwardFlow, packet *forwardPack } return } - reply, ok := buildPacketTooBig(packet.network.(header.IPv6), flow.effectiveMTU, d.writeback.ReturnHeadroom()) + reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, d.writeback.ReturnHeadroom()) if ok { d.writebackBatch = append(d.writebackBatch, reply) } @@ -839,9 +839,10 @@ func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed if flow.dnatAddress || flow.dnatPort { rewriteEmbeddedDestination(&embedded, addrToTCPIP(flow.clientDestinationAddress), flow.clientDestinationPort, flow.dnatPort) } - parsed.network.SetDestinationAddr(flow.clientAddress) - if parsed.network.SourceAddr() == flow.serverAddress { - parsed.network.SetSourceAddr(flow.clientDestinationAddress) + networkHeader := parsed.networkHeader() + networkHeader.SetDestinationAddr(flow.clientAddress) + if networkHeader.SourceAddr() == flow.serverAddress { + networkHeader.SetSourceAddr(flow.clientDestinationAddress) } recomputeChecksums(parsed) return true diff --git a/flow_mtu.go b/flow_mtu.go index 6ce275b3..bd9551fc 100644 --- a/flow_mtu.go +++ b/flow_mtu.go @@ -17,7 +17,7 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke } headerLength := len(raw) - len(packet.transport) if packet.ipVersion == 6 && headerLength != header.IPv6MinimumSize { - reply, ok := buildPacketTooBig(packet.network.(header.IPv6), flow.effectiveMTU, d.writeback.ReturnHeadroom()) + reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, d.writeback.ReturnHeadroom()) if ok { d.writebackBatch = append(d.writebackBatch, reply) } diff --git a/flow_parse.go b/flow_parse.go index 2a346710..210846ce 100644 --- a/flow_parse.go +++ b/flow_parse.go @@ -20,7 +20,7 @@ func (k flowKey) reversed() flowKey { type forwardPacket struct { ipVersion uint8 protocol uint8 - network header.Network + network []byte transport []byte source netip.AddrPort destination netip.AddrPort @@ -34,6 +34,13 @@ func (p *forwardPacket) flowKey() flowKey { return flowKey{protocol: p.protocol, source: p.source, destination: p.destination} } +func (p *forwardPacket) networkHeader() header.Network { + if p.ipVersion == 4 { + return header.IPv4(p.network) + } + return header.IPv6(p.network) +} + func (p *forwardPacket) isTCPSyn() bool { return p.protocol == uint8(header.TCPProtocolNumber) && p.tcpFlags&header.TCPFlagSyn != 0 } @@ -48,7 +55,7 @@ func parseForwardPacket(packet []byte) (forwardPacket, bool) { parsed := forwardPacket{ ipVersion: 4, protocol: uint8(ipHdr.TransportProtocol()), - network: ipHdr, + network: packet, source: netip.AddrPortFrom(ipHdr.SourceAddr(), 0), destination: netip.AddrPortFrom(ipHdr.DestinationAddr(), 0), } @@ -67,7 +74,7 @@ func parseForwardPacket(packet []byte) (forwardPacket, bool) { parsed := forwardPacket{ ipVersion: 6, protocol: protocol, - network: ipHdr, + network: packet, source: netip.AddrPortFrom(ipHdr.SourceAddr(), 0), destination: netip.AddrPortFrom(ipHdr.DestinationAddr(), 0), fragment: fragment, diff --git a/flow_reject.go b/flow_reject.go index af82eb5c..5610531d 100644 --- a/flow_reject.go +++ b/flow_reject.go @@ -14,32 +14,24 @@ func buildReject(packet *forwardPacket, headroom int) ([]byte, bool) { return nil, false } tcpHdr := header.TCP(packet.transport) - switch ipHdr := packet.network.(type) { - case header.IPv4: - return buildResetIPv4(ipHdr, tcpHdr, headroom), true - case header.IPv6: - return buildResetIPv6(ipHdr, tcpHdr, headroom), true - default: - return nil, false + if packet.ipVersion == 4 { + return buildResetIPv4(header.IPv4(packet.network), tcpHdr, headroom), true } + return buildResetIPv6(header.IPv6(packet.network), tcpHdr, headroom), true case uint8(header.UDPProtocolNumber): - switch ipHdr := packet.network.(type) { - case header.IPv4: + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) return buildRejectICMPv4(ipHdr, header.ICMPv4PortUnreachable, ipHdr.DestinationAddr(), headroom) - case header.IPv6: - return buildRejectICMPv6(ipHdr, header.ICMPv6PortUnreachable, ipHdr.DestinationAddr(), headroom) - default: - return nil, false } + ipHdr := header.IPv6(packet.network) + return buildRejectICMPv6(ipHdr, header.ICMPv6PortUnreachable, ipHdr.DestinationAddr(), headroom) default: - switch ipHdr := packet.network.(type) { - case header.IPv4: + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) return buildRejectICMPv4(ipHdr, header.ICMPv4HostUnreachable, ipHdr.DestinationAddr(), headroom) - case header.IPv6: - return buildRejectICMPv6(ipHdr, header.ICMPv6AddressUnreachable, ipHdr.DestinationAddr(), headroom) - default: - return nil, false } + ipHdr := header.IPv6(packet.network) + return buildRejectICMPv6(ipHdr, header.ICMPv6AddressUnreachable, ipHdr.DestinationAddr(), headroom) } } diff --git a/flow_rewrite.go b/flow_rewrite.go index a4227654..12559190 100644 --- a/flow_rewrite.go +++ b/flow_rewrite.go @@ -18,8 +18,16 @@ type rewriteRule struct { } func applyRewrite(packet *forwardPacket, rule *rewriteRule) { - oldSource := packet.network.SourceAddress() - oldDestination := packet.network.DestinationAddress() + var oldSource, oldDestination tcpip.Address + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) + oldSource = ipHdr.SourceAddress() + oldDestination = ipHdr.DestinationAddress() + } else { + ipHdr := header.IPv6(packet.network) + oldSource = ipHdr.SourceAddress() + oldDestination = ipHdr.DestinationAddress() + } newSource := oldSource newDestination := oldDestination if rule.sourceAddress.Len() > 0 { @@ -28,7 +36,8 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { if rule.destinationAddress.Len() > 0 { newDestination = rule.destinationAddress } - if ipHdr, isIPv4 := packet.network.(header.IPv4); isIPv4 { + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) if newSource != oldSource { ipHdr.SetSourceAddressWithChecksumUpdate(newSource) } @@ -36,11 +45,12 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { ipHdr.SetDestinationAddressWithChecksumUpdate(newDestination) } } else { + ipHdr := header.IPv6(packet.network) if newSource != oldSource { - packet.network.SetSourceAddress(newSource) + ipHdr.SetSourceAddress(newSource) } if newDestination != oldDestination { - packet.network.SetDestinationAddress(newDestination) + ipHdr.SetDestinationAddress(newDestination) } } transport := packet.transport @@ -121,11 +131,12 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { + networkHeader := packet.networkHeader() if rule.sourceAddress.Len() > 0 { - packet.network.SetSourceAddress(rule.sourceAddress) + networkHeader.SetSourceAddress(rule.sourceAddress) } if rule.destinationAddress.Len() > 0 { - packet.network.SetDestinationAddress(rule.destinationAddress) + networkHeader.SetDestinationAddress(rule.destinationAddress) } transport := packet.transport switch packet.protocol { @@ -175,7 +186,9 @@ func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { } func recomputeChecksums(packet *forwardPacket) { - if ipHdr, isIPv4 := packet.network.(header.IPv4); isIPv4 { + networkHeader := packet.networkHeader() + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) ipHdr.SetChecksum(0) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) } @@ -188,7 +201,7 @@ func recomputeChecksums(packet *forwardPacket) { tcpHdr := header.TCP(transport) tcpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(tcpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, packet.network.SourceAddressSlice(), packet.network.DestinationAddressSlice(), uint16(len(transport))) + pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, networkHeader.SourceAddressSlice(), networkHeader.DestinationAddressSlice(), uint16(len(transport))) tcpHdr.SetChecksum(^tcpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum))) case uint8(header.UDPProtocolNumber): if len(transport) < header.UDPMinimumSize { @@ -200,7 +213,7 @@ func recomputeChecksums(packet *forwardPacket) { } udpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(udpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, packet.network.SourceAddressSlice(), packet.network.DestinationAddressSlice(), udpHdr.Length()) + pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, networkHeader.SourceAddressSlice(), networkHeader.DestinationAddressSlice(), udpHdr.Length()) udpChecksum := ^udpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum)) if udpChecksum == 0 { udpChecksum = 0xffff @@ -221,8 +234,8 @@ func recomputeChecksums(packet *forwardPacket) { icmpHdr.SetChecksum(0) icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ Header: icmpHdr, - Src: packet.network.SourceAddressSlice(), - Dst: packet.network.DestinationAddressSlice(), + Src: networkHeader.SourceAddressSlice(), + Dst: networkHeader.DestinationAddressSlice(), })) } } @@ -240,13 +253,7 @@ func clampTCPMSS(packet *forwardPacket, effectiveMTU uint32) { if tcpHeaderLength < header.TCPMinimumSize || tcpHeaderLength > len(transport) { return } - var networkHeaderLength int - switch packet.ipVersion { - case 4: - networkHeaderLength = len(packet.network.(header.IPv4)) - len(transport) - default: - networkHeaderLength = len(packet.network.(header.IPv6)) - len(transport) - } + networkHeaderLength := len(packet.network) - len(transport) if effectiveMTU <= uint32(networkHeaderLength+header.TCPMinimumSize) { return } From 9ec0e6631c329c4a7103ca4ef3e3651a8db59eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Sep 2026 18:52:26 +0800 Subject: [PATCH 05/25] Drop IPv6 packets carrying extension headers from the forward dispatcher --- flow_parse.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flow_parse.go b/flow_parse.go index 210846ce..7ea3d1a5 100644 --- a/flow_parse.go +++ b/flow_parse.go @@ -70,19 +70,22 @@ func parseForwardPacket(packet []byte) (forwardPacket, bool) { if !ipHdr.IsValid(len(packet)) { return forwardPacket{}, false } - protocol, payload, fragment, transportPresent := skipIPv6ExtensionHeaders(uint8(ipHdr.TransportProtocol()), ipHdr.Payload()) + protocol := uint8(ipHdr.TransportProtocol()) parsed := forwardPacket{ ipVersion: 6, protocol: protocol, network: packet, source: netip.AddrPortFrom(ipHdr.SourceAddr(), 0), destination: netip.AddrPortFrom(ipHdr.DestinationAddr(), 0), - fragment: fragment, } - if fragment || !transportPresent { + switch header.IPv6ExtensionHeaderIdentifier(protocol) { + case header.IPv6HopByHopOptionsExtHdrIdentifier, header.IPv6RoutingExtHdrIdentifier, header.IPv6DestinationOptionsExtHdrIdentifier: + return forwardPacket{}, false + case header.IPv6FragmentExtHdrIdentifier: + parsed.fragment = true return parsed, true } - parsed.parseTransport(payload) + parsed.parseTransport(ipHdr.Payload()) return parsed, true default: return forwardPacket{}, false From a5282d0df59ed972ddca50d1e777b70bae163a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Sep 2026 18:52:45 +0800 Subject: [PATCH 06/25] Raise the utun receive buffer and derive the pending packet limit from it --- tun.go | 10 +++++++--- tun_darwin.go | 48 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/tun.go b/tun.go index 528bd22c..53c032c4 100644 --- a/tun.go +++ b/tun.go @@ -117,11 +117,15 @@ type Options struct { EXP_DisableDNSHijack bool EXP_ExternalConfiguration bool - // For gvisor stack, it should be enabled when MTU is less than 32768; otherwise it should be less than or equal to 8192. - // The above condition is just an estimate and not exact, calculated on M4 pro. + // Safe at every MTU: the darwin pending packet limit is derived from the utun control socket + // receive buffer. EXP_MultiPendingPackets bool - // Will cause the darwin network to die, do not use. + // Do not enable. In netif mode (every utun handed out by Network Extension) utun_pkt_input + // returns ENOSPC without freeing the mbuf once 512 packets are queued + // (net.utun.max_pending_input), leaking one mbuf per rejected write until reboot; sendmsg_x + // pushes the write rate past the netif drain rate (about 500k to 700k packets per second on an + // M4 Pro) and reproduces the device-wide network loss, writev stays below it by its own cost. EXP_SendMsgX bool } diff --git a/tun_darwin.go b/tun_darwin.go index 4b00ac59..46f9922c 100644 --- a/tun_darwin.go +++ b/tun_darwin.go @@ -111,14 +111,14 @@ func New(options Options) (Tun, error) { unix.Close(tunFd) return nil, err } - err = configure(tunFd, options.EXP_MultiPendingPackets, batchSize) + err = configure(tunFd, options.EXP_MultiPendingPackets, int(options.MTU)) if err != nil { unix.Close(tunFd) return nil, err } } else { tunFd = options.FileDescriptor - err := configure(tunFd, options.EXP_MultiPendingPackets, batchSize) + err := configure(tunFd, options.EXP_MultiPendingPackets, int(options.MTU)) if err != nil { return nil, err } @@ -341,21 +341,51 @@ func create(tunFd int, ifIndex int, name string, options Options) error { return nil } -func configure(tunFd int, multiPendingPackets bool, batchSize int) error { +const ( + utunReceiveBufferTarget = 8 << 20 + utunReceiveBufferMinimum = 1 << 20 + utunReceiveBufferDefault = 512 << 10 + utunMaxPendingPackets = 64 +) + +func configure(tunFd int, multiPendingPackets bool, mtu int) error { err := unix.SetNonblock(tunFd, true) if err != nil { return os.NewSyscallError("SetNonblock", err) } - if multiPendingPackets { - const UTUN_OPT_MAX_PENDING_PACKETS = 16 - err = unix.SetsockoptInt(tunFd, 2, UTUN_OPT_MAX_PENDING_PACKETS, batchSize) - if err != nil { - return os.NewSyscallError("SetsockoptInt UTUN_OPT_MAX_PENDING_PACKETS", err) - } + if !multiPendingPackets { + return nil + } + // The utun control socket drops outbound packets with ENOBUFS once the queued bytes reach + // SO_RCVBUF (kern_control.c ctl_rcvbspace; sbspace counts bytes only for SB_KCTL, default + // 512 KB), whereas reaching UTUN_OPT_MAX_PENDING_PACKETS pauses the interface until the + // socket is read (if_utun.c utun_start / utun_ctl_rcvd). SO_RCVBUF is clamped to + // kern.ipc.maxsockbuf by sbreserve. + receiveBuffer := raiseReceiveBuffer(tunFd) + pending := receiveBuffer / 8 * 7 / (mtu + PacketOffset) + pending = max(min(pending, utunMaxPendingPackets), 1) + const UTUN_OPT_MAX_PENDING_PACKETS = 16 + err = unix.SetsockoptInt(tunFd, 2, UTUN_OPT_MAX_PENDING_PACKETS, pending) + if err != nil { + return os.NewSyscallError("SetsockoptInt UTUN_OPT_MAX_PENDING_PACKETS", err) } return nil } +func raiseReceiveBuffer(tunFd int) int { + for size := utunReceiveBufferTarget; size >= utunReceiveBufferMinimum; size /= 2 { + err := unix.SetsockoptInt(tunFd, unix.SOL_SOCKET, unix.SO_RCVBUF, size) + if err == nil { + break + } + } + current, err := unix.GetsockoptInt(tunFd, unix.SOL_SOCKET, unix.SO_RCVBUF) + if err != nil || current <= 0 { + return utunReceiveBufferDefault + } + return current +} + func (t *NativeTun) BatchRead() ([]*buf.Buffer, error) { for i := 0; i < t.batchSize; i++ { iovecs := t.iovecs[i].nextIovecs() From 2fe937fc090b6424604bda0a62db59d9bcc8093e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Sep 2026 18:53:07 +0800 Subject: [PATCH 07/25] Add go stack --- .github/workflows/test.yml | 107 +- Makefile | 2 +- flow_dispatch.go | 510 +++-- flow_mtu.go | 71 +- flow_nat.go | 24 +- flow_parse.go | 1 + go.mod | 2 +- go.sum | 4 +- internal/afd/afd_windows.go | 196 ++ internal/afd/zsyscall_windows.go | 84 + stack.go | 19 +- stack_go.go | 217 ++ stack_go_conn.go | 734 ++++++ stack_go_engine.go | 899 ++++++++ stack_go_integration_darwin_test.go | 323 +++ stack_go_integration_flow_linux_test.go | 478 ++++ stack_go_integration_lifecycle_linux_test.go | 303 +++ stack_go_integration_lifecycle_test.go | 436 ++++ stack_go_integration_linux_test.go | 57 + stack_go_integration_packet_flow_test.go | 347 +++ ...o_integration_packet_offload_linux_test.go | 182 ++ stack_go_integration_packet_test.go | 717 ++++++ stack_go_integration_splice_test.go | 187 ++ stack_go_integration_test.go | 701 ++++++ stack_go_integration_windows_test.go | 40 + stack_go_io.go | 114 + stack_go_io_darwin.go | 832 +++++++ stack_go_io_linux.go | 568 +++++ stack_go_io_other.go | 76 + stack_go_io_windows.go | 431 ++++ stack_go_profile_default.go | 17 + stack_go_profile_ios.go | 15 + stack_go_ring.go | 768 +++++++ stack_go_slab_other.go | 10 + stack_go_slab_unix.go | 23 + stack_go_socket_darwin.go | 170 ++ stack_go_socket_linux.go | 256 +++ stack_go_socket_nonunix.go | 38 + stack_go_socket_unix.go | 122 + stack_go_socket_windows.go | 201 ++ stack_go_splice.go | 965 ++++++++ stack_go_tcp_input.go | 2002 +++++++++++++++++ stack_go_tcp_output.go | 742 ++++++ stack_go_udp.go | 507 +++++ stack_go_wheel.go | 148 ++ stack_gvisor_filter.go | 10 +- stack_mixed.go | 8 +- stack_system.go | 70 +- tun.go | 1 + tun_darwin.go | 25 + tun_linux.go | 95 +- tun_windows.go | 75 +- udp_nat.go | 131 +- udp_nat_cleanup.go | 2 +- 54 files changed, 14646 insertions(+), 417 deletions(-) create mode 100644 internal/afd/afd_windows.go create mode 100644 internal/afd/zsyscall_windows.go create mode 100644 stack_go.go create mode 100644 stack_go_conn.go create mode 100644 stack_go_engine.go create mode 100644 stack_go_integration_darwin_test.go create mode 100644 stack_go_integration_flow_linux_test.go create mode 100644 stack_go_integration_lifecycle_linux_test.go create mode 100644 stack_go_integration_lifecycle_test.go create mode 100644 stack_go_integration_linux_test.go create mode 100644 stack_go_integration_packet_flow_test.go create mode 100644 stack_go_integration_packet_offload_linux_test.go create mode 100644 stack_go_integration_packet_test.go create mode 100644 stack_go_integration_splice_test.go create mode 100644 stack_go_integration_test.go create mode 100644 stack_go_integration_windows_test.go create mode 100644 stack_go_io.go create mode 100644 stack_go_io_darwin.go create mode 100644 stack_go_io_linux.go create mode 100644 stack_go_io_other.go create mode 100644 stack_go_io_windows.go create mode 100644 stack_go_profile_default.go create mode 100644 stack_go_profile_ios.go create mode 100644 stack_go_ring.go create mode 100644 stack_go_slab_other.go create mode 100644 stack_go_slab_unix.go create mode 100644 stack_go_socket_darwin.go create mode 100644 stack_go_socket_linux.go create mode 100644 stack_go_socket_nonunix.go create mode 100644 stack_go_socket_unix.go create mode 100644 stack_go_socket_windows.go create mode 100644 stack_go_splice.go create mode 100644 stack_go_tcp_input.go create mode 100644 stack_go_tcp_output.go create mode 100644 stack_go_udp.go create mode 100644 stack_go_wheel.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be40dea8..3ab6ab14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,15 +9,28 @@ on: - '**.md' - '.github/**' - '!.github/workflows/debug.yml' + - '!.github/workflows/test.yml' pull_request: branches: - main - dev jobs: - build: - name: Linux - runs-on: ubuntu-latest + test: + name: ${{ matrix.os }} (Go ${{ matrix.go }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + go: ['1.25.x', '1.26.x', stable] + include: + - os: windows-latest + go: stable + - os: macos-latest + go: stable + env: + GOTOOLCHAIN: local steps: - name: Checkout uses: actions/checkout@v4 @@ -26,81 +39,21 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: ^1.25.0 - - name: Build + go-version: ${{ matrix.go }} + - name: Install network test tools + if: runner.os == 'Linux' run: | - make test - go test -c -o ping_test ./ping - sudo ./ping_test -test.v - build_go124: - name: Linux (Go 1.24) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ~1.24 - continue-on-error: true - - name: Build - run: | - make test - go test -c -o ping_test ./ping - sudo ./ping_test -test.v - build_go123: - name: Linux (Go 1.23) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ~1.23 - continue-on-error: true - - name: Build + sudo apt-get update + sudo apt-get install -y iproute2 + - name: Test + env: + TEST_FLAGS: ${{ runner.os != 'Windows' && '-exec sudo' || '' }} + run: make test + - name: Test unprivileged ping + if: runner.os != 'Linux' + run: go test -v ./ping + - name: Test privileged ping + if: runner.os != 'Windows' run: | - make test go test -c -o ping_test ./ping sudo ./ping_test -test.v - build_windows: - name: Windows - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ^1.23 - continue-on-error: true - - name: Build - run: | - make test - go test -v ./ping - build_darwin: - name: macOS - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ^1.23 - continue-on-error: true - - name: Build - run: | - make test - go test -v ./ping - go test -c -o ping_test ./ping - sudo ./ping_test -test.v \ No newline at end of file diff --git a/Makefile b/Makefile index 6c5581aa..3b836dde 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,4 @@ lint_install: test: go build -v . #go test -bench=. ./internal/checksum_test - go test -v . + go test -v $(TEST_FLAGS) . diff --git a/flow_dispatch.go b/flow_dispatch.go index 74d49645..a17d3b7c 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -1,13 +1,16 @@ package tun import ( + "encoding/binary" "maps" "net/netip" + "slices" "sync" "sync/atomic" "time" "github.com/sagernet/sing-tun/gtcpip" + "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/logger" @@ -25,6 +28,7 @@ const ( flowTombstoneTimeout = 4 * time.Minute flowTableCapacity = 16384 + flowStageCapacity = 1024 flowSweepInterval = 30 * time.Second flowSweepLimit = flowTableCapacity / int(flowTombstoneTimeout/flowSweepInterval) @@ -35,15 +39,39 @@ type ForwardWriteback interface { WriteReturnPackets(packets [][]byte) error } +type ForwardFrameMeta struct { + needsChecksum bool + checksumStart uint16 + checksumOffset uint16 + gsoType uint8 + gsoSize uint16 +} + +func (m *ForwardFrameMeta) completeChecksum(raw []byte) { + if !m.needsChecksum { + return + } + m.needsChecksum = false + checksumAt := int(m.checksumStart) + int(m.checksumOffset) + if int(m.checksumStart) >= len(raw) || checksumAt+2 > len(raw) { + return + } + initial := binary.BigEndian.Uint16(raw[checksumAt:]) + raw[checksumAt], raw[checksumAt+1] = 0, 0 + binary.BigEndian.PutUint16(raw[checksumAt:], ^checksum.Checksum(raw[m.checksumStart:], initial)) +} + type flowEntry struct { action FlowAction deadline int64 idle time.Duration flow *forwardFlow + verdict FlowVerdict } type forwardFlow struct { nat *portNAT + owner *ForwardStage reverseKey flowKey forwardRule rewriteRule reverseRule rewriteRule @@ -114,20 +142,34 @@ type ForwardDispatcher struct { logger logger.Logger udpTimeout time.Duration icmpTimeout time.Duration - access sync.RWMutex - table map[flowKey]*flowEntry - lastSweep int64 - resetPending atomic.Bool - ports map[Port]*portNAT - natList atomic.Pointer[[]*portNAT] - revNAT atomic.Pointer[map[netip.Addr]*portNAT] + portsAccess sync.Mutex + ports map[Port]*portNAT + natList atomic.Pointer[[]*portNAT] + revNAT atomic.Pointer[map[netip.Addr]*portNAT] + returnPath forwardReturn + stagesAccess sync.Mutex + stages []*ForwardStage + stageCount atomic.Int32 + resetGeneration atomic.Uint32 +} - activeNATs []*portNAT - writebackBatch [][]byte - returnPath forwardReturn - exhaustedLogAt int64 +type stagedPort struct { + nat *portNAT + packets [][]byte +} +type ForwardStage struct { + dispatcher *ForwardDispatcher + writeback ForwardWriteback + access sync.Mutex + table map[flowKey]*flowEntry + lastSweep int64 + resetSeen uint32 + + exhaustedLogAt int64 + ports []stagedPort + writebackBatch [][]byte segmentBuffers [][]byte segmentSizes []int segmentUsed int @@ -148,7 +190,6 @@ func NewForwardDispatcher(handler Handler, writeback ForwardWriteback, logger lo logger: logger, udpTimeout: udpTimeout, icmpTimeout: icmpTimeout, - table: make(map[flowKey]*flowEntry), ports: make(map[Port]*portNAT), } if dispatcher.udpTimeout <= 0 { @@ -161,6 +202,26 @@ func NewForwardDispatcher(handler Handler, writeback ForwardWriteback, logger lo return dispatcher } +func (d *ForwardDispatcher) NewStage(writeback ForwardWriteback) *ForwardStage { + if d == nil { + return nil + } + if writeback == nil { + writeback = d.writeback + } + stage := &ForwardStage{ + dispatcher: d, + writeback: writeback, + table: make(map[flowKey]*flowEntry), + resetSeen: d.resetGeneration.Load(), + } + d.stagesAccess.Lock() + d.stages = append(d.stages, stage) + d.stageCount.Store(int32(len(d.stages))) + d.stagesAccess.Unlock() + return stage +} + func (d *ForwardDispatcher) now() int64 { return int64(time.Since(d.epoch)) } @@ -170,76 +231,100 @@ func (d *ForwardDispatcher) Close() { return } d.returnPath.closed.Store(true) - d.access.Lock() - flows := make([]*forwardFlow, 0, len(d.table)) - for _, entry := range d.table { - if entry.flow != nil { - flows = append(flows, entry.flow) + d.stagesAccess.Lock() + stages := d.stages + d.stagesAccess.Unlock() + for _, stage := range stages { + stage.access.Lock() + for key, entry := range stage.table { + stage.removeEntry(key, entry, FlowCloseReset) } + stage.access.Unlock() } + d.portsAccess.Lock() ports := make([]Port, 0, len(d.ports)) - for port, nat := range d.ports { - if nat != nil { - ports = append(ports, port) - } - } - d.access.Unlock() - for _, flow := range flows { - flow.close(FlowCloseReset) + for port := range d.ports { + ports = append(ports, port) } + d.portsAccess.Unlock() for _, port := range ports { port.DetachReturn(&d.returnPath) } } -func (d *ForwardDispatcher) Dispatch(packet []byte) bool { - if d == nil || d.returnPath.closed.Load() { +func (s *ForwardStage) Dispatch(packet []byte) bool { + if s == nil || s.dispatcher.returnPath.closed.Load() { return false } parsed, ok := parseForwardPacket(packet) - if !ok || parsed.fragment || !parsed.hasFlow { + if !ok { return false } - d.access.RLock() - if d.returnPath.closed.Load() { - d.access.RUnlock() + return s.dispatch(packet, &parsed, nil) +} + +func (s *ForwardStage) DispatchParsed(packet []byte, meta ForwardFrameMeta, parsed *forwardPacket) bool { + if s == nil || s.dispatcher.returnPath.closed.Load() { return false } - key := parsed.flowKey() - now := d.now() - entry, loaded := d.table[key] - if loaded && d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) - loaded = false + return s.dispatch(packet, parsed, &meta) +} + +func (s *ForwardStage) dispatch(packet []byte, parsed *forwardPacket, meta *ForwardFrameMeta) bool { + if parsed.fragment || !parsed.hasFlow { + return false } + key := parsed.flowKey() + now := s.dispatcher.now() + s.access.Lock() + entry, loaded := s.table[key] if loaded { - handled := d.handleHit(key, entry, &parsed, packet, now) - d.access.RUnlock() - return handled + if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) + } else { + handled, remove := s.handleHit(entry, parsed, packet, now, meta) + if remove { + s.removeEntry(key, entry, FlowCloseReset) + } + s.access.Unlock() + return handled + } } - d.access.RUnlock() + s.access.Unlock() if parsed.protocol == uint8(header.TCPProtocolNumber) && (parsed.tcpFlags&header.TCPFlagSyn == 0 || parsed.tcpFlags&header.TCPFlagAck != 0) { return false } - return d.judgeAndInstall(key, &parsed, packet) + return s.judgeAndInstall(key, parsed, packet, meta) } -func (d *ForwardDispatcher) handleHit(key flowKey, entry *flowEntry, packet *forwardPacket, raw []byte, now int64) bool { +func (s *ForwardStage) teardownFlow(key flowKey, reason FlowCloseReason) { + if s == nil || s.dispatcher.returnPath.closed.Load() { + return + } + s.access.Lock() + entry, loaded := s.table[key] + if loaded { + s.removeEntry(key, entry, reason) + } + s.access.Unlock() +} + +func (s *ForwardStage) handleHit(entry *flowEntry, packet *forwardPacket, raw []byte, now int64, meta *ForwardFrameMeta) (handled bool, remove bool) { switch entry.action { case ActionFlow: flow := entry.flow if flow.closed.Load() { - d.tombstoneEntry(entry, now) - return true + tombstoneEntry(entry, now) + return true, false } var flowFinished bool if packet.protocol == uint8(header.TCPProtocolNumber) { if packet.tcpFlags&header.TCPFlagRst != 0 { - d.forwardToPort(flow, packet, raw) + s.forwardToPort(flow, packet, raw, meta) flow.close(FlowCloseReset) - d.tombstoneEntry(entry, now) - return true + tombstoneEntry(entry, now) + return true, false } if packet.tcpFlags&header.TCPFlagFin != 0 { flow.finForward.Store(true) @@ -247,36 +332,38 @@ func (d *ForwardDispatcher) handleHit(key flowKey, entry *flowEntry, packet *for flowFinished = true } } - entry.idle = d.flowIdle(flow) + entry.idle = s.dispatcher.flowIdle(flow) entry.deadline = now + int64(entry.idle) - d.forwardToPort(flow, packet, raw) + s.forwardToPort(flow, packet, raw, meta) if flowFinished { flow.report(FlowCloseFinished) } - return true + return true, false case ActionAccept: if packet.protocol == uint8(header.TCPProtocolNumber) { if packet.tcpFlags&header.TCPFlagRst != 0 { - d.removeEntry(key, entry, FlowCloseReset) - return false + return false, true } if packet.tcpFlags&header.TCPFlagSyn == 0 { entry.idle = tcpEstablishedTimeout } + } else if packet.protocol == uint8(header.UDPProtocolNumber) { + packet.verdict = entry.verdict } entry.deadline = now + int64(entry.idle) - return false + return false, false case ActionReject: entry.deadline = now + int64(entry.idle) - d.stageReject(packet) - return true + s.stageReject(packet, raw, meta) + return true, false default: entry.deadline = now + int64(entry.idle) - return true + return true, false } } -func (d *ForwardDispatcher) judgeAndInstall(key flowKey, packet *forwardPacket, raw []byte) bool { +func (s *ForwardStage) judgeAndInstall(key flowKey, packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) bool { + d := s.dispatcher var firstPacket []byte if packet.protocol == uint8(header.UDPProtocolNumber) { firstPacket = header.UDP(packet.transport).Payload() @@ -285,59 +372,80 @@ func (d *ForwardDispatcher) judgeAndInstall(key flowKey, packet *forwardPacket, if verdict.Action == ActionBypass && verdict.Port != nil { verdict.Action = ActionFlow } - d.access.RLock() - defer d.access.RUnlock() + s.access.Lock() + defer s.access.Unlock() if d.returnPath.closed.Load() { return false } now := d.now() + existing, loaded := s.table[key] + if loaded { + if entryExpired(existing, now) { + s.removeEntry(key, existing, FlowCloseTimeout) + } else { + handled, remove := s.handleHit(existing, packet, raw, now, meta) + if remove { + s.removeEntry(key, existing, FlowCloseReset) + } + return handled + } + } switch verdict.Action { case ActionFlow: if verdict.Port != nil { - flow, result := d.createFlow(packet, verdict) + flow, result := s.createFlow(packet, verdict) if result == createFlowOK { entry := &flowEntry{action: ActionFlow, flow: flow, idle: d.flowIdle(flow)} entry.deadline = now + int64(entry.idle) - d.insertEntry(key, entry, now) - d.forwardToPort(flow, packet, raw) + s.insertEntry(key, entry, now) + s.forwardToPort(flow, packet, raw, meta) return true } if result == createFlowExhausted { - if now-d.exhaustedLogAt >= int64(exhaustedLogInterval) { - d.exhaustedLogAt = now + if now-s.exhaustedLogAt >= int64(exhaustedLogInterval) { + s.exhaustedLogAt = now d.logger.Warn("port selector range exhausted, rejecting flow to ", packet.destination) } - d.installSimple(key, ActionReject, packet.protocol, now) - d.stageReject(packet) + s.installSimple(key, ActionReject, packet.protocol, now) + s.stageReject(packet, raw, meta) return true } } - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false case ActionReject: - d.installSimple(key, ActionReject, packet.protocol, now) - d.stageReject(packet) + s.installSimple(key, ActionReject, packet.protocol, now) + s.stageReject(packet, raw, meta) return true case ActionDrop: - d.installSimple(key, ActionDrop, packet.protocol, now) + s.installSimple(key, ActionDrop, packet.protocol, now) return true case ActionHijackDNS: if packet.protocol == uint8(header.UDPProtocolNumber) { d.hijackDNSPacket(packet) return true } - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false default: - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false } } -func (d *ForwardDispatcher) installSimple(key flowKey, action FlowAction, protocol uint8, now int64) { - entry := &flowEntry{action: action, idle: d.idleTimeout(protocol, false)} +func (s *ForwardStage) installSimple(key flowKey, action FlowAction, protocol uint8, now int64) *flowEntry { + entry := &flowEntry{action: action, idle: s.dispatcher.idleTimeout(protocol, false)} entry.deadline = now + int64(entry.idle) - d.insertEntry(key, entry, now) + s.insertEntry(key, entry, now) + return entry +} + +func (s *ForwardStage) installAccept(key flowKey, packet *forwardPacket, verdict FlowVerdict, now int64) { + entry := s.installSimple(key, ActionAccept, packet.protocol, now) + if packet.protocol == uint8(header.UDPProtocolNumber) { + entry.verdict = verdict + packet.verdict = verdict + } } func (d *ForwardDispatcher) idleTimeout(protocol uint8, established bool) time.Duration { @@ -375,7 +483,7 @@ const ( const exhaustedLogInterval = 5 * time.Second -func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdict) (*forwardFlow, createFlowResult) { +func (s *ForwardStage) createFlow(packet *forwardPacket, verdict FlowVerdict) (*forwardFlow, createFlowResult) { var portAddress netip.Addr inet4Address, inet6Address := verdict.Port.PortAddresses() if packet.ipVersion == 6 { @@ -404,11 +512,11 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic if verdict.Destination.Port() != 0 && !isICMP { serverPort = verdict.Destination.Port() } - nat := d.natFor(verdict.Port) + nat := s.dispatcher.natFor(verdict.Port) if nat == nil { return nil, createFlowUnsupported } - selector, reverseKey, allocated := nat.allocateSelector(packet.protocol, portAddress, serverAddress, serverPort, packet.source.Port()) + selector, reverseKey, allocated := nat.reserveSelector(packet.protocol, portAddress, serverAddress, serverPort, packet.source.Port()) if !allocated { return nil, createFlowExhausted } @@ -418,6 +526,7 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic } flow := &forwardFlow{ nat: nat, + owner: s, reverseKey: reverseKey, effectiveMTU: effectiveMTU, protocol: packet.protocol, @@ -465,6 +574,8 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic } func (d *ForwardDispatcher) natFor(port Port) *portNAT { + d.portsAccess.Lock() + defer d.portsAccess.Unlock() nat, loaded := d.ports[port] if loaded { return nat @@ -498,14 +609,25 @@ func (d *ForwardDispatcher) natFor(port Port) *portNAT { return nat } -func (d *ForwardDispatcher) forwardToPort(flow *forwardFlow, packet *forwardPacket, raw []byte) { - if flow.effectiveMTU != 0 && uint32(len(raw)) > flow.effectiveMTU { +func (s *ForwardStage) forwardToPort(flow *forwardFlow, packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) { + effectiveMTU := flow.effectiveMTU + if meta != nil { + meta.completeChecksum(raw) + if meta.gsoSize != 0 && packet.protocol == uint8(header.TCPProtocolNumber) { + headerLength := len(raw) - len(packet.transport) + int(header.TCP(packet.transport).DataOffset()) + offloadMTU := uint32(headerLength) + uint32(meta.gsoSize) + if effectiveMTU == 0 || offloadMTU < effectiveMTU { + effectiveMTU = offloadMTU + } + } + } + if effectiveMTU != 0 && uint32(len(raw)) > effectiveMTU { if packet.protocol == uint8(header.TCPProtocolNumber) { if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) - d.resegmentTCP(flow, packet, raw) + rewriteForward(flow, packet) + s.resegmentTCP(flow, packet, raw, effectiveMTU) return } if packet.ipVersion == 4 { @@ -514,35 +636,44 @@ func (d *ForwardDispatcher) forwardToPort(flow *forwardFlow, packet *forwardPack if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) + rewriteForward(flow, packet) fragments, ok := fragmentIPv4Packet(ipHdr, flow.effectiveMTU) if ok { for _, fragment := range fragments { - d.stagePort(flow.nat, fragment) + s.stagePort(flow.nat, fragment) } } return } - reply, ok := buildFragmentationNeeded(ipHdr, flow.effectiveMTU, d.writeback.ReturnHeadroom()) + reply, ok := buildFragmentationNeeded(ipHdr, flow.effectiveMTU, s.writeback.ReturnHeadroom()) if ok { - d.writebackBatch = append(d.writebackBatch, reply) + s.writebackBatch = append(s.writebackBatch, reply) } return } - reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, d.writeback.ReturnHeadroom()) + reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, s.writeback.ReturnHeadroom()) if ok { - d.writebackBatch = append(d.writebackBatch, reply) + s.writebackBatch = append(s.writebackBatch, reply) } return } if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) - d.stagePort(flow.nat, raw) + rewriteForward(flow, packet) + if meta != nil { + raw = s.copyForStage(raw) + } + s.stagePort(flow.nat, raw) } -func (d *ForwardDispatcher) rewriteForward(flow *forwardFlow, packet *forwardPacket) { +func (s *ForwardStage) copyForStage(raw []byte) []byte { + staged, _ := s.reserveSegments(1, len(raw)) + copy(staged[0], raw) + return staged[0] +} + +func rewriteForward(flow *forwardFlow, packet *forwardPacket) { if packet.isTCPSyn() { applyRewriteRaw(packet, &flow.forwardRule) clampTCPMSS(packet, flow.effectiveMTU) @@ -552,28 +683,23 @@ func (d *ForwardDispatcher) rewriteForward(flow *forwardFlow, packet *forwardPac } } -func (d *ForwardDispatcher) stagePort(nat *portNAT, packet []byte) { - if len(nat.pending) == 0 { - d.activeNATs = append(d.activeNATs, nat) +func (s *ForwardStage) stagePort(nat *portNAT, packet []byte) { + for index := range s.ports { + if s.ports[index].nat == nat { + s.ports[index].packets = append(s.ports[index].packets, packet) + return + } } - nat.pending = append(nat.pending, packet) + s.ports = append(s.ports, stagedPort{nat: nat, packets: [][]byte{packet}}) } -func (d *ForwardDispatcher) flushPort(nat *portNAT) { - if len(nat.pending) == 0 { - return - } - err := nat.port.WritePackets(nat.pending) - if err != nil { - d.logger.Trace(E.Cause(err, "forward packets")) +func (s *ForwardStage) stageReject(packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) { + if meta != nil { + meta.completeChecksum(raw) } - nat.pending = nat.pending[:0] -} - -func (d *ForwardDispatcher) stageReject(packet *forwardPacket) { - reply, ok := buildReject(packet, d.writeback.ReturnHeadroom()) + reply, ok := buildReject(packet, s.writeback.ReturnHeadroom()) if ok { - d.writebackBatch = append(d.writebackBatch, reply) + s.writebackBatch = append(s.writebackBatch, reply) } } @@ -581,44 +707,62 @@ func (d *ForwardDispatcher) ResetNetwork() { if d == nil { return } - d.resetPending.Store(true) + d.resetGeneration.Add(1) } -func (d *ForwardDispatcher) Flush() { - if d == nil || d.returnPath.closed.Load() { +func (s *ForwardStage) Flush() { + if s == nil { return } - d.access.RLock() - defer d.access.RUnlock() + d := s.dispatcher if d.returnPath.closed.Load() { return } - if d.resetPending.Swap(false) { - for key, entry := range d.table { - d.removeEntry(key, entry, FlowCloseReset) + now := d.now() + generation := d.resetGeneration.Load() + if s.resetSeen != generation { + s.resetSeen = generation + s.access.Lock() + for key, entry := range s.table { + s.removeEntry(key, entry, FlowCloseReset) } + s.access.Unlock() } - for _, nat := range d.activeNATs { - d.flushPort(nat) + for index := range s.ports { + staged := &s.ports[index] + if len(staged.packets) == 0 { + continue + } + err := staged.nat.port.WritePackets(staged.packets) + if err != nil { + d.logger.Trace(E.Cause(err, "forward packets")) + } + clear(staged.packets) + staged.packets = staged.packets[:0] } - d.activeNATs = d.activeNATs[:0] - if retain := max(d.segmentUsed, segmentRetainCount); len(d.segmentBuffers) > retain { - clear(d.segmentBuffers[retain:]) - d.segmentBuffers = d.segmentBuffers[:retain] - d.segmentSizes = d.segmentSizes[:retain] + if retain := max(s.segmentUsed, segmentRetainCount); len(s.segmentBuffers) > retain { + clear(s.segmentBuffers[retain:]) + s.segmentBuffers = s.segmentBuffers[:retain] + s.segmentSizes = s.segmentSizes[:retain] } - d.segmentUsed = 0 - if len(d.writebackBatch) > 0 { - err := d.writeback.WriteReturnPackets(d.writebackBatch) + s.segmentUsed = 0 + if len(s.writebackBatch) > 0 { + err := s.writeback.WriteReturnPackets(s.writebackBatch) if err != nil { d.logger.Trace(E.Cause(err, "write back packets")) } - d.writebackBatch = d.writebackBatch[:0] + clear(s.writebackBatch) + s.writebackBatch = s.writebackBatch[:0] + } + if now-s.lastSweep >= int64(flowSweepInterval) { + s.lastSweep = now + s.access.Lock() + s.sweep(now) + s.access.Unlock() } - d.maybeSweep(d.now()) } -func (d *ForwardDispatcher) entryExpired(entry *flowEntry, now int64) bool { +func entryExpired(entry *flowEntry, now int64) bool { if now <= entry.deadline { return false } @@ -633,14 +777,14 @@ func (d *ForwardDispatcher) entryExpired(entry *flowEntry, now int64) bool { return true } -func (d *ForwardDispatcher) tombstoneEntry(entry *flowEntry, now int64) { +func tombstoneEntry(entry *flowEntry, now int64) { entry.action = ActionDrop entry.idle = flowTombstoneTimeout entry.deadline = now + int64(entry.idle) } -func (d *ForwardDispatcher) removeEntry(key flowKey, entry *flowEntry, reason FlowCloseReason) { - delete(d.table, key) +func (s *ForwardStage) removeEntry(key flowKey, entry *flowEntry, reason FlowCloseReason) { + delete(s.table, key) if entry.flow != nil { if reason == FlowCloseTimeout && entry.flow.finForward.Load() && entry.flow.finReverse.Load() { reason = FlowCloseFinished @@ -650,23 +794,23 @@ func (d *ForwardDispatcher) removeEntry(key flowKey, entry *flowEntry, reason Fl } } -func (d *ForwardDispatcher) insertEntry(key flowKey, entry *flowEntry, now int64) { - if len(d.table) >= flowTableCapacity { - d.evictEntries(now) +func (s *ForwardStage) insertEntry(key flowKey, entry *flowEntry, now int64) { + if len(s.table) >= max(flowTableCapacity/int(s.dispatcher.stageCount.Load()), flowStageCapacity) { + s.evictEntries(now) } - d.table[key] = entry + s.table[key] = entry } -func (d *ForwardDispatcher) evictEntries(now int64) { +func (s *ForwardStage) evictEntries(now int64) { var ( freed int visited int oldestKey flowKey oldest *flowEntry ) - for key, entry := range d.table { - if d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) + for key, entry := range s.table { + if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) freed++ } else if oldest == nil || entry.deadline < oldest.deadline { oldestKey = key @@ -678,21 +822,17 @@ func (d *ForwardDispatcher) evictEntries(now int64) { } } if freed == 0 && oldest != nil { - d.removeEntry(oldestKey, oldest, FlowCloseReset) + s.removeEntry(oldestKey, oldest, FlowCloseReset) } } -func (d *ForwardDispatcher) maybeSweep(now int64) { - if now-d.lastSweep < int64(flowSweepInterval) { - return - } - d.lastSweep = now +func (s *ForwardStage) sweep(now int64) { visited := 0 - for key, entry := range d.table { + for key, entry := range s.table { if entry.action == ActionFlow && entry.flow.closed.Load() { - d.tombstoneEntry(entry, now) - } else if d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) + tombstoneEntry(entry, now) + } else if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) } visited++ if visited >= flowSweepLimit { @@ -724,6 +864,11 @@ const ( returnDrop ) +type returnBatch struct { + writeback ForwardWriteback + packets [][]byte +} + func (r *forwardReturn) ReturnPackets(packets [][]byte) [][]byte { if r.closed.Load() { return packets @@ -739,60 +884,55 @@ func (r *forwardReturn) ReturnPackets(packets [][]byte) [][]byte { } headroom := r.dispatcher.writeback.ReturnHeadroom() now := r.dispatcher.now() - - if len(packets) == 1 { - switch r.classifyReturn(packets[0], natList, revMap, headroom, now) { - case returnWrite: - if err := r.dispatcher.writeback.WriteReturnPackets(packets[:1]); err != nil { - r.dispatcher.logger.Trace(E.Cause(err, "write return packets")) - } - return packets[:0] - case returnDrop: - return packets[:0] - default: - return packets - } - } - unconsumed := packets[:0] - var writeBatch [][]byte + var batches []returnBatch for _, raw := range packets { - switch r.classifyReturn(raw, natList, revMap, headroom, now) { + decision, writeback := r.classifyReturn(raw, natList, revMap, headroom, now) + switch decision { case returnWrite: - writeBatch = append(writeBatch, raw) + index := slices.IndexFunc(batches, func(batch returnBatch) bool { return batch.writeback == writeback }) + if index < 0 { + batches = append(batches, returnBatch{writeback: writeback}) + index = len(batches) - 1 + } + batches[index].packets = append(batches[index].packets, raw) case returnDrop: default: unconsumed = append(unconsumed, raw) } } - if len(writeBatch) > 0 { - if err := r.dispatcher.writeback.WriteReturnPackets(writeBatch); err != nil { + for _, batch := range batches { + err := batch.writeback.WriteReturnPackets(batch.packets) + if err != nil { r.dispatcher.logger.Trace(E.Cause(err, "write return packets")) } } return unconsumed } -func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap map[netip.Addr]*portNAT, headroom int, now int64) returnDecision { +func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap map[netip.Addr]*portNAT, headroom int, now int64) (returnDecision, ForwardWriteback) { if len(raw) < headroom+header.IPv4MinimumSize { - return returnPass + return returnPass, nil } parsed, ok := parseForwardPacket(raw[headroom:]) if !ok || parsed.fragment { - return returnPass + return returnPass, nil } if !parsed.hasFlow { - if parsed.isICMPError() && returnICMPError(natList, revMap, &parsed) { - return returnWrite + if parsed.isICMPError() { + flow := returnICMPError(natList, revMap, &parsed) + if flow != nil { + return returnWrite, flow.owner.writeback + } } - return returnPass + return returnPass, nil } flow := findReverseFlow(natList, revMap, parsed.flowKey()) if flow == nil { - return returnPass + return returnPass, nil } if flow.closed.Load() { - return returnDrop + return returnDrop, nil } if flow.tracker != nil { flow.tracker.CountReverse(len(raw) - headroom) @@ -805,7 +945,7 @@ func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap ma } else { applyRewrite(&parsed, &flow.reverseRule) } - return returnWrite + return returnWrite, flow.owner.writeback } func findReverseFlow(natList []*portNAT, revMap map[netip.Addr]*portNAT, key flowKey) *forwardFlow { @@ -822,18 +962,18 @@ func findReverseFlow(natList []*portNAT, revMap map[netip.Addr]*portNAT, key flo return nil } -func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed *forwardPacket) bool { +func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed *forwardPacket) *forwardFlow { inner, ok := parsed.icmpErrorInner() if !ok { - return false + return nil } embedded, parsedInner := parseEmbedded(inner) if !parsedInner { - return false + return nil } flow := findReverseFlow(natList, revMap, embedded.flowKey().reversed()) if flow == nil || flow.closed.Load() { - return false + return nil } rewriteEmbeddedSource(&embedded, addrToTCPIP(flow.clientAddress), flow.clientSelector, true) if flow.dnatAddress || flow.dnatPort { @@ -845,5 +985,5 @@ func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed networkHeader.SetSourceAddr(flow.clientDestinationAddress) } recomputeChecksums(parsed) - return true + return flow } diff --git a/flow_mtu.go b/flow_mtu.go index bd9551fc..bfbf9c9e 100644 --- a/flow_mtu.go +++ b/flow_mtu.go @@ -1,6 +1,8 @@ package tun import ( + "encoding/binary" + "github.com/sagernet/sing-tun/gtcpip/header" E "github.com/sagernet/sing/common/exceptions" ) @@ -11,15 +13,15 @@ const segmentRetainCount = 128 // Linux delivers TSO aggregates to the TUN even with IFF_VNET_HDR off // (observed on 6.x: the pre-segmentation skb is handed to the fd as-is). -func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacket, raw []byte) { +func (s *ForwardStage) resegmentTCP(flow *forwardFlow, packet *forwardPacket, raw []byte, effectiveMTU uint32) { if len(packet.transport) < header.TCPMinimumSize { return } headerLength := len(raw) - len(packet.transport) if packet.ipVersion == 6 && headerLength != header.IPv6MinimumSize { - reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, d.writeback.ReturnHeadroom()) + reply, ok := buildPacketTooBig(header.IPv6(packet.network), effectiveMTU, s.writeback.ReturnHeadroom()) if ok { - d.writebackBatch = append(d.writebackBatch, reply) + s.writebackBatch = append(s.writebackBatch, reply) } return } @@ -28,7 +30,7 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke return } totalHeaderLength := headerLength + tcpHeaderLength - segmentSize := int(flow.effectiveMTU) - totalHeaderLength + segmentSize := int(effectiveMTU) - totalHeaderLength if segmentSize <= 0 { return } @@ -37,7 +39,7 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke gsoType = GSOTCPv6 } neededSegments := max((len(raw)-totalHeaderLength+segmentSize-1)/segmentSize, 1) - bufs, sizes := d.reserveSegments(neededSegments, int(flow.effectiveMTU)) + bufs, sizes := s.reserveSegments(neededSegments, int(effectiveMTU)) n, err := GSOSplit(raw, GSOOptions{ GSOType: gsoType, HdrLen: uint16(totalHeaderLength), @@ -46,30 +48,30 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke GSOSize: uint16(segmentSize), }, bufs, sizes, 0) if err != nil { - d.logger.Trace(E.Cause(err, "resegment packet")) + s.dispatcher.logger.Trace(E.Cause(err, "resegment packet")) return } for i := range n { - d.stagePort(flow.nat, bufs[i][:sizes[i]]) + s.stagePort(flow.nat, bufs[i][:sizes[i]]) } } -func (d *ForwardDispatcher) reserveSegments(count, size int) ([][]byte, []int) { - start := d.segmentUsed +func (s *ForwardStage) reserveSegments(count, size int) ([][]byte, []int) { + start := s.segmentUsed end := start + count - for len(d.segmentBuffers) < end { - d.segmentBuffers = append(d.segmentBuffers, make([]byte, size)) - d.segmentSizes = append(d.segmentSizes, 0) + for len(s.segmentBuffers) < end { + s.segmentBuffers = append(s.segmentBuffers, make([]byte, size)) + s.segmentSizes = append(s.segmentSizes, 0) } for i := start; i < end; i++ { - if cap(d.segmentBuffers[i]) < size { - d.segmentBuffers[i] = make([]byte, size) + if cap(s.segmentBuffers[i]) < size { + s.segmentBuffers[i] = make([]byte, size) } else { - d.segmentBuffers[i] = d.segmentBuffers[i][:size] + s.segmentBuffers[i] = s.segmentBuffers[i][:size] } } - d.segmentUsed = end - return d.segmentBuffers[start:end], d.segmentSizes[start:end] + s.segmentUsed = end + return s.segmentBuffers[start:end], s.segmentSizes[start:end] } const synthesizedTTL = 64 @@ -106,6 +108,41 @@ func fragmentIPv4Packet(packet header.IPv4, effectiveMTU uint32) ([][]byte, bool return fragments, true } +func fragmentIPv6Packet(packet header.IPv6, effectiveMTU uint32, ident uint32) ([][]byte, bool) { + if len(packet) < header.IPv6MinimumSize { + return nil, false + } + payload := packet.Payload() + maxFragmentPayload := (int(effectiveMTU) - header.IPv6MinimumSize - header.IPv6FragmentHeaderSize) &^ 7 + if maxFragmentPayload <= 0 { + return nil, false + } + transportProtocol := packet.NextHeader() + fragments := make([][]byte, 0, (len(payload)+maxFragmentPayload-1)/maxFragmentPayload) + for start := 0; start < len(payload); start += maxFragmentPayload { + end := min(start+maxFragmentPayload, len(payload)) + fragment := header.IPv6(make([]byte, header.IPv6MinimumSize+header.IPv6FragmentHeaderSize+end-start)) + copy(fragment, packet[:header.IPv6MinimumSize]) + fragment.SetNextHeader(header.IPv6FragmentHeader) + fragment.SetPayloadLength(uint16(header.IPv6FragmentHeaderSize + end - start)) + encodeIPv6FragmentHeader(fragment[header.IPv6MinimumSize:], transportProtocol, start, end < len(payload), ident) + copy(fragment[header.IPv6MinimumSize+header.IPv6FragmentHeaderSize:], payload[start:end]) + fragments = append(fragments, fragment) + } + return fragments, true +} + +func encodeIPv6FragmentHeader(target []byte, nextHeader uint8, offset int, more bool, ident uint32) { + target[0] = nextHeader + target[1] = 0 + value := uint16(offset) + if more { + value |= 1 + } + binary.BigEndian.PutUint16(target[2:], value) + binary.BigEndian.PutUint32(target[4:], ident) +} + func buildFragmentationNeeded(packet header.IPv4, effectiveMTU uint32, headroom int) ([]byte, bool) { advertised := max(effectiveMTU, header.IPv4MinimumMTU) originalLength := min(int(packet.TotalLength()), len(packet)) diff --git a/flow_nat.go b/flow_nat.go index 4d2bca3b..39e4737a 100644 --- a/flow_nat.go +++ b/flow_nat.go @@ -22,8 +22,8 @@ type portNAT struct { selectorStart uint16 selectorCount uint16 - counter uint32 - pending [][]byte + allocateAccess sync.Mutex + counter uint32 } type natShard struct { @@ -70,6 +70,18 @@ func (n *portNAT) insert(key flowKey, flow *forwardFlow) { shard.access.Unlock() } +func (n *portNAT) reserve(key flowKey) bool { + shard := n.shard(key) + shard.access.Lock() + defer shard.access.Unlock() + _, occupied := shard.flows[key] + if occupied { + return false + } + shard.flows[key] = nil + return true +} + func (n *portNAT) delete(key flowKey) { shard := n.shard(key) shard.access.Lock() @@ -100,20 +112,22 @@ func (n *portNAT) selectorRange(protocol uint8) (uint16, uint32) { return n.selectorStart, uint32(n.selectorCount) } -func (n *portNAT) allocateSelector(protocol uint8, portAddress, serverAddress netip.Addr, serverPort, clientSelector uint16) (uint16, flowKey, bool) { +func (n *portNAT) reserveSelector(protocol uint8, portAddress, serverAddress netip.Addr, serverPort, clientSelector uint16) (uint16, flowKey, bool) { rangeStart, rangeCount := n.selectorRange(protocol) if clientSelector != 0 && clientSelector >= rangeStart && uint32(clientSelector-rangeStart) < rangeCount { key := n.reverseKeyFor(protocol, portAddress, serverAddress, serverPort, clientSelector) - if n.lookup(key) == nil { + if n.reserve(key) { return clientSelector, key, true } } + n.allocateAccess.Lock() + defer n.allocateAccess.Unlock() for range rangeCount { n.counter++ candidate := rangeStart + uint16(n.counter%rangeCount) key := n.reverseKeyFor(protocol, portAddress, serverAddress, serverPort, candidate) - if n.lookup(key) == nil { + if n.reserve(key) { return candidate, key, true } } diff --git a/flow_parse.go b/flow_parse.go index 7ea3d1a5..a5faa265 100644 --- a/flow_parse.go +++ b/flow_parse.go @@ -28,6 +28,7 @@ type forwardPacket struct { icmpType uint8 fragment bool hasFlow bool + verdict FlowVerdict } func (p *forwardPacket) flowKey() flowKey { diff --git a/go.mod b/go.mod index 87a60420..56b24c84 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf github.com/sagernet/nftables v0.3.0-mod.4 - github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 + github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd github.com/stretchr/testify v1.11.1 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc diff --git a/go.sum b/go.sum index 6d1e1b40..40a12c3a 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf h1:b/3rm+KvxzuYRc github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= github.com/sagernet/nftables v0.3.0-mod.4 h1:vnOtcDYeSXv2e5RoRuGH0lrpttQFJ8iC4ICS2nhlDSo= github.com/sagernet/nftables v0.3.0-mod.4/go.mod h1:8kslHG4VvYNihcco+i6uxIX7qbT8A56T0y5q7U44ZaQ= -github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 h1:dyRIj+MZ2rc9JVzJoG04jxu+MpvHrLIZLJr0QjNAMGg= -github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8/go.mod h1:olXxWQNqRW/l2Q6JI3b2Qmz8iQnIFlOeeH8bx6JhgUA= +github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd h1:awZQVpsmNdo/EZgEW1nwj9sMQbtgfD130oFIH/7fIDM= +github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd/go.mod h1:K3Owt3xPhHugvlnlPPxZJ/exXdaJfEPOTNorGk4AXjo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= diff --git a/internal/afd/afd_windows.go b/internal/afd/afd_windows.go new file mode 100644 index 00000000..02d0b7e4 --- /dev/null +++ b/internal/afd/afd_windows.go @@ -0,0 +1,196 @@ +package afd + +import ( + "math" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + IOCTL_AFD_POLL = 0x00012024 + + POLL_RECEIVE = 0x0001 + POLL_RECEIVE_EXPEDITED = 0x0002 + POLL_SEND = 0x0004 + POLL_DISCONNECT = 0x0008 + POLL_ABORT = 0x0010 + POLL_LOCAL_CLOSE = 0x0020 + POLL_ACCEPT = 0x0080 + POLL_CONNECT_FAIL = 0x0100 + + SIO_BASE_HANDLE = 0x48000022 + SIO_BSP_HANDLE_POLL = 0x4800001D + + STATUS_PENDING = 0x00000103 + STATUS_CANCELLED = 0xC0000120 + STATUS_NOT_FOUND = 0xC0000225 + + FILE_OPEN = 0x00000001 + OBJ_CASE_INSENSITIVE = 0x00000040 +) + +type PollHandleInfo struct { + Handle windows.Handle + Events uint32 + Status uint32 +} + +type PollInfo struct { + Timeout int64 + NumberOfHandles uint32 + Exclusive uint32 + Handles [1]PollHandleInfo +} + +type OverlappedEntry struct { + CompletionKey uintptr + Overlapped *windows.Overlapped + Internal uintptr + NumberOfBytesTransferred uint32 +} + +type UnicodeString struct { + Length uint16 + MaximumLength uint16 + Buffer *uint16 +} + +type ObjectAttributes struct { + Length uint32 + RootDirectory windows.Handle + ObjectName *UnicodeString + Attributes uint32 + SecurityDescriptor uintptr + SecurityQualityOfService uintptr +} + +type Device struct { + handle windows.Handle +} + +func Open(iocp windows.Handle, name string) (*Device, error) { + deviceName := `\Device\Afd\` + name + deviceNameUTF16, err := windows.UTF16FromString(deviceName) + if err != nil { + return nil, err + } + unicodeString := UnicodeString{ + Length: uint16(len(deviceName) * 2), + MaximumLength: uint16(len(deviceName) * 2), + Buffer: &deviceNameUTF16[0], + } + objectAttributes := ObjectAttributes{ + Length: uint32(unsafe.Sizeof(ObjectAttributes{})), + ObjectName: &unicodeString, + Attributes: OBJ_CASE_INSENSITIVE, + } + var handle windows.Handle + var ioStatusBlock windows.IO_STATUS_BLOCK + err = NtCreateFile(&handle, windows.SYNCHRONIZE, &objectAttributes, &ioStatusBlock, nil, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, FILE_OPEN, 0, 0, 0) + if err != nil { + return nil, err + } + _, err = windows.CreateIoCompletionPort(handle, iocp, 0, 0) + if err != nil { + windows.CloseHandle(handle) + return nil, err + } + err = windows.SetFileCompletionNotificationModes(handle, windows.FILE_SKIP_SET_EVENT_ON_HANDLE) + if err != nil { + windows.CloseHandle(handle) + return nil, err + } + return &Device{handle: handle}, nil +} + +// Poll completes through the completion port with the io status block pointer as the overlapped pointer. +func (d *Device) Poll(baseSocket windows.Handle, events uint32, ioStatusBlock *windows.IO_STATUS_BLOCK, pollInfo *PollInfo) error { + pollInfo.Timeout = math.MaxInt64 + pollInfo.NumberOfHandles = 1 + pollInfo.Exclusive = 0 + pollInfo.Handles[0].Handle = baseSocket + pollInfo.Handles[0].Events = events + pollInfo.Handles[0].Status = 0 + ioStatusBlock.Status = windows.NTStatus(STATUS_PENDING) + size := uint32(unsafe.Sizeof(*pollInfo)) + err := NtDeviceIoControlFile(d.handle, 0, 0, uintptr(unsafe.Pointer(ioStatusBlock)), ioStatusBlock, IOCTL_AFD_POLL, unsafe.Pointer(pollInfo), size, unsafe.Pointer(pollInfo), size) + if err != nil { + if ntstatus, isStatus := err.(windows.NTStatus); isStatus && uint32(ntstatus) == STATUS_PENDING { + return nil + } + return err + } + return nil +} + +func (d *Device) Cancel(ioStatusBlock *windows.IO_STATUS_BLOCK) error { + if uint32(ioStatusBlock.Status) != STATUS_PENDING { + return nil + } + var cancelIOStatusBlock windows.IO_STATUS_BLOCK + err := NtCancelIoFileEx(d.handle, ioStatusBlock, &cancelIOStatusBlock) + if err != nil { + if ntstatus, isStatus := err.(windows.NTStatus); isStatus && (uint32(ntstatus) == STATUS_CANCELLED || uint32(ntstatus) == STATUS_NOT_FOUND) { + return nil + } + return err + } + return nil +} + +func (d *Device) Close() error { + return windows.CloseHandle(d.handle) +} + +func BaseSocket(socket windows.Handle) (windows.Handle, error) { + var baseSocket windows.Handle + var bytesReturned uint32 + for { + err := windows.WSAIoctl(socket, SIO_BASE_HANDLE, nil, 0, (*byte)(unsafe.Pointer(&baseSocket)), uint32(unsafe.Sizeof(baseSocket)), &bytesReturned, nil, 0) + if err != nil { + err = windows.WSAIoctl(socket, SIO_BSP_HANDLE_POLL, nil, 0, (*byte)(unsafe.Pointer(&baseSocket)), uint32(unsafe.Sizeof(baseSocket)), &bytesReturned, nil, 0) + if err != nil { + return socket, nil + } + } + if baseSocket == socket { + return baseSocket, nil + } + socket = baseSocket + } +} + +type WaitCompletionPacket struct { + handle windows.Handle +} + +func WaitCompletionPacketSupported() bool { + return procNtCreateWaitCompletionPacket.Find() == nil && procNtAssociateWaitCompletionPacket.Find() == nil && procNtCancelWaitCompletionPacket.Find() == nil +} + +func NewWaitCompletionPacket() (*WaitCompletionPacket, error) { + var handle windows.Handle + err := NtCreateWaitCompletionPacket(&handle, windows.GENERIC_ALL, nil) + if err != nil { + return nil, err + } + return &WaitCompletionPacket{handle: handle}, nil +} + +func (p *WaitCompletionPacket) Associate(iocp windows.Handle, target windows.Handle, key uintptr) (bool, error) { + var alreadySignaled byte + err := NtAssociateWaitCompletionPacket(p.handle, iocp, target, key, 0, 0, 0, &alreadySignaled) + if err != nil { + return false, err + } + return alreadySignaled != 0, nil +} + +func (p *WaitCompletionPacket) Cancel() error { + return NtCancelWaitCompletionPacket(p.handle, 1) +} + +func (p *WaitCompletionPacket) Close() error { + return windows.CloseHandle(p.handle) +} diff --git a/internal/afd/zsyscall_windows.go b/internal/afd/zsyscall_windows.go new file mode 100644 index 00000000..f56989b8 --- /dev/null +++ b/internal/afd/zsyscall_windows.go @@ -0,0 +1,84 @@ +package afd + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + + procGetQueuedCompletionStatusEx = modkernel32.NewProc("GetQueuedCompletionStatusEx") + procNtCancelIoFileEx = modntdll.NewProc("NtCancelIoFileEx") + procNtCreateFile = modntdll.NewProc("NtCreateFile") + procNtDeviceIoControlFile = modntdll.NewProc("NtDeviceIoControlFile") + procNtCreateWaitCompletionPacket = modntdll.NewProc("NtCreateWaitCompletionPacket") + procNtAssociateWaitCompletionPacket = modntdll.NewProc("NtAssociateWaitCompletionPacket") + procNtCancelWaitCompletionPacket = modntdll.NewProc("NtCancelWaitCompletionPacket") +) + +func GetQueuedCompletionStatusEx(cphandle windows.Handle, entries *OverlappedEntry, count uint32, numRemoved *uint32, timeout uint32, alertable bool) error { + var alertableValue uint32 + if alertable { + alertableValue = 1 + } + r1, _, e1 := syscall.SyscallN(procGetQueuedCompletionStatusEx.Addr(), uintptr(cphandle), uintptr(unsafe.Pointer(entries)), uintptr(count), uintptr(unsafe.Pointer(numRemoved)), uintptr(timeout), uintptr(alertableValue)) + if r1 == 0 { + if e1 != 0 { + return e1 + } + return syscall.EINVAL + } + return nil +} + +func NtCancelIoFileEx(handle windows.Handle, ioRequestToCancel *windows.IO_STATUS_BLOCK, ioStatusBlock *windows.IO_STATUS_BLOCK) error { + r0, _, _ := syscall.SyscallN(procNtCancelIoFileEx.Addr(), uintptr(handle), uintptr(unsafe.Pointer(ioRequestToCancel)), uintptr(unsafe.Pointer(ioStatusBlock))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtCreateFile(handle *windows.Handle, access uint32, objectAttributes *ObjectAttributes, ioStatusBlock *windows.IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eaBuffer uintptr, eaLength uint32) error { + r0, _, _ := syscall.SyscallN(procNtCreateFile.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(objectAttributes)), uintptr(unsafe.Pointer(ioStatusBlock)), uintptr(unsafe.Pointer(allocationSize)), uintptr(attributes), uintptr(share), uintptr(disposition), uintptr(options), uintptr(eaBuffer), uintptr(eaLength)) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtDeviceIoControlFile(handle windows.Handle, event windows.Handle, apcRoutine uintptr, apcContext uintptr, ioStatusBlock *windows.IO_STATUS_BLOCK, ioControlCode uint32, inputBuffer unsafe.Pointer, inputBufferLength uint32, outputBuffer unsafe.Pointer, outputBufferLength uint32) error { + r0, _, _ := syscall.SyscallN(procNtDeviceIoControlFile.Addr(), uintptr(handle), uintptr(event), uintptr(apcRoutine), uintptr(apcContext), uintptr(unsafe.Pointer(ioStatusBlock)), uintptr(ioControlCode), uintptr(inputBuffer), uintptr(inputBufferLength), uintptr(outputBuffer), uintptr(outputBufferLength)) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtCreateWaitCompletionPacket(handle *windows.Handle, access uint32, objectAttributes *ObjectAttributes) error { + r0, _, _ := syscall.SyscallN(procNtCreateWaitCompletionPacket.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(objectAttributes))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtAssociateWaitCompletionPacket(handle windows.Handle, iocp windows.Handle, target windows.Handle, keyContext uintptr, apcContext uintptr, ioStatus uint32, ioStatusInformation uintptr, alreadySignaled *byte) error { + r0, _, _ := syscall.SyscallN(procNtAssociateWaitCompletionPacket.Addr(), uintptr(handle), uintptr(iocp), uintptr(target), keyContext, apcContext, uintptr(ioStatus), ioStatusInformation, uintptr(unsafe.Pointer(alreadySignaled))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtCancelWaitCompletionPacket(handle windows.Handle, removeSignaledPacket byte) error { + r0, _, _ := syscall.SyscallN(procNtCancelWaitCompletionPacket.Addr(), uintptr(handle), uintptr(removeSignaledPacket)) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} diff --git a/stack.go b/stack.go index 613e45dc..8e17419e 100644 --- a/stack.go +++ b/stack.go @@ -18,6 +18,14 @@ type Stack interface { Close() error } +type MemoryPressure uint8 + +const ( + MemoryPressureNone MemoryPressure = iota + MemoryPressureWarning + MemoryPressureCritical +) + type StackOptions struct { Context context.Context Tun Tun @@ -32,6 +40,7 @@ type StackOptions struct { ForwarderBindInterface bool IncludeAllNetworks bool InterfaceFinder control.InterfaceFinder + MemoryPressure func() MemoryPressure } func NewStack( @@ -39,14 +48,8 @@ func NewStack( options StackOptions, ) (Stack, error) { switch stack { - case "": - if options.IncludeAllNetworks { - return NewGVisor(options) - } else if WithGVisor && !options.TunOptions.GSO { - return NewMixed(options) - } else { - return NewSystem(options) - } + case "", "go": + return NewGo(options), nil case "gvisor": return NewGVisor(options) case "mixed": diff --git a/stack_go.go b/stack_go.go new file mode 100644 index 00000000..fa470e11 --- /dev/null +++ b/stack_go.go @@ -0,0 +1,217 @@ +package tun + +import ( + "context" + "errors" + "net/netip" + "runtime" + "sync" + "sync/atomic" + "time" + + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" +) + +type Go struct { + ctx context.Context + tun Tun + mtu int + handler Handler + logger logger.Logger + broadcastAddr netip.Addr + inet4LoopbackAddress []netip.Addr + inet6LoopbackAddress []netip.Addr + udpTimeout time.Duration + icmpTimeout time.Duration + udpNATOptions UDPNatOptions + memoryPressure func() MemoryPressure + udpNats []*UDPNat + udpIdentification atomic.Uint32 + dispatcher *ForwardDispatcher + directory goFlowDirectory + access sync.Mutex + engines []*goEngine + closed atomic.Bool +} + +func NewGo(options StackOptions) *Go { + return &Go{ + ctx: options.Context, + tun: options.Tun, + mtu: int(options.TunOptions.MTU), + handler: options.Handler, + logger: options.Logger, + broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), + inet4LoopbackAddress: options.TunOptions.Inet4LoopbackAddress, + inet6LoopbackAddress: options.TunOptions.Inet6LoopbackAddress, + udpTimeout: options.UDPTimeout, + icmpTimeout: options.ICMPTimeout, + udpNATOptions: UDPNatOptions{ + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, + ExcludeInterface: []string{options.TunOptions.Name}, + }, + memoryPressure: options.MemoryPressure, + } +} + +func (s *Go) Start() error { + s.access.Lock() + defer s.access.Unlock() + if s.closed.Load() { + return E.New("stack is closed") + } + queues, err := newGoPlatformQueues(s) + if err != nil { + return err + } + for index, queue := range queues { + err = queue.start() + if err == nil { + continue + } + if index > 0 && errors.Is(err, errGoQueueUnavailable) { + queues = queues[:index] + break + } + return E.Errors(err, goCloseQueues(queues[:index])) + } + udpNATOptions := s.udpNATOptions + udpNATOptions.Handler = s.handler + udpNATOptions.Prepare = s.prepareUDPConnection + maxSize := udpNATOptions.MaxSize + if maxSize == 0 { + if runtime.GOOS == "ios" { + maxSize = 4096 + } else { + maxSize = 16384 + } + } + natCount := uint32(1) + if udpNATOptions.Mapping == NATMappingAddressAndPortDependent { + natCount = min(uint32(len(queues)), maxSize) + } + udpNats := make([]*UDPNat, natCount) + for index := range udpNats { + udpNATOptions.Shared = index+int(natCount) < len(queues) + udpNATOptions.MaxSize = maxSize / natCount + if uint32(index) < maxSize%natCount { + udpNATOptions.MaxSize++ + } + udpNat := NewUDPNat(udpNATOptions) + err = udpNat.Start() + if err != nil { + udpNat.Close() + for _, started := range udpNats[:index] { + started.Close() + } + return E.Errors(err, goCloseQueues(queues)) + } + udpNats[index] = udpNat + } + s.udpNats = udpNats + s.dispatcher = NewForwardDispatcher(s.handler, &goWriteback{platformIO: queues[0]}, s.logger, s.udpTimeout, s.icmpTimeout) + if len(queues) > 1 { + s.directory.flows = make(map[flowKey]*GoConn) + if natCount > 1 { + s.directory.udpFlows = make(map[udpNatSessionKey]*GoPacketConn) + } + } + s.engines = make([]*goEngine, len(queues)) + for index, queue := range queues { + s.engines[index] = newGoEngine(s, queue, len(queues)) + s.engines[index].udpNat = udpNats[index%int(natCount)] + } + for _, engine := range s.engines { + go engine.run() + } + return nil +} + +func goCloseQueues(queues []goPlatformIO) error { + var err error + for _, queue := range queues { + err = E.Errors(err, queue.close()) + } + return err +} + +type goFlowDirectory struct { + access sync.RWMutex + flows map[flowKey]*GoConn + udpFlows map[udpNatSessionKey]*GoPacketConn +} + +func (d *goFlowDirectory) insert(key flowKey, conn *GoConn) { + if d.flows == nil { + return + } + d.access.Lock() + d.flows[key] = conn + d.access.Unlock() +} + +func (d *goFlowDirectory) remove(key flowKey, conn *GoConn) { + if d.flows == nil { + return + } + d.access.Lock() + if d.flows[key] == conn { + delete(d.flows, key) + } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookup(key flowKey) *GoConn { + if d.flows == nil { + return nil + } + d.access.RLock() + conn := d.flows[key] + d.access.RUnlock() + return conn +} + +func (s *Go) ResetNetwork() { + s.access.Lock() + defer s.access.Unlock() + if s.closed.Load() { + return + } + for _, udpNat := range s.udpNats { + udpNat.Purge() + } + for _, engine := range s.engines { + engine.postMessage(&engine.resetMessage) + } +} + +func (s *Go) Close() error { + s.access.Lock() + if !s.closed.CompareAndSwap(false, true) { + s.access.Unlock() + return nil + } + for _, engine := range s.engines { + engine.postMessage(&engine.closeMessage) + } + s.access.Unlock() + for _, engine := range s.engines { + <-engine.exitSignal + } + if s.dispatcher != nil { + s.dispatcher.Close() + } + for _, udpNat := range s.udpNats { + udpNat.Close() + } + var err error + for _, engine := range s.engines { + err = E.Errors(err, engine.platformIO.close()) + } + return err +} diff --git a/stack_go_conn.go b/stack_go_conn.go new file mode 100644 index 00000000..38199e83 --- /dev/null +++ b/stack_go_conn.go @@ -0,0 +1,734 @@ +package tun + +import ( + "context" + "io" + "net" + "os" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/common/pipe" +) + +const ( + goConnStateJudged uint32 = iota + goConnStateEngaged + goConnStateEstablished + goConnStateAborted + goConnStateDead +) + +const ( + goTCPSynReceived uint8 = iota + goTCPEstablished + goTCPFinWait1 + goTCPFinWait2 + goTCPClosing + goTCPCloseWait + goTCPLastAck + goTCPClosed +) + +const ( + goCloseModeNone uint32 = iota + goCloseModeGraceful + goCloseModeAbort +) + +const ( + goDeathImmediate uint8 = iota + goDeathFinLinger + goDeathAbortLinger +) + +const ( + goTimeUnit = 4096 + goTimestampShift = 20 + goLocalWindowShift = 7 +) + +var errGoReset = E.Cause(syscall.ECONNRESET, "go: connection reset by peer") + +type goPostedReceive struct { + buffer *buf.Buffer + direct []byte + target []byte + filled int +} + +type goConnError struct { + err error +} + +type GoConn struct { + engine *goEngine + key flowKey + source M.Socksaddr + destination M.Socksaddr + epoch int64 + receiveNext uint64 + receiveCapacity uint64 + receiveCapacityMax uint64 + publishedEdge uint64 + receiveRoundTripMark uint64 + receiveRoundTripStamp int64 + receiveSpaceConsumed uint64 + receiveSpaceCopied uint64 + receiveSpaceStamp int64 + receiveChain goSlabChain + oooRanges *goRangeSet + dsackStart uint64 + dsackEnd uint64 + congestionWindow uint32 + slowStartThreshold uint32 + undoCongestionWindow uint32 + undoSlowStartThreshold uint32 + undoRetransmits int32 + smoothedRoundTrip int32 + roundTripVariance int32 + retransmitTimeout int32 + receiveRoundTrip int32 + recoveryPoint uint64 + highestSacked uint64 + highestRetransmit uint64 + pipe uint64 + undoMarker uint64 + undoLimit uint64 + peerWindow uint64 + ackPending uint64 + ackCovered uint64 + lastAckSent uint64 + lastActivity int64 + sweepSentTail uint64 + retransmitPoint uint64 + retransmitHighWater uint64 + windowLeft1 int64 + windowLeft2 int64 + ackedBytes uint32 + keepaliveProbes uint8 + finWait2Since int64 + scoreboard goScoreboard + frtoRecoveryPoint uint64 + frtoSendLimit uint64 + timerNode goWheelNode + retransmitDeadline int64 + probeDeadline int64 + persistDeadline int64 + lingerDeadline int64 + handshakeDeadline int64 + retransmitAttempts uint8 + probeAttempts uint8 + handshakeAttempts uint8 + persistAttempts uint8 + duplicateAckCount uint8 + frtoState uint8 + state uint8 + ipVersion uint8 + deathClass uint8 + peerWindowShift uint8 + localWindowShift uint8 + peerMSS uint16 + effectiveMSS uint16 + lastPeerWindow uint16 + synAckLength uint8 + keyed bool + finPending bool + sackPermitted bool + timestampsEnabled bool + discardReceive bool + finSent bool + finAcked bool + finReceived bool + inRecovery bool + ackForced bool + ackDirty bool + onBlockedList bool + onDyingList bool + clientISN uint32 + sendISN uint32 + finOffset uint64 + dyingSince int64 + ackNext *GoConn + blockedNext *GoConn + dyingNext *GoConn + synAckImage [96]byte + _ [64]byte + + sendPermit atomic.Uint64 + sendUnacked atomic.Uint64 + sendReleased atomic.Uint64 + receiveNextAck atomic.Uint64 + receiveEdge atomic.Uint64 + sentEdge atomic.Uint64 + receiveAvailable atomic.Uint64 + receiveCapacityPublished atomic.Uint64 + windowUpdateThreshold atomic.Uint64 + tsRecent atomic.Uint32 + ident atomic.Uint32 + connState atomic.Uint32 + receiveShutdown atomic.Bool + everEstablished atomic.Bool + receiveDrainable atomic.Bool + receiveReleased atomic.Bool + _ [64]byte + + sentTail atomic.Uint64 + transmittedTail atomic.Uint64 + bufferedTail atomic.Uint64 + writerParked atomic.Bool + retransmitArmed atomic.Bool + writerActive atomic.Int32 + writerNeeds atomic.Int32 + transmitterActive atomic.Int32 + transmitOwner atomic.Int32 + _ [64]byte + + consumedTail atomic.Uint64 + readerParked atomic.Bool + readerActive atomic.Int32 + postedTarget atomic.Pointer[goPostedReceive] + targetDone atomic.Bool + _ [64]byte + + transmitStore goTransmitStore + slabHolder goSlabHolder + transmitScratch [goHeaderScratchSize]byte + transmitSegments [][]byte + descriptors goDescriptorRing + blockedSegment goSegment + blockedFrame [][]byte + blockedMeta ForwardFrameMeta + blockedValid bool + + receiveTarget goPostedReceive + readWaitOptions N.ReadWaitOptions + readDeadline pipe.Deadline + writeDeadline pipe.Deadline + + writeAccess sync.Mutex + readAccess sync.Mutex + readSignal goSignal + writeSignal goSignal + transmitSignal goSignal + establishedSignal chan struct{} + closeSignal chan struct{} + connErr atomic.Pointer[goConnError] + closeMode atomic.Uint32 + writeShut atomic.Bool + readShut atomic.Bool + userClosed atomic.Bool + finRequested atomic.Bool + engageMessage goMessage + closeMessage goMessage + readShutMessage goMessage + retransmitMessage goMessage + windowMessage goMessage + blockedMessage goMessage + droppedMessage goMessage + spliceMessage goMessage + splicePending atomic.Pointer[goSpliceStream] + spliced atomic.Bool + splice *goSpliceStream +} + +func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, destination M.Socksaddr) { + c.engine = engine + c.key = key + c.source = source + c.destination = destination + c.epoch = engine.now() + c.readDeadline = pipe.MakeDeadline() + c.writeDeadline = pipe.MakeDeadline() + c.readSignal = make(goSignal, 1) + c.writeSignal = make(goSignal, 1) + c.transmitSignal = make(goSignal, 1) + c.establishedSignal = make(chan struct{}) + c.closeSignal = make(chan struct{}) + c.receiveChain = goSlabChain{maxSlots: goReceiveCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} + c.transmitStore.chain = goSlabChain{maxSlots: goTransmitCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} + c.descriptors.pool = &engine.descriptorPool + c.timerNode.expire = c.expireTimer + c.engageMessage = goMessage{kind: goMessageConnEngage, conn: c} + c.closeMessage = goMessage{kind: goMessageConnClose, conn: c} + c.readShutMessage = goMessage{kind: goMessageConnReadShut, conn: c} + c.retransmitMessage = goMessage{kind: goMessageConnRetransmit, conn: c} + c.windowMessage = goMessage{kind: goMessageConnWindow, conn: c} + c.blockedMessage = goMessage{kind: goMessageConnBlocked, conn: c} + c.droppedMessage = goMessage{kind: goMessageConnDropped, conn: c} + c.spliceMessage = goMessage{kind: goMessageConnSplice, conn: c} +} + +func (c *GoConn) sendNext() uint64 { + if c.finSent { + return c.finOffset + 1 + } + return c.transmittedTail.Load() +} + +func (c *GoConn) ackThreshold() uint64 { + threshold := min(2*uint64(c.effectiveMSS), c.receiveCapacity/2) + if c.engine.ackCoalescing { + threshold = min(max(threshold, min(c.receiveCapacity/4, goAckCoalesceBytes)), c.receiveCapacity/2) + } + return threshold +} + +func (c *GoConn) stamp(now int64) int32 { + return int32((now - c.epoch) / goTimeUnit) +} + +func (c *GoConn) storeError(err error) { + c.connErr.CompareAndSwap(nil, &goConnError{err: err}) +} + +func (c *GoConn) loadError() error { + value := c.connErr.Load() + if value == nil { + return nil + } + return value.err +} + +func (c *GoConn) closeError() error { + err := c.loadError() + if err != nil { + return err + } + return net.ErrClosed +} + +func (c *GoConn) enterWriter() bool { + c.writerActive.Add(1) + if c.connState.Load() >= goConnStateAborted { + c.writerActive.Add(-1) + c.engine.reapAfterExit() + return false + } + return true +} + +func (c *GoConn) exitWriter() { + c.writerActive.Add(-1) + c.engine.reapAfterExit() + if c.finRequested.Load() { + c.engine.postMessage(&c.closeMessage) + } +} + +func (c *GoConn) enterReader() bool { + c.readerActive.Add(1) + state := c.connState.Load() + if state < goConnStateAborted { + return true + } + if state == goConnStateDead && c.receiveDrainable.Load() && !c.receiveReleased.Load() { + return true + } + c.exitReader() + return false +} + +func (c *GoConn) exitReader() { + c.readerActive.Add(-1) + c.engine.reapAfterExit() +} + +func (c *GoConn) requestAbort(err error) { + c.storeError(err) + c.closeMode.Store(goCloseModeAbort) + c.engine.postMessage(&c.closeMessage) +} + +func (c *GoConn) awaitHandshake(ctx context.Context, deadlineSignal <-chan struct{}) error { + state := c.connState.Load() + if state >= goConnStateEstablished { + return c.handshakeResult() + } + select { + case <-deadlineSignal: + return os.ErrDeadlineExceeded + default: + } + if c.connState.CompareAndSwap(goConnStateJudged, goConnStateEngaged) { + err := c.writeSynAck() + if err != nil { + c.requestAbort(E.Cause(err, "go: send SYN-ACK")) + return err + } + c.engine.postMessage(&c.engageMessage) + } + select { + case <-c.establishedSignal: + return c.handshakeResult() + case <-deadlineSignal: + return os.ErrDeadlineExceeded + case <-ctx.Done(): + c.requestAbort(ctx.Err()) + return E.Cause(ctx.Err(), "go: handshake") + case <-c.engine.stack.ctx.Done(): + cause := context.Cause(c.engine.stack.ctx) + c.requestAbort(cause) + return E.Cause(cause, "go: handshake") + } +} + +func (c *GoConn) handshakeResult() error { + err := c.loadError() + if err != nil { + return err + } + if c.everEstablished.Load() { + return nil + } + if c.connState.Load() >= goConnStateAborted { + return net.ErrClosed + } + return nil +} + +func (c *GoConn) HandshakeFailure(err error) error { + if c.connState.CompareAndSwap(goConnStateJudged, goConnStateAborted) { + c.storeError(err) + writeErr := c.writeReset() + c.closeMode.Store(goCloseModeAbort) + c.engine.postMessage(&c.closeMessage) + return writeErr + } + if c.connState.Load() >= goConnStateAborted { + return os.ErrInvalid + } + c.userClosed.Store(true) + c.requestAbort(err) + return nil +} + +func (c *GoConn) HandshakeSuccess() error { + return c.awaitHandshake(context.Background(), nil) +} + +func (c *GoConn) NeedHandshakeForRead() bool { + return c.connState.Load() < goConnStateEstablished +} + +func (c *GoConn) NeedHandshakeForWrite() bool { + return c.connState.Load() < goConnStateEstablished +} + +func (c *GoConn) InitializeReadWaiter(options N.ReadWaitOptions) bool { + c.readWaitOptions = options + return false +} + +func (c *GoConn) WaitReadBuffer() (*buf.Buffer, error) { + err := c.awaitHandshake(context.Background(), c.readDeadline.Wait()) + if err != nil { + return nil, err + } + c.readAccess.Lock() + defer c.readAccess.Unlock() + if !c.enterReader() { + return nil, c.receiveResult() + } + defer c.exitReader() + for { + select { + case <-c.readDeadline.Wait(): + return nil, os.ErrDeadlineExceeded + default: + } + available := c.receiveAvailable.Load() + consumed := c.consumedTail.Load() + if available > consumed { + buffer := c.readWaitOptions.NewBuffer() + n := c.copyReceived(buffer.FreeBytes(), consumed, available) + buffer.Truncate(n) + c.advanceConsumed(n) + c.readWaitOptions.PostReturn(buffer) + return buffer, nil + } + result := c.receiveResult() + if result != nil { + return nil, result + } + c.receiveTarget.buffer = c.readWaitOptions.NewBuffer() + delivered, parkErr := c.parkReader() + if delivered { + buffer := c.receiveTarget.buffer + c.receiveTarget.buffer = nil + buffer.Truncate(c.receiveTarget.filled) + c.advanceConsumed(c.receiveTarget.filled) + c.readWaitOptions.PostReturn(buffer) + return buffer, nil + } + c.receiveTarget.buffer.Release() + c.receiveTarget.buffer = nil + if parkErr != nil { + return nil, parkErr + } + } +} + +func (c *GoConn) Read(p []byte) (int, error) { + err := c.awaitHandshake(context.Background(), c.readDeadline.Wait()) + if err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + c.readAccess.Lock() + defer c.readAccess.Unlock() + if !c.enterReader() { + return 0, c.receiveResult() + } + defer c.exitReader() + for { + select { + case <-c.readDeadline.Wait(): + return 0, os.ErrDeadlineExceeded + default: + } + available := c.receiveAvailable.Load() + consumed := c.consumedTail.Load() + if available > consumed { + n := c.copyReceived(p, consumed, available) + c.advanceConsumed(n) + return n, nil + } + result := c.receiveResult() + if result != nil { + return 0, result + } + c.receiveTarget.buffer = nil + c.receiveTarget.direct = p + delivered, parkErr := c.parkReader() + c.receiveTarget.direct = nil + if delivered { + n := c.receiveTarget.filled + c.advanceConsumed(n) + return n, nil + } + if parkErr != nil { + return 0, parkErr + } + } +} + +func (c *GoConn) receiveResult() error { + err := c.loadError() + if err != nil { + return err + } + if c.userClosed.Load() { + return net.ErrClosed + } + if c.readShut.Load() || c.receiveShutdown.Load() { + return io.EOF + } + if c.connState.Load() >= goConnStateAborted { + return io.EOF + } + return nil +} + +func (c *GoConn) wakeUser() { + c.readSignal.notify() + c.writeSignal.notify() +} + +func (c *GoConn) copyReceived(target []byte, consumed uint64, available uint64) int { + length := min(len(target), int(available-consumed)) + c.receiveChain.readAt(consumed, target[:length]) + return length +} + +func (c *GoConn) advanceConsumed(n int) { + if c.windowUpdateDue(c.consumedTail.Add(uint64(n))) { + c.engine.postMessage(&c.windowMessage) + } +} + +func (c *GoConn) windowUpdateDue(consumed uint64) bool { + edge := c.sentEdge.Load() + capacity := c.receiveCapacityPublished.Load() + free := consumed + capacity + if free <= edge || free-edge < c.windowUpdateThreshold.Load() { + return false + } + available := c.receiveAvailable.Load() + if available < edge { + window := edge - available + if 2*window > capacity || free-available < 2*window { + return false + } + } + return true +} + +func (c *GoConn) parkReader() (bool, error) { + target := &c.receiveTarget + target.filled = 0 + if target.buffer != nil { + target.target = target.buffer.FreeBytes() + } else { + target.target = target.direct + } + c.targetDone.Store(false) + c.readSignal.drain() + c.postedTarget.Store(target) + c.readerParked.Store(true) + if c.receiveAvailable.Load() > c.consumedTail.Load() || c.receiveResult() != nil { + c.readerParked.Store(false) + return c.claimTarget(target), nil + } + deadlineSignal := c.readDeadline.Wait() + select { + case <-c.readSignal: + c.readerParked.Store(false) + return c.claimTarget(target), nil + case <-deadlineSignal: + c.readerParked.Store(false) + if c.claimTarget(target) { + return true, nil + } + if c.receiveAvailable.Load() > c.consumedTail.Load() { + return false, nil + } + return false, os.ErrDeadlineExceeded + case <-c.closeSignal: + c.readerParked.Store(false) + if c.claimTarget(target) { + return true, nil + } + return false, nil + } +} + +func (c *GoConn) claimTarget(target *goPostedReceive) bool { + if c.postedTarget.CompareAndSwap(target, nil) { + return false + } + for !c.targetDone.Load() { + <-c.readSignal + } + return true +} + +func (c *GoConn) Close() error { + if c.connState.CompareAndSwap(goConnStateJudged, goConnStateAborted) { + c.storeError(net.ErrClosed) + writeErr := c.writeReset() + c.closeMode.Store(goCloseModeAbort) + c.engine.postMessage(&c.closeMessage) + return writeErr + } + if !c.userClosed.CompareAndSwap(false, true) { + return nil + } + c.writeShut.Store(true) + c.readShut.Store(true) + if c.connState.Load() == goConnStateEngaged || c.splicePending.Load() != nil || c.spliced.Load() { + c.requestAbort(net.ErrClosed) + return nil + } + if c.connState.Load() >= goConnStateAborted { + c.engine.postMessage(&c.closeMessage) + c.wakeUser() + return nil + } + if c.receiveAvailable.Load() > c.consumedTail.Load() { + c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeAbort) + } else { + c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeGraceful) + c.finRequested.Store(true) + c.engine.postMessage(&c.readShutMessage) + } + c.engine.postMessage(&c.closeMessage) + c.wakeUser() + return nil +} + +func (c *GoConn) CloseRead() error { + if c.connState.Load() >= goConnStateAborted { + return nil + } + c.readShut.Store(true) + c.engine.postMessage(&c.readShutMessage) + c.readSignal.notify() + return nil +} + +func (c *GoConn) CloseWrite() error { + if c.connState.Load() >= goConnStateAborted { + return nil + } + c.writeShut.Store(true) + c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeGraceful) + c.finRequested.Store(true) + c.engine.postMessage(&c.closeMessage) + c.writeSignal.notify() + return nil +} + +func (c *GoConn) LocalAddr() net.Addr { + return c.source.TCPAddr() +} + +func (c *GoConn) RemoteAddr() net.Addr { + return c.destination.TCPAddr() +} + +func (c *GoConn) SetDeadline(t time.Time) error { + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil +} + +func (c *GoConn) SetReadDeadline(t time.Time) error { + c.readDeadline.Set(t) + return nil +} + +func (c *GoConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline.Set(t) + return nil +} + +type goSignal chan struct{} + +func (s goSignal) notify() bool { + select { + case s <- struct{}{}: + return true + default: + return false + } +} + +func (s goSignal) drain() { + select { + case <-s: + default: + } +} + +var ( + _ net.Conn = (*GoConn)(nil) + _ N.ReadWaiter = (*GoConn)(nil) + _ N.ExtendedWriter = (*GoConn)(nil) + _ N.FrontHeadroom = (*GoConn)(nil) + _ N.WriterWithMTU = (*GoConn)(nil) + _ N.ReadCloser = (*GoConn)(nil) + _ N.WriteCloser = (*GoConn)(nil) + _ N.EarlyReader = (*GoConn)(nil) + _ N.EarlyWriter = (*GoConn)(nil) + _ N.HandshakeSuccess = (*GoConn)(nil) + _ N.HandshakeFailure = (*GoConn)(nil) +) diff --git a/stack_go_engine.go b/stack_go_engine.go new file mode 100644 index 00000000..d9aa1da7 --- /dev/null +++ b/stack_go_engine.go @@ -0,0 +1,899 @@ +package tun + +import ( + "hash/maphash" + "math" + "net" + "net/netip" + "runtime" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +const ( + goWaitIndefinite = time.Duration(-1) + goShutdownBudget = 250 * time.Millisecond + goEngineLoadWindow = 100 * time.Millisecond + goEngineBusyPercent = 70 +) + +const ( + goMessageStackClose uint8 = iota + goMessageStackReset + goMessageConnEngage + goMessageConnClose + goMessageConnReadShut + goMessageConnRetransmit + goMessageConnWindow + goMessageConnBlocked + goMessageConnDropped + goMessageConnSplice + goMessagePacketSplice + goMessagePacketClose + goMessageInject +) + +const goInjectQueueLimit = 512 + +type goInjectedFrame struct { + next *goInjectedFrame + buffer *buf.Buffer + meta ForwardFrameMeta +} + +type goInjectStack struct { + head atomic.Pointer[goInjectedFrame] +} + +func (s *goInjectStack) push(frame *goInjectedFrame) { + for { + head := s.head.Load() + frame.next = head + if s.head.CompareAndSwap(head, frame) { + return + } + } +} + +func (s *goInjectStack) popAll() *goInjectedFrame { + head := s.head.Swap(nil) + var ordered *goInjectedFrame + for head != nil { + next := head.next + head.next = ordered + ordered = head + head = next + } + return ordered +} + +type goMessage struct { + next *goMessage + conn *GoConn + packet *GoPacketConn + queued atomic.Bool + kind uint8 +} + +type goControlStack struct { + head atomic.Pointer[goMessage] +} + +func (s *goControlStack) push(message *goMessage) { + for { + head := s.head.Load() + message.next = head + if s.head.CompareAndSwap(head, message) { + return + } + } +} + +const ( + goEngineRunning uint32 = iota + goEngineParked + goEngineExited +) + +type goEngine struct { + stack *Go + platformIO goPlatformIO + dispatcher *ForwardDispatcher + dispatchStage *ForwardStage + wheel goWheel + controlStack goControlStack + injectStack goInjectStack + injectCount atomic.Int32 + injectMessage goMessage + droppedInjectFrames goDropCounter + engineState atomic.Uint32 + exitReapPending atomic.Bool + exitReapAccess sync.Mutex + coarseTime atomic.Int64 + epoch time.Time + frames []goFrame + closeMessage goMessage + resetMessage goMessage + delayedAckTickNode goWheelNode + reassemblyTickNode goWheelNode + sweepTickNode goWheelNode + reclaimTickNode goWheelNode + reassemblyEntries []goReassemblyEntry + udpUserData goUDPUserData + udpNat *UDPNat + transmitFrame [1][]byte + exitSignal chan struct{} + wokeHandlerThisBurst bool + tunPending bool + transmitReady bool + parkedNanos int64 + loadWindowStart int64 + ackCoalescing bool + inlineTransmitBudget int + socketEvents []goSocketEvent + spliceSlots []goSpliceSlot + spliceFree []uint32 + spliceDirtyList *GoConn + spliceSegments [][]byte + spliceIovecs []goIOVector + packetReceiveBuffers [goPacketBatchSize]*buf.Buffer + packetReceiveBatch int + packetMessages [goPacketBatchSize]goPacketMessage + packetFrames [goPacketBatchSize]goUDPFrame + packetFrameCount int + packetUploads [goPacketBatchSize]goPacketUpload + packetUploadCount int + packetDirtyList *goSplicePacket + packetIO goPacketBatchIO + packetFlows map[flowKey]*GoPacketConn + packetReadOptions atomic.Pointer[N.ReadWaitOptions] + + flows map[flowKey]*GoConn + flowCapacity int + slabPool *goSlabPool + descriptorPool goDescriptorPool + sequenceSeed maphash.Seed + ackList *GoConn + blockedList *GoConn + dyingList *GoConn + reclaimPending bool + resetBurst int + controlIdent uint32 + controlScratch [goHeaderScratchSize]byte + controlSegments [][]byte + sackScratch [goMaxSackBlocks]goSackBlock + sackRaw [goMaxSackBlocks]goRawSackBlock +} + +func (e *goEngine) singleFrame(packet []byte) [][]byte { + e.transmitFrame[0] = packet + return e.transmitFrame[:] +} + +func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine { + engine := &goEngine{ + stack: stack, + platformIO: platformIO, + dispatcher: stack.dispatcher, + dispatchStage: stack.dispatcher.NewStage(&goWriteback{platformIO: platformIO}), + epoch: time.Now(), + frames: make([]goFrame, goReadBatch), + reassemblyEntries: make([]goReassemblyEntry, goReassemblyEntries), + exitSignal: make(chan struct{}), + flows: make(map[flowKey]*GoConn), + flowCapacity: max(goFlowCapacity/engineCount, 1024), + slabPool: newGoSlabPool(stack.memoryPressure, max(goSlabPoolLowWater/engineCount, 8)), + descriptorPool: goDescriptorPool{lowWater: max(goDescriptorPoolLowWater/engineCount, 4)}, + sequenceSeed: maphash.MakeSeed(), + controlSegments: make([][]byte, 0, 8), + socketEvents: make([]goSocketEvent, goSocketEventBatch), + spliceSegments: make([][]byte, 0, 8), + spliceIovecs: make([]goIOVector, 0, 8), + packetFlows: make(map[flowKey]*GoPacketConn, goReadBatch), + packetReceiveBatch: goReceiveBatchMin, + } + engine.closeMessage.kind = goMessageStackClose + engine.resetMessage.kind = goMessageStackReset + engine.injectMessage.kind = goMessageInject + engine.packetReadOptions.Store(new(N.ReadWaitOptions)) + engine.delayedAckTickNode.expire = engine.expireDelayedAckTick + engine.reassemblyTickNode.expire = engine.expireReassemblyTick + engine.sweepTickNode.expire = engine.expireSweepTick + engine.reclaimTickNode.expire = engine.expireReclaimTick + engine.refreshCoarseTime() + engine.wheel.schedule(&engine.sweepTickNode, engine.now()+int64(goSweepInterval)) + engine.wheel.schedule(&engine.reclaimTickNode, engine.now()+int64(goReclaimInterval)) + return engine +} + +func (e *goEngine) run() { + defer close(e.exitSignal) + defer e.exit() + e.loadWindowStart = e.now() + for { + parkStart := int64(time.Since(e.epoch)) + tunReadable, eventCount, err := e.park() + e.parkedNanos += e.now() - parkStart + e.updateLoad(e.now()) + e.drainControlQueue() + if e.stack.closed.Load() { + return + } + transmitWritable := e.platformIO.takeTransmitWritable() + if e.transmitReady || transmitWritable { + e.transmitReady = false + e.releaseBlockedWriters() + } + if err != nil { + e.stack.logger.Error(E.Cause(err, "go: engine wait")) + return + } + e.resetBurst = 0 + e.inlineTransmitBudget = goEngineInlineBurst + e.platformIO.flush() + if tunReadable || e.tunPending { + err = e.processBurst() + if err != nil { + if e.stack.closed.Load() || E.IsClosedOrCanceled(err) { + return + } + e.stack.logger.Error(E.Cause(err, "go: engine read")) + if goFatalReadError(err) { + return + } + } + } + e.dispatchSocketEvents(eventCount) + e.flushSpliceDirty() + e.wheel.advance(e.now()) + e.drainAckList(e.now(), true) + e.dispatchStage.Flush() + e.flushPacketUploads() + e.flushPacketFrames() + e.platformIO.flush() + e.reapDying() + if e.reclaimPending { + e.reclaimPending = false + e.reclaim() + } + if e.wokeHandlerThisBurst { + e.wokeHandlerThisBurst = false + runtime.Gosched() + } + } +} + +func (e *goEngine) exit() { + e.shutdown() + e.releaseReadBuffers() + e.engineState.Store(goEngineExited) + message := e.controlStack.head.Swap(nil) + for message != nil { + next := message.next + message.next = nil + message.queued.Store(false) + e.releasePending(message) + message = next + } + e.releaseInjected() + e.slabPool.close() + if e.dyingList != nil { + e.exitReapPending.Store(true) + e.reapAfterExit() + } +} + +func (e *goEngine) reapAfterExit() { + if !e.exitReapPending.Load() { + return + } + e.exitReapAccess.Lock() + defer e.exitReapAccess.Unlock() + e.refreshCoarseTime() + e.reapDying() + e.exitReapPending.Store(e.dyingList != nil) +} + +func (e *goEngine) updateLoad(now int64) { + elapsed := now - e.loadWindowStart + if elapsed < int64(goEngineLoadWindow) { + return + } + busy := elapsed - e.parkedNanos + e.ackCoalescing = busy*100 >= elapsed*goEngineBusyPercent && len(e.flows) >= 2 + e.parkedNanos = 0 + e.loadWindowStart = now +} + +func (e *goEngine) park() (bool, int, error) { + clear(e.packetFlows) + deadline, scheduled := e.wheel.nextDeadline() + e.engineState.Store(goEngineParked) + if e.controlStack.head.Load() != nil { + e.engineState.Store(goEngineRunning) + e.refreshCoarseTime() + return false, 0, nil + } + timeout := goWaitIndefinite + if scheduled { + timeout = max(time.Duration(deadline-e.now()), 0) + } + if e.tunPending { + timeout = 0 + } + tunReadable, eventCount, err := e.platformIO.wait(0, e.socketEvents) + e.transmitReady = e.platformIO.takeTransmitWritable() + if timeout != 0 && !tunReadable && eventCount == 0 && !e.transmitReady && err == nil && e.controlStack.head.Load() == nil { + e.releaseReadBuffers() + tunReadable, eventCount, err = e.platformIO.wait(timeout, e.socketEvents) + } + e.engineState.Store(goEngineRunning) + e.refreshCoarseTime() + return tunReadable, eventCount, err +} + +func (e *goEngine) releaseReadBuffers() { + e.flushPacketUploads() + e.flushPacketFrames() + clear(e.frames) + e.udpUserData.packet = nil + clear(e.transmitFrame[:]) + clear(e.spliceSegments[:cap(e.spliceSegments)]) + clear(e.spliceIovecs[:cap(e.spliceIovecs)]) + clear(e.controlSegments[:cap(e.controlSegments)]) + buf.ReleaseMulti(e.packetReceiveBuffers[:]) + clear(e.packetReceiveBuffers[:]) + clear(e.packetMessages[:]) + clear(e.packetFrames[:]) + clear(e.packetUploads[:]) + e.packetIO.reset() + e.platformIO.releaseReadBuffers() + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + entry.buffer.Release() + entry.buffer = nil + } + } +} + +func (e *goEngine) postMessage(message *goMessage) { + if !message.queued.CompareAndSwap(false, true) { + return + } + e.controlStack.push(message) + switch e.engineState.Load() { + case goEngineParked: + e.platformIO.wake() + case goEngineExited: + e.releasePending(message) + } +} + +func (e *goEngine) drainControlQueue() { + message := e.controlStack.head.Swap(nil) + for message != nil { + next := message.next + message.next = nil + message.queued.Store(false) + e.handleMessage(message) + message = next + } +} + +func (e *goEngine) handleMessage(message *goMessage) { + switch message.kind { + case goMessageStackReset: + e.dispatcher.ResetNetwork() + e.reclaimPending = true + case goMessageConnEngage: + e.handleEngage(message.conn) + case goMessageConnClose: + e.handleCloseRequest(message.conn) + case goMessageConnReadShut: + e.handleReadShut(message.conn) + case goMessageConnRetransmit: + e.handleRetransmitArm(message.conn) + case goMessageConnWindow: + e.handleWindowUpdate(message.conn) + case goMessageConnBlocked: + e.handleTransmitBlocked(message.conn) + case goMessageConnDropped: + e.handleDroppedFrames(message.conn) + case goMessageConnSplice: + e.handleSpliceEngage(message.conn) + case goMessagePacketSplice: + e.handlePacketSpliceEngage(message.packet) + case goMessagePacketClose: + e.handlePacketSpliceClose(message.packet) + case goMessageInject: + e.processInjected() + } +} + +func (e *goEngine) inject(packet []byte, meta ForwardFrameMeta) { + if e.injectCount.Add(1) > goInjectQueueLimit { + e.injectCount.Add(-1) + e.droppedInjectFrames.record(e.stack.logger, "frames handed over between queues") + return + } + options := e.packetReadOptions.Load() + frame := &goInjectedFrame{buffer: options.NewBufferSize(len(packet)), meta: meta} + common.Must1(frame.buffer.Write(packet)) + options.PostReturn(frame.buffer) + e.injectStack.push(frame) + e.postMessage(&e.injectMessage) +} + +func (e *goEngine) processInjected() { + for frame := e.injectStack.popAll(); frame != nil; { + next := frame.next + e.injectCount.Add(-1) + frame.buffer = e.packetReadOptions.Load().Copy(frame.buffer) + e.processFrame(&goFrame{buffer: frame.buffer, meta: frame.meta}) + e.flushPacketUploads() + frame.buffer.Release() + frame = next + } +} + +func (e *goEngine) releaseInjected() { + for frame := e.injectStack.popAll(); frame != nil; { + next := frame.next + e.injectCount.Add(-1) + frame.buffer.Release() + frame = next + } +} + +func (e *goEngine) handleEngage(conn *GoConn) { + if conn.connState.Load() != goConnStateEngaged { + return + } + conn.handshakeAttempts = 0 + conn.handshakeDeadline = e.now() + int64(goSynAckRetransmit) + e.rearmTimer(conn) +} + +func (e *goEngine) shutdown() { + e.closeAllPacketSplices() + for index := range e.reassemblyEntries { + e.reassemblyEntries[index].active = false + } + e.releaseReadBuffers() + unreset := e.closeAllFlows() + for conn := e.dyingList; conn != nil; conn = conn.dyingNext { + e.spliceDetach(conn, net.ErrClosed) + conn.receiveDrainable.Store(false) + } + deadline := e.now() + int64(goShutdownBudget) + for len(unreset) > 0 { + e.releaseReadBuffers() + remaining := deadline - e.refreshCoarseTime() + if remaining <= 0 { + break + } + armed, err := e.platformIO.armTransmitWritable() + if err != nil || !armed { + time.Sleep(time.Millisecond) + } else { + e.platformIO.wait(min(time.Duration(remaining), 10*time.Millisecond), nil) + if !e.platformIO.takeTransmitWritable() { + time.Sleep(time.Millisecond) + } + } + kept := unreset[:0] + for _, conn := range unreset { + if !e.sendReset(conn) { + kept = append(kept, conn) + } + } + unreset = kept + } + for { + e.reapDying() + if e.dyingList == nil || e.refreshCoarseTime() >= deadline { + return + } + time.Sleep(time.Millisecond) + } +} + +func (e *goEngine) processBurst() error { + remaining := goReadBatch + budget := goEngineBurstBytes + e.tunPending = false + for remaining > 0 && budget > 0 { + count, drained, err := e.platformIO.readBurst(e.frames[:remaining], *e.packetReadOptions.Load()) + if err != nil { + return err + } + for index := range count { + budget -= e.frames[index].buffer.Len() + e.processFrame(&e.frames[index]) + } + e.flushPacketUploads() + if drained || count == 0 { + return nil + } + remaining -= count + } + e.tunPending = true + return nil +} + +func (e *goEngine) processFrame(frame *goFrame) { + if frame.meta.gsoType == goUDPGSOType { + e.processUDPSegments(frame) + return + } + parsed, ok := parseForwardPacket(frame.buffer.Bytes()) + if !ok { + return + } + if e.dropNonUnicast(&parsed) { + return + } + if parsed.fragment { + e.reassemble(frame.buffer.Bytes(), &parsed) + return + } + e.processParsed(frame.buffer, frame.meta, &parsed) +} + +func (e *goEngine) processParsed(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + packet := buffer.Bytes() + if e.handleLoopbackHairpin(packet, meta, parsed) { + return + } + if parsed.protocol == uint8(header.UDPProtocolNumber) && parsed.hasFlow { + writer := e.packetFlows[parsed.flowKey()] + if writer != nil && writer.splice != nil { + conn := writer.conn.Load() + if conn != nil && !conn.isClosed() && conn.service.state.Load() == udpNatStateStarted { + e.packetSpliceInput(writer, buffer, parsed) + return + } + } + } + establishedTCP := parsed.protocol == uint8(header.TCPProtocolNumber) && parsed.hasFlow && !parsed.isPureTCPSyn() + if establishedTCP { + conn := e.flows[parsed.flowKey()] + if conn != nil { + e.inputTCP(conn, parsed) + return + } + } + if e.dispatchStage.DispatchParsed(packet, meta, parsed) { + return + } + if establishedTCP { + owner := e.stack.directory.lookup(parsed.flowKey()) + if owner != nil && owner.engine != e { + owner.engine.inject(packet, meta) + return + } + } + e.demuxL4(buffer, meta, parsed) +} + +func (e *goEngine) demuxL4(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + switch parsed.protocol { + case uint8(header.TCPProtocolNumber): + e.demuxTCP(buffer.Bytes(), parsed) + case uint8(header.UDPProtocolNumber): + e.demuxUDP(buffer, meta, parsed) + case uint8(header.ICMPv4ProtocolNumber), uint8(header.ICMPv6ProtocolNumber): + e.answerEcho(buffer.Bytes(), parsed) + } +} + +func (e *goEngine) dropNonUnicast(parsed *forwardPacket) bool { + destination := parsed.destination.Addr() + if parsed.ipVersion == 4 && destination == e.stack.broadcastAddr { + return true + } + return !destination.IsGlobalUnicast() +} + +func (e *goEngine) handleLoopbackHairpin(packet []byte, meta ForwardFrameMeta, parsed *forwardPacket) bool { + if parsed.protocol != uint8(header.TCPProtocolNumber) || len(parsed.transport) < header.TCPMinimumSize { + return false + } + destination := parsed.destination.Addr() + var loopbackAddresses []netip.Addr + if parsed.ipVersion == 4 { + loopbackAddresses = e.stack.inet4LoopbackAddress + } else { + loopbackAddresses = e.stack.inet6LoopbackAddress + } + if !slices.Contains(loopbackAddresses, destination) { + return false + } + tcpHdr := header.TCP(parsed.transport) + if parsed.ipVersion == 4 { + rewriteIPv4TCP(header.IPv4(parsed.network), tcpHdr, false, + destination, 0, false, + parsed.source.Addr(), 0, false) + } else { + rewriteIPv6TCP(header.IPv6(parsed.network), tcpHdr, false, + destination, 0, false, + parsed.source.Addr(), 0, false) + } + err := e.platformIO.writeFrame(e.singleFrame(packet), meta) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: hairpin write")) + } + return true +} + +func (e *goEngine) now() int64 { + return e.coarseTime.Load() +} + +func (e *goEngine) refreshCoarseTime() int64 { + now := int64(time.Since(e.epoch)) + e.coarseTime.Store(now) + return now +} + +const ( + goReassemblyTimeout = 3 * time.Second + goReassemblyHeadroom = 64 + goReassemblyCapacity = goReassemblyHeadroom + 65535 + goReassemblyMaxRanges = 8 +) + +type goReassemblyRange struct { + start int + end int +} + +type goReassemblyEntry struct { + active bool + headerSeen bool + ipVersion uint8 + protocol uint8 + source netip.Addr + destination netip.Addr + ident uint32 + deadline int64 + headerLength int + totalLength int + buffer *buf.Buffer + headroom int + ranges [goReassemblyMaxRanges]goReassemblyRange + rangeCount int +} + +func (e *goEngine) reassemble(packet []byte, parsed *forwardPacket) { + if parsed.ipVersion == 4 { + e.reassembleIPv4(packet, parsed) + } else { + e.reassembleIPv6(packet, parsed) + } +} + +func (e *goEngine) reassembleIPv4(packet []byte, parsed *forwardPacket) { + ipHdr := header.IPv4(parsed.network) + headerLength := int(ipHdr.HeaderLength()) + if headerLength < header.IPv4MinimumSize || headerLength > len(packet) || headerLength > goReassemblyHeadroom { + return + } + entry := e.reassemblyEntry(4, uint8(ipHdr.TransportProtocol()), parsed.source.Addr(), parsed.destination.Addr(), uint32(ipHdr.ID())) + offset := int(ipHdr.FragmentOffset()) + if offset == 0 { + copy(entry.buffer.Range(entry.headroom-headerLength, entry.headroom), packet[:headerLength]) + entry.headerLength = headerLength + entry.headerSeen = true + } + e.reassemblyAdd(entry, offset, ipHdr.Payload(), ipHdr.More()) +} + +func (e *goEngine) reassembleIPv6(packet []byte, parsed *forwardPacket) { + ipHdr := header.IPv6(parsed.network) + if ipHdr.NextHeader() != header.IPv6FragmentHeader { + return + } + fragHdr := header.IPv6Fragment(ipHdr.Payload()) + if !fragHdr.IsValid() { + return + } + entry := e.reassemblyEntry(6, fragHdr.NextHeader(), parsed.source.Addr(), parsed.destination.Addr(), fragHdr.ID()) + if !entry.headerSeen { + copy(entry.buffer.Range(entry.headroom-header.IPv6MinimumSize, entry.headroom), packet[:header.IPv6MinimumSize]) + entry.headerLength = header.IPv6MinimumSize + entry.headerSeen = true + } + e.reassemblyAdd(entry, int(fragHdr.FragmentOffset())*8, fragHdr.Payload(), fragHdr.More()) +} + +func (e *goEngine) reassemblyEntry(ipVersion uint8, protocol uint8, source netip.Addr, destination netip.Addr, ident uint32) *goReassemblyEntry { + now := e.now() + var free *goReassemblyEntry + var oldest *goReassemblyEntry + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if entry.active && entry.deadline <= now { + entry.active = false + } + if !entry.active { + if free == nil { + free = entry + } + continue + } + if entry.ipVersion == ipVersion && entry.protocol == protocol && entry.ident == ident && + entry.source == source && entry.destination == destination { + return entry + } + if oldest == nil || entry.deadline < oldest.deadline { + oldest = entry + } + } + entry := free + if entry == nil { + entry = oldest + } + options := e.packetReadOptions.Load() + bufferSize := options.FrontHeadroom + goReassemblyCapacity + options.RearHeadroom + if entry.buffer == nil || entry.buffer.RawCap() < bufferSize { + entry.buffer.Release() + entry.buffer = buf.NewSize(bufferSize) + } else { + entry.buffer.Reset() + } + entry.headroom = options.FrontHeadroom + goReassemblyHeadroom + entry.active = true + entry.headerSeen = false + entry.ipVersion = ipVersion + entry.protocol = protocol + entry.source = source + entry.destination = destination + entry.ident = ident + entry.deadline = now + int64(goReassemblyTimeout) + entry.headerLength = 0 + entry.totalLength = -1 + entry.rangeCount = 0 + if e.reassemblyTickNode.slot == nil { + e.wheel.schedule(&e.reassemblyTickNode, entry.deadline) + } + return entry +} + +func (e *goEngine) reassemblyAdd(entry *goReassemblyEntry, offset int, fragment []byte, more bool) { + if len(fragment) == 0 { + return + } + end := offset + len(fragment) + if end > 65535 { + entry.active = false + return + } + if !more { + if entry.totalLength >= 0 && entry.totalLength != end { + entry.active = false + return + } + entry.totalLength = end + } + if entry.totalLength >= 0 && end > entry.totalLength { + entry.active = false + return + } + copy(entry.buffer.Range(entry.headroom+offset, entry.headroom+end), fragment) + if !reassemblyMergeRanges(entry, offset, end) { + entry.active = false + return + } + if entry.headerSeen && entry.totalLength >= 0 && entry.rangeCount == 1 && + entry.ranges[0].start == 0 && entry.ranges[0].end == entry.totalLength { + e.completeReassembly(entry) + } +} + +func reassemblyMergeRanges(entry *goReassemblyEntry, start int, end int) bool { + count := entry.rangeCount + index := 0 + for index < count && entry.ranges[index].end < start { + index++ + } + if index == count || entry.ranges[index].start > end { + if count == goReassemblyMaxRanges { + return false + } + copy(entry.ranges[index+1:count+1], entry.ranges[index:count]) + entry.ranges[index] = goReassemblyRange{start: start, end: end} + entry.rangeCount = count + 1 + return true + } + mergedStart := min(start, entry.ranges[index].start) + mergedEnd := end + mergeEnd := index + for mergeEnd < count && entry.ranges[mergeEnd].start <= end { + mergedEnd = max(mergedEnd, entry.ranges[mergeEnd].end) + mergeEnd++ + } + entry.ranges[index] = goReassemblyRange{start: mergedStart, end: mergedEnd} + copy(entry.ranges[index+1:], entry.ranges[mergeEnd:count]) + entry.rangeCount = count - (mergeEnd - index) + 1 + return true +} + +func (e *goEngine) completeReassembly(entry *goReassemblyEntry) { + entry.active = false + if entry.ipVersion == 4 && entry.headerLength+entry.totalLength > 65535 { + return + } + buffer := entry.buffer + buffer.Resize(entry.headroom-entry.headerLength, entry.headerLength+entry.totalLength) + packet := buffer.Bytes() + if entry.ipVersion == 4 { + ipHdr := header.IPv4(packet) + ipHdr.SetTotalLength(uint16(len(packet))) + ipHdr.SetFlagsFragmentOffset(ipHdr.Flags()&^header.IPv4FlagMoreFragments, 0) + ipHdr.SetChecksum(0) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + } else { + ipHdr := header.IPv6(packet) + ipHdr.SetNextHeader(entry.protocol) + ipHdr.SetPayloadLength(uint16(entry.totalLength)) + } + parsed, ok := parseForwardPacket(packet) + if !ok || parsed.fragment { + return + } + e.processParsed(buffer, ForwardFrameMeta{}, &parsed) + e.flushPacketUploads() +} + +func (e *goEngine) expireReassemblyTick(now int64) { + next := int64(math.MaxInt64) + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + continue + } + if entry.deadline <= now { + entry.active = false + } else { + next = min(next, entry.deadline) + } + } + if next != math.MaxInt64 { + e.wheel.schedule(&e.reassemblyTickNode, next) + } +} + +func (e *goEngine) answerEcho(packet []byte, parsed *forwardPacket) { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + if len(parsed.transport) < header.ICMPv4MinimumSize { + return + } + if !rewriteEchoReplyIPv4(header.IPv4(parsed.network), header.ICMPv4(parsed.transport)) { + return + } + case uint8(header.ICMPv6ProtocolNumber): + if len(parsed.transport) < header.ICMPv6MinimumSize { + return + } + if !rewriteEchoReplyIPv6(header.IPv6(parsed.network), header.ICMPv6(parsed.transport)) { + return + } + default: + return + } + err := e.platformIO.writeFrame(e.singleFrame(packet), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write echo reply")) + } +} diff --git a/stack_go_integration_darwin_test.go b/stack_go_integration_darwin_test.go new file mode 100644 index 00000000..daf4b4ec --- /dev/null +++ b/stack_go_integration_darwin_test.go @@ -0,0 +1,323 @@ +//go:build darwin && !ios + +package tun + +import ( + "bytes" + "fmt" + "io" + "net" + "os/exec" + "runtime" + "sync/atomic" + "testing" + "time" + + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +const ( + kernelConnectionRefused = unix.ECONNREFUSED + kernelConnectionReset = unix.ECONNRESET +) + +func TestGoKernelMemoryPressure(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 1500, prepare: prepareKernelNetif}, {mtu: 9000, prepare: prepareKernelNetif}} + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/netif=%v", config.mtu, config.prepare != nil), func(configTest *testing.T) { + var pressure atomic.Uint32 + config.pressure = func() MemoryPressure { return MemoryPressure(pressure.Load()) } + fixture := newKernelStackFixture(configTest, config) + configTest.Logf("utun netif mode: %v", fixture.stack.engines[0].platformIO.(*goDarwinIO).netif) + for _, level := range []MemoryPressure{MemoryPressureWarning, MemoryPressureCritical, MemoryPressureNone} { + configTest.Run(fmt.Sprintf("level=%d", level), func(test *testing.T) { + pressure.Store(uint32(level)) + for _, splice := range []bool{false, true} { + test.Run(fmt.Sprintf("splice=%v", splice), func(flowTest *testing.T) { + var client, server net.Conn + if splice { + client, server, _, _ = fixture.splicePair(flowTest, true) + } else { + client, server = fixture.pair(flowTest, true) + } + payload := kernelPayload(8<<20, 79) + response := kernelPayload(8<<20, 83) + result := make(chan error, 1) + go func() { result <- kernelTransfer(client, server, payload, false) }() + err := kernelTransfer(server, client, response, !splice) + if err != nil { + flowTest.Error("download:", err) + } + err = <-result + if err != nil { + flowTest.Error("upload:", err) + } + }) + } + }) + } + }) + } +} + +func prepareKernelNetif(t *testing.T, options *Options) { + t.Helper() + socketFD, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, sysprotoControl) + if err != nil { + t.Fatal(err) + } + transferred := false + defer func() { + if !transferred { + unix.Close(socketFD) + } + }() + var controlInfo unix.CtlInfo + copy(controlInfo.Name[:], utunControlName) + err = unix.IoctlCtlInfo(socketFD, &controlInfo) + if err != nil { + t.Fatal(err) + } + var index int + _, err = fmt.Sscanf(options.Name, "utun%d", &index) + if err != nil { + t.Fatal(err) + } + address := &unix.SockaddrCtl{ID: controlInfo.Id, Unit: uint32(index + 1)} + err = unix.Bind(socketFD, address) + if err != nil { + t.Fatal(err) + } + err = unix.SetsockoptInt(socketFD, sysprotoControl, utunOptionEnableNetif, 1) + if err != nil { + t.Fatal(err) + } + err = unix.Connect(socketFD, address) + if err != nil { + t.Fatal(err) + } + options.FileDescriptor = socketFD + transferred = true +} + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + ipv4 := options.Inet4Address[0] + ipv6 := options.Inet6Address[0] + commands := [][]string{ + {"/sbin/ifconfig", options.Name, "mtu", fmt.Sprint(options.MTU)}, + {"/sbin/ifconfig", options.Name, "inet", ipv4.Addr().String(), ipv4.Addr().String(), "netmask", "255.255.255.0", "up"}, + {"/sbin/ifconfig", options.Name, "inet6", ipv6.String(), "-dad"}, + {"/sbin/route", "-n", "add", "-net", ipv4.Masked().String(), "-interface", options.Name}, + } + for _, command := range commands { + output, err := exec.Command(command[0], command[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%v: %s: %v", command, output, err) + } + } +} + +func TestGoKernelDeviceBackpressure(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + for _, mtu := range []uint32{1500, 9000} { + for _, shutdown := range []bool{false, true} { + t.Run(fmt.Sprintf("mtu=%d/shutdown=%v", mtu, shutdown), func(test *testing.T) { + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: mtu, prepare: prepareKernelNetif}) + platform := fixture.stack.engines[0].platformIO.(*goDarwinIO) + if !platform.netif { + test.Fatal("utun netif was not enabled") + } + observed := make(chan struct{}, 1) + stop := make(chan struct{}) + defer close(stop) + go func() { + ticker := time.NewTicker(50 * time.Microsecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if platform.gateBlocked.Load() { + observed <- struct{}{} + return + } + case <-stop: + return + } + } + }() + payload := kernelPayload(32<<20, 223) + written := make(chan error, 8) + read := make(chan error, 8) + start := make(chan struct{}) + var clients []net.Conn + for index := range 8 { + var client, server net.Conn + splice := index%2 != 0 + if splice { + client, server, _, _ = fixture.splicePair(test, index%4 >= 2) + } else { + client, server = fixture.pair(test, index%4 >= 2) + } + clients = append(clients, client) + go func() { + <-start + var accepted atomic.Int64 + written <- kernelFlowWrite(server, payload, !splice, &accepted) + }() + go func() { + <-start + data, err := io.ReadAll(io.LimitReader(client, int64(len(payload)+1))) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("device backpressure payload mismatch") + } + read <- err + }() + } + close(start) + select { + case <-observed: + case <-time.After(3 * time.Second): + test.Fatal("real netif transmit gate never became blocked") + } + if shutdown { + started := time.Now() + err := fixture.stack.Close() + if err != nil || time.Since(started) > time.Second { + test.Fatalf("close during device backpressure: %v", err) + } + } + interrupted := 0 + limit := time.After(12 * time.Second) + if shutdown { + limit = time.After(time.Second) + } + for range 8 { + select { + case err := <-written: + if shutdown { + if E.IsTimeout(err) { + test.Error("device-blocked I/O timed out after close:", err) + } + if err != nil { + interrupted++ + } + } else if err != nil { + test.Error("transfer after device backpressure:", err) + } + case <-limit: + test.Fatal("device-blocked I/O did not complete") + } + } + if shutdown && interrupted == 0 { + test.Error("all writers finished before shutdown") + } + if shutdown { + for _, client := range clients { + client.Close() + } + } + for range 8 { + select { + case err := <-read: + if !shutdown && err != nil { + test.Error("read after device backpressure:", err) + } + case <-limit: + test.Fatal("device backpressure reader did not complete") + } + } + if platform.leakedClusters.total.Load() != 0 { + test.Fatal("utun reported ENOSPC while applying device backpressure") + } + }) + } + } +} + +func TestGoKernelMemoryPressureTransition(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + for _, netif := range []bool{false, true} { + for _, ipv6 := range []bool{false, true} { + for _, splice := range []bool{false, true} { + t.Run(fmt.Sprintf("netif=%v/ipv6=%v/splice=%v", netif, ipv6, splice), func(test *testing.T) { + test.Parallel() + var pressure, observed atomic.Uint32 + config := kernelStackConfig{mtu: 1500, socketBuffer: 128 << 10, upstreamSocketBuffer: 128 << 10} + config.pressure = func() MemoryPressure { + level := pressure.Load() + observed.Or(1 << level) + return MemoryPressure(level) + } + if netif { + config.prepare = prepareKernelNetif + } + fixture := newKernelStackFixture(test, config) + var client, server net.Conn + if splice { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + const phaseSize = 4 << 20 + upload := kernelPayload(3*phaseSize, 227) + download := kernelPayload(3*phaseSize, 229) + var uploadAccepted, downloadAccepted atomic.Int64 + written := make(chan error, 2) + go func() { written <- kernelFlowWrite(client, upload, false, &uploadAccepted) }() + go func() { written <- kernelFlowWrite(server, download, !splice, &downloadAccepted) }() + time.Sleep(100 * time.Millisecond) + if uploadAccepted.Load() == int64(len(upload)) || downloadAccepted.Load() == int64(len(download)) { + test.Fatal("pressure transition requires both writers to be active") + } + for phase, level := range []MemoryPressure{MemoryPressureWarning, MemoryPressureCritical, MemoryPressureNone} { + observed.And(^(1 << uint32(level))) + pressure.Store(uint32(level)) + read := make(chan error, 2) + for direction, conn := range []net.Conn{server, client} { + expected := upload + if direction == 1 { + expected = download + } + go func() { + data := make([]byte, phaseSize) + _, err := io.ReadFull(conn, data) + if err == nil && !bytes.Equal(data, expected[phase*phaseSize:(phase+1)*phaseSize]) { + err = E.New("pressure transition payload mismatch") + } + read <- err + }() + } + for range 2 { + err := <-read + if err != nil { + test.Fatal("active stream pressure transition:", err) + } + } + if observed.Load()&(1<= 1<= 7 { + break + } + time.Sleep(time.Millisecond) + } + if info == nil || info.Rcv_ooopack < 7 || server.sendUnacked.Load() != 1 { + test.Fatalf("missing out-of-order kernel queue: info=%+v unacked=%d", info, server.sendUnacked.Load()) + } + pruned := kernelTCPStatistic(test, "OfoPruned") + err = client.SetReadBuffer(512) + if err != nil { + test.Fatal(err) + } + runTC("qdisc", "del", "dev", fixture.options.Name, "ingress") + deadline = time.Now().Add(2 * time.Second) + for kernelTCPStatistic(test, "OfoPruned") == pruned && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if kernelTCPStatistic(test, "OfoPruned") == pruned { + test.Fatal("kernel did not discard SACKed data") + } + err = client.SetReadBuffer(64 << 10) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + data := make([]byte, len(payload)) + n, err := io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("SACK reneging: received=%d/%d pruned=%d: %v", n, len(payload), kernelTCPStatistic(test, "OfoPruned")-pruned, err) + } + err = server.CloseWrite() + if err != nil { + test.Fatal(err) + } + _, err = client.Read(data[:1]) + if err != io.EOF { + test.Fatalf("stream end after SACK reneging: %v", err) + } + }) + } +} + +func TestGoKernelZeroWindowHalfClose(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + config := kernelStackConfig{mtu: 9000, socketBuffer: 512} + config.dialer = net.Dialer{Control: func(_, _ string, rawConn syscall.RawConn) error { + var optionErr error + err := rawConn.Control(func(descriptor uintptr) { + optionErr = unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, 512) + if optionErr == nil { + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_WINDOW_CLAMP, 1024) + } + }) + return E.Errors(err, optionErr) + }} + fixture := newKernelStackFixture(test, config) + client, server := fixture.pair(test, ipv6) + payload := kernelPayload(int(server.sendPermit.Load()-1), 139) + _, err := server.Write(payload) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendUnacked.Load() == uint64(len(payload)+1) && server.sendPermit.Load() == server.sendUnacked.Load() { + break + } + time.Sleep(time.Millisecond) + } + if server.sendUnacked.Load() != uint64(len(payload)+1) || server.sendPermit.Load() != server.sendUnacked.Load() { + test.Fatalf("window did not close after acknowledging all data: length=%d unacked=%d permit=%d", len(payload), server.sendUnacked.Load(), server.sendPermit.Load()) + } + output, err := exec.Command("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "loss", "100%").CombinedOutput() + if err != nil { + test.Fatalf("drop FIN acknowledgment: %s: %v", output, err) + } + err = server.CloseWrite() + if err != nil { + test.Fatal(err) + } + time.Sleep(1600 * time.Millisecond) + err = client.SetReadBuffer(64 << 10) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("half-close data: %v", err) + } + time.Sleep(100 * time.Millisecond) + output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "root").CombinedOutput() + if err != nil { + test.Fatalf("restore acknowledgments: %s: %v", output, err) + } + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = client.Read(data[:1]) + if err != io.EOF { + test.Fatalf("zero-window FIN: %v", err) + } + err = client.CloseWrite() + if err != nil { + test.Fatal(err) + } + _, err = server.Read(data[:1]) + if err != io.EOF { + test.Fatalf("peer FIN: %v", err) + } + }) + } +} + +func kernelTCPStatistic(t *testing.T, name string) uint64 { + t.Helper() + data, err := os.ReadFile("/proc/net/netstat") + if err != nil { + t.Fatal(err) + } + lines := strings.Split(string(data), "\n") + for index := 0; index+1 < len(lines); index += 2 { + fields := strings.Fields(lines[index]) + if len(fields) == 0 || fields[0] != "TcpExt:" { + continue + } + values := strings.Fields(lines[index+1]) + for fieldIndex, field := range fields { + if field == name { + value, parseErr := strconv.ParseUint(values[fieldIndex], 10, 64) + if parseErr != nil { + t.Fatal(parseErr) + } + return value + } + } + } + t.Fatal("missing TCP statistic:", name) + return 0 +} + +func TestGoKernelWindowLimitedRecovery(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 9000}) + client, server := fixture.pair(test, ipv6) + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + output, err := exec.Command("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "delay", "50ms").CombinedOutput() + if err != nil { + test.Fatalf("delay ACKs: %s: %v", output, err) + } + payload := kernelPayload(8<<20, 223) + initial := 4 * int(server.effectiveMSS) + _, err = server.Write(payload[:initial]) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendPermit.Load() == server.sendUnacked.Load() && server.sentTail.Load() > server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + break + } + time.Sleep(time.Millisecond) + } + if server.sendPermit.Load() != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + test.Fatal("window did not shrink below outstanding data") + } + completed := make(chan kernelIOResult, 1) + server.SetWriteDeadline(time.Now().Add(2 * time.Second)) + go func() { + n, writeErr := server.Write(payload[initial:]) + completed <- kernelIOResult{n: n + initial, err: writeErr} + }() + deadline = time.Now().Add(time.Second) + for !server.writerParked.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !server.writerParked.Load() { + test.Fatal("write did not block at the closed window") + } + server.SetWriteDeadline(time.Now()) + var accepted int + select { + case result := <-completed: + if !errors.Is(result.err, os.ErrDeadlineExceeded) || result.n <= 0 || result.n >= len(payload) { + test.Fatalf("interrupt blocked write: %+v", result) + } + accepted = result.n + case <-time.After(time.Second): + test.Fatal("write did not react to deadline") + } + output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "root").CombinedOutput() + if err != nil { + test.Fatalf("restore ACKs: %s: %v", output, err) + } + err = client.SetReadBuffer(4 << 20) + if err != nil { + test.Fatal(err) + } + rawConn, err := client.SyscallConn() + if err != nil { + test.Fatal(err) + } + var optionErr error + err = rawConn.Control(func(descriptor uintptr) { + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_WINDOW_CLAMP, 6144) + }) + if err != nil || optionErr != nil { + test.Fatalf("limit receive window: %v %v", err, optionErr) + } + server.SetWriteDeadline(time.Now().Add(4 * time.Second)) + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + go func() { + n, writeErr := server.Write(payload[accepted:]) + if writeErr == nil { + writeErr = server.CloseWrite() + } + completed <- kernelIOResult{n: n + accepted, err: writeErr} + }() + data, err := io.ReadAll(client) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("window-limited recovery: received=%d/%d accepted=%d: %v", len(data), len(payload), accepted, err) + } + select { + case result := <-completed: + if result.err != nil || result.n != len(payload) { + test.Fatalf("resume blocked write: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("write remained blocked after recovery") + } + }) + } +} + +func TestGoKernelZeroWindow(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, server := fixture.pair(test, ipv6) + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + runTC := func(args ...string) { + test.Helper() + output, commandErr := exec.Command("tc", args...).CombinedOutput() + if commandErr != nil { + test.Fatalf("tc %v: %s: %v", args, output, commandErr) + } + } + runTC("qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") + payload := kernelPayload(32768, 107) + _, err = server.Write(payload) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendPermit.Load() == server.sendUnacked.Load() && server.sentTail.Load() == server.bufferedTail.Load() { + break + } + time.Sleep(time.Millisecond) + } + unacked := server.sendUnacked.Load() + if server.sendPermit.Load() != unacked || server.sentTail.Load() != uint64(len(payload)+1) || unacked <= 1 || unacked >= server.sentTail.Load() { + test.Fatalf("window did not close with only outstanding data: unacked=%d sent=%d buffered=%d permit=%d", unacked, server.sentTail.Load(), server.bufferedTail.Load(), server.sendPermit.Load()) + } + runTC("qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "loss", "100%") + err = client.SetReadBuffer(4 << 20) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(client, data[:unacked-1]) + if err != nil { + test.Fatal(err) + } + time.Sleep(100 * time.Millisecond) + runTC("qdisc", "del", "dev", fixture.options.Name, "root") + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = io.ReadFull(client, data[unacked-1:]) + if err != nil { + test.Fatal("recover lost window update:", err) + } + if !bytes.Equal(data, payload) { + test.Fatal("zero-window recovery corrupted the stream") + } + }) + } +} diff --git a/stack_go_integration_lifecycle_linux_test.go b/stack_go_integration_lifecycle_linux_test.go new file mode 100644 index 00000000..682bd0e2 --- /dev/null +++ b/stack_go_integration_lifecycle_linux_test.go @@ -0,0 +1,303 @@ +//go:build linux && !android + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os/exec" + "regexp" + "runtime" + "testing" + "time" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelHandshakeLoss(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, phase := range []string{"syn_ack", "final_ack"} { + for _, cancelHandshake := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/phase=%s/cancel=%v", ipv6, phase, cancelHandshake), func(test *testing.T) { + test.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + observed := make(chan *GoConn, 1) + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500, ctx: ctx, handshake: func(conn *GoConn) error { + observed <- conn + return conn.HandshakeSuccess() + }}) + direction, parent, flags := "ingress", "ffff:", "0x12" + if phase == "final_ack" { + direction, parent, flags = "root", "1:", "0x10" + } + protocol, offset := "ip", "33" + if ipv6 { + protocol, offset = "ipv6", "53" + } + command := []string{"qdisc", "add", "dev", fixture.options.Name, direction} + if direction == "root" { + command = append(command, "handle", "1:", "prio") + } + output, err := exec.Command("tc", command...).CombinedOutput() + if err != nil { + test.Fatalf("handshake qdisc: %s: %v", output, err) + } + test.Cleanup(func() { exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, direction).Run() }) + output, err = exec.Command("tc", "filter", "add", "dev", fixture.options.Name, "parent", parent, "protocol", protocol, + "u32", "match", "u8", flags, "0x12", "at", offset, "action", "drop").CombinedOutput() + if err != nil { + test.Fatalf("handshake filter: %s: %v", output, err) + } + port := uint16(20000 + fixture.port.Add(1)) + accepted := make(chan kernelAccept, 1) + fixture.access.Lock() + fixture.tcp[port] = accepted + fixture.access.Unlock() + type dialResult struct { + conn net.Conn + err error + } + dialed := make(chan dialResult, 1) + go func() { + conn, dialErr := fixture.dialer.DialContext(ctx, "tcp", fixture.address(ipv6, port)) + dialed <- dialResult{conn: conn, err: dialErr} + }() + select { + case conn := <-observed: + test.Cleanup(func() { conn.Close() }) + case <-time.After(time.Second): + test.Fatal("SYN did not reach the application") + } + select { + case result := <-accepted: + test.Fatalf("handshake completed while packets were dropped: %v", result.err) + case <-time.After(250 * time.Millisecond): + } + output, err = exec.Command("tc", "-s", "filter", "show", "dev", fixture.options.Name, "parent", parent).CombinedOutput() + if err != nil || !regexp.MustCompile(`dropped [1-9][0-9]*`).Match(output) { + test.Fatalf("no handshake packet was dropped: %s: %v", output, err) + } + if cancelHandshake { + cancel() + } else { + output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, direction).CombinedOutput() + if err != nil { + test.Fatalf("restore handshake traffic: %s: %v", output, err) + } + } + var result kernelAccept + select { + case result = <-accepted: + if cancelHandshake { + if !errors.Is(result.err, context.Canceled) { + test.Fatalf("cancel pending handshake: %v", result.err) + } + } else if result.err != nil { + test.Fatal("recover handshake:", result.err) + } + case <-time.After(2 * time.Second): + test.Fatal("handshake did not react to recovery/cancellation") + } + select { + case peer := <-dialed: + if peer.conn != nil { + defer peer.conn.Close() + } + if !cancelHandshake { + if peer.err != nil { + test.Fatal(peer.err) + } + peer.conn.SetDeadline(time.Now().Add(3 * time.Second)) + err = kernelTransfer(result.conn, peer.conn, kernelPayload(32769, 233), true) + if err != nil { + test.Fatal("stream after handshake loss:", err) + } + } + case <-time.After(time.Second): + test.Fatal("native dial remained pending") + } + }) + } + } + } +} + +func TestGoKernelLinkInterruption(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + t.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500}) + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + runTC := func(args ...string) []byte { + test.Helper() + output, err := exec.Command("tc", args...).CombinedOutput() + if err != nil { + test.Fatalf("tc %v: %s: %v", args, output, err) + } + return output + } + for phase := range 3 { + runTC("qdisc", "add", "dev", fixture.options.Name, "root", "netem", "loss", "100%") + runTC("qdisc", "add", "dev", fixture.options.Name, "ingress") + runTC("filter", "add", "dev", fixture.options.Name, "parent", "ffff:", "protocol", "all", + "u32", "match", "u32", "0", "0", "action", "drop") + written := make(chan error, 2) + read := make(chan error, 2) + for direction := range 2 { + sender, receiver := client, server + if direction == 1 { + sender, receiver = server, client + } + payload := kernelPayload(256<<10|123, uint32(239+phase*7+direction)) + go func() { + var err error + if direction == 1 && mode == "buffer" { + buffer := buf.NewSize(len(payload) + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload)) + err = sender.(N.ExtendedWriter).WriteBuffer(buffer) + } else { + _, err = sender.Write(payload) + } + if err == nil && phase == 2 { + err = N.CloseWrite(sender) + } + written <- err + }() + go func() { + data := make([]byte, len(payload)) + _, err := io.ReadFull(receiver, data) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("link recovery payload mismatch") + } + if err == nil && phase == 2 { + _, err = receiver.Read(data[:1]) + if err == io.EOF { + err = nil + } else { + err = E.New("link recovery stream end: ", err) + } + } + read <- err + }() + } + select { + case err := <-read: + test.Fatalf("phase %d: stream completed during link outage: %v", phase, err) + case <-time.After(250 * time.Millisecond): + } + dropPattern := regexp.MustCompile(`dropped [1-9][0-9]*`) + deadline := time.Now().Add(2 * time.Second) + for { + bothDropped := true + for _, output := range [][]byte{ + runTC("-s", "qdisc", "show", "dev", fixture.options.Name), + runTC("-s", "filter", "show", "dev", fixture.options.Name, "parent", "ffff:"), + } { + if dropPattern.Match(output) { + continue + } + if time.Now().After(deadline) { + test.Fatalf("phase %d: link outage did not drop traffic: %s", phase, output) + } + bothDropped = false + } + if bothDropped { + break + } + select { + case err := <-read: + test.Fatalf("phase %d: stream completed during link outage: %v", phase, err) + case <-time.After(20 * time.Millisecond): + } + } + runTC("qdisc", "del", "dev", fixture.options.Name, "root") + runTC("qdisc", "del", "dev", fixture.options.Name, "ingress") + for range 2 { + err := <-written + if err != nil { + test.Fatal("write through link outage:", err) + } + err = <-read + if err != nil { + test.Fatal("read after link recovery:", err) + } + } + } + }) + } + } +} + +func configureKernelLoss(t *testing.T, options Options) { + t.Helper() + ifbName := "ifb-" + options.Name + commands := [][]string{ + {"ip", "link", "add", ifbName, "type", "ifb"}, + {"ip", "link", "set", ifbName, "up"}, + {"tc", "qdisc", "add", "dev", options.Name, "root", "netem", "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%", "limit", "4096"}, + {"tc", "qdisc", "add", "dev", options.Name, "ingress"}, + {"tc", "filter", "add", "dev", options.Name, "parent", "ffff:", "protocol", "all", "u32", "match", "u32", "0", "0", "action", "mirred", "egress", "redirect", "dev", ifbName}, + {"tc", "qdisc", "add", "dev", ifbName, "root", "netem", "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%", "limit", "4096"}, + } + for index, command := range commands { + output, err := exec.Command(command[0], command[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%v: %s: %v", command, output, err) + } + if index == 0 { + t.Cleanup(func() { exec.Command("ip", "link", "delete", ifbName).Run() }) + } + } +} + +func TestGoKernelBidirectionalLoss(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + for _, gso := range []bool{false, true} { + for _, multiQueue := range []bool{false, true} { + t.Run(fmt.Sprintf("gso=%v/mq=%v", gso, multiQueue), func(offloadTest *testing.T) { + fixture := newKernelStackFixture(offloadTest, kernelStackConfig{mtu: 1500, gso: gso, multiQueue: multiQueue, configure: configureKernelLoss}) + for _, ipv6 := range []bool{false, true} { + for _, splice := range []bool{false, true} { + offloadTest.Run(fmt.Sprintf("ipv6=%v/splice=%v", ipv6, splice), func(flowTest *testing.T) { + flowTest.Parallel() + var client, server net.Conn + if splice { + client, server, _, _ = fixture.splicePair(flowTest, ipv6) + } else { + client, server = fixture.pair(flowTest, ipv6) + } + payload := kernelPayload(1<<20, 43) + response := kernelPayload(1<<20, 47) + result := make(chan error, 1) + go func() { result <- kernelTransfer(client, server, payload, false) }() + err := kernelTransfer(server, client, response, !splice) + if err != nil { + flowTest.Error("download:", err) + } + err = <-result + if err != nil { + flowTest.Error("upload:", err) + } + }) + } + } + }) + } + } +} diff --git a/stack_go_integration_lifecycle_test.go b/stack_go_integration_lifecycle_test.go new file mode 100644 index 00000000..7ef2708b --- /dev/null +++ b/stack_go_integration_lifecycle_test.go @@ -0,0 +1,436 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "runtime" + "sync" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelHandshake(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, action := range []string{"accept", "reject", "close"} { + t.Run(fmt.Sprintf("ipv6=%v/action=%s", ipv6, action), func(test *testing.T) { + test.Parallel() + observed := make(chan *GoConn, 1) + var observe sync.Once + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500, dialer: net.Dialer{Timeout: 5 * time.Second}, handshake: func(conn *GoConn) error { + observe.Do(func() { observed <- conn }) + test.Cleanup(func() { conn.Close() }) + <-release + switch action { + case "reject": + return conn.HandshakeFailure(kernelConnectionRefused) + case "close": + return conn.Close() + default: + return conn.HandshakeSuccess() + } + }}) + test.Cleanup(unblock) + port := uint16(20000 + fixture.port.Add(1)) + fixture.access.Lock() + fixture.tcp[port] = make(chan kernelAccept, 8) + fixture.access.Unlock() + type dialResult struct { + conn net.Conn + err error + } + dialed := make(chan dialResult, 1) + go func() { + conn, err := fixture.dialer.Dial("tcp", fixture.address(ipv6, port)) + dialed <- dialResult{conn: conn, err: err} + }() + var server *GoConn + select { + case server = <-observed: + case <-time.After(time.Second): + test.Fatal("connection did not reach the application") + } + select { + case result := <-dialed: + if result.conn != nil { + result.conn.Close() + } + test.Fatalf("dial completed before application decision: %v", result.err) + case <-time.After(80 * time.Millisecond): + } + unblock() + select { + case result := <-dialed: + if action != "accept" { + if result.conn != nil { + result.conn.Close() + } + if !errors.Is(result.err, kernelConnectionRefused) { + test.Fatalf("application %s: %v", action, result.err) + } + return + } + if result.err != nil { + test.Fatal(result.err) + } + defer result.conn.Close() + result.conn.SetDeadline(time.Now().Add(3 * time.Second)) + err := kernelTransfer(server, result.conn, kernelPayload(32769, 157), true) + if err != nil { + test.Fatal("accepted stream:", err) + } + case <-time.After(6 * time.Second): + test.Fatal("application decision did not complete the dial") + } + }) + } + } +} + +func TestGoKernelHalfClose(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + for _, splice := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/splice=%v", ipv6, splice), func(test *testing.T) { + test.Parallel() + var client, server net.Conn + if splice { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + err := kernelTransfer(client, server, kernelPayload(256<<10, 163), false) + if err != nil { + test.Fatal("request and FIN:", err) + } + err = kernelTransfer(server, client, kernelPayload(2<<20, 167), false) + if err != nil { + test.Fatal("response after peer FIN:", err) + } + }) + } + t.Run(fmt.Sprintf("ipv6=%v/close_read", ipv6), func(test *testing.T) { + test.Parallel() + client, server := fixture.pair(test, ipv6) + completed := make(chan kernelIOResult, 1) + go func() { + n, err := server.Read(make([]byte, 1)) + completed <- kernelIOResult{n: n, err: err} + }() + select { + case result := <-completed: + test.Fatalf("read did not wait: %+v", result) + case <-time.After(40 * time.Millisecond): + } + err := server.CloseRead() + if err != nil { + test.Fatal(err) + } + select { + case result := <-completed: + if result.n != 0 || result.err != io.EOF { + test.Fatalf("CloseRead: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("CloseRead did not wake reader") + } + uploaded := make(chan error, 1) + go func() { + _, writeErr := client.Write(kernelPayload(1<<20, 173)) + if writeErr == nil { + writeErr = client.CloseWrite() + } + uploaded <- writeErr + }() + err = kernelTransfer(server, client, kernelPayload(256<<10, 179), true) + if err != nil { + test.Fatal("write after CloseRead:", err) + } + select { + case err = <-uploaded: + if err != nil { + test.Fatal("peer upload after CloseRead:", err) + } + case <-time.After(time.Second): + test.Fatal("discarded input blocked the peer") + } + }) + t.Run(fmt.Sprintf("ipv6=%v/upstream_reset", ipv6), func(test *testing.T) { + test.Parallel() + client, upstream, _, _ := fixture.splicePair(test, ipv6) + marker := kernelPayload(1024, 181) + _, err := upstream.Write(marker) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(marker)) + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, marker) { + test.Fatal("upstream data before reset:", err) + } + upstream.SetLinger(0) + upstream.Close() + client.SetReadDeadline(time.Now().Add(time.Second)) + _, err = client.Read(data[:1]) + if !errors.Is(err, kernelConnectionReset) { + test.Fatalf("upstream reset: %v", err) + } + }) + } +} + +func TestGoKernelClose(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + t.Run("reset", func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + client, conn := fixture.pair(scenarioTest, ipv6) + client.SetLinger(0) + client.Close() + _, err := conn.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + scenarioTest.Errorf("reset ipv6=%v: %v", ipv6, err) + } + } + }) + t.Run("drain_after_fin", func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + client, conn := fixture.pair(scenarioTest, ipv6) + payload := kernelPayload(32769, 41) + _, err := client.Write(payload) + if err != nil { + scenarioTest.Fatal(err) + } + client.CloseWrite() + conn.CloseWrite() + _, err = io.ReadAll(client) + if err != nil { + scenarioTest.Fatal(err) + } + data, err := io.ReadAll(conn) + if err != nil || !bytes.Equal(data, payload) { + scenarioTest.Fatalf("drain ipv6=%v bytes=%d: %v", ipv6, len(data), err) + } + } + }) + for _, splice := range []bool{false, true} { + t.Run(fmt.Sprintf("busy_shutdown/splice=%v", splice), func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500, multiQueue: runtime.GOOS == "linux"}) + payload := kernelPayload(8<<20, 33) + completed := make(chan error, 32) + for index := range 16 { + var client, conn net.Conn + if splice { + client, conn, _, _ = fixture.splicePair(scenarioTest, index%2 != 0) + } else { + client, conn = fixture.pair(scenarioTest, index%2 != 0) + } + client.SetDeadline(time.Time{}) + client.(*net.TCPConn).SetReadBuffer(4096) + conn.SetDeadline(time.Time{}) + go func() { conn.Write(payload); completed <- nil }() + go func() { + _, err := conn.Read(make([]byte, 1)) + if err == nil { + completed <- E.New("pending read completed without close error") + return + } + completed <- nil + }() + } + time.Sleep(30 * time.Millisecond) + started := time.Now() + err := fixture.stack.Close() + if err != nil { + scenarioTest.Fatal(err) + } + if time.Since(started) > time.Second { + scenarioTest.Error("stack close exceeded 1 second") + } + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for range 32 { + select { + case operationErr := <-completed: + if operationErr != nil { + scenarioTest.Error(operationErr) + } + case <-deadline.C: + scenarioTest.Fatal("pending I/O did not unblock after stack shutdown") + } + } + for _, engine := range fixture.stack.engines { + inUse := engine.slabPool.inUse.Load() + if inUse != 0 { + scenarioTest.Errorf("shutdown retained %d allocated slabs", inUse) + } + } + }) + } +} + +type kernelIOResult struct { + n int + err error +} + +func TestGoKernelDeadline(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000}) + for _, ipv6 := range []bool{false, true} { + for _, operation := range []string{"read", "wait_read", "write"} { + for _, change := range []string{"advance", "extend", "clear", "close"} { + t.Run(fmt.Sprintf("ipv6=%v/operation=%s/change=%s", ipv6, operation, change), func(test *testing.T) { + test.Parallel() + client, server := fixture.pair(test, ipv6) + writing := operation == "write" + payload := []byte{0x71} + setDeadline := server.SetReadDeadline + if writing { + payload = kernelPayload(8<<20, 151) + setDeadline = server.SetWriteDeadline + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + } + var waiter N.ReadWaiter + if operation == "wait_read" { + var created bool + waiter, created = bufio.CreateReadWaiter(server) + if !created { + test.Fatal("missing read waiter") + } + waiter.InitializeReadWaiter(N.ReadWaitOptions{FrontHeadroom: 91, RearHeadroom: 73}) + } + read := func() kernelIOResult { + if waiter != nil { + buffer, err := waiter.WaitReadBuffer() + result := kernelIOResult{err: err} + if buffer != nil { + result.n = buffer.Len() + if !bytes.Equal(buffer.Bytes(), payload) { + result.err = E.New("read waiter payload mismatch") + } + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + return result + } + data := make([]byte, len(payload)) + n, err := io.ReadFull(server, data) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("read payload mismatch") + } + return kernelIOResult{n: n, err: err} + } + original := time.Now().Add(300 * time.Millisecond) + if change == "advance" || change == "close" { + original = time.Now().Add(4 * time.Second) + } + setDeadline(original) + completed := make(chan kernelIOResult, 1) + go func() { + if writing { + n, err := server.Write(payload) + completed <- kernelIOResult{n: n, err: err} + } else { + completed <- read() + } + }() + select { + case result := <-completed: + test.Fatalf("I/O did not wait: %+v", result) + case <-time.After(40 * time.Millisecond): + } + switch change { + case "advance": + setDeadline(time.Now()) + case "extend": + setDeadline(time.Now().Add(4 * time.Second)) + case "clear": + setDeadline(time.Time{}) + case "close": + server.Close() + } + accepted := 0 + if change == "advance" || change == "close" { + select { + case result := <-completed: + accepted = result.n + if accepted < 0 || accepted >= len(payload) { + test.Fatalf("blocked I/O accepted %d/%d bytes", accepted, len(payload)) + } + if change == "advance" && !errors.Is(result.err, os.ErrDeadlineExceeded) { + test.Fatalf("updated deadline: %v", result.err) + } + if change == "close" && (!E.IsClosed(result.err) || E.IsTimeout(result.err)) { + test.Fatalf("close pending I/O: %v", result.err) + } + case <-time.After(time.Second): + test.Fatal("pending I/O did not react to deadline/close") + } + if change == "close" { + return + } + setDeadline(time.Now().Add(4 * time.Second)) + go func() { + if writing { + n, err := server.Write(payload[accepted:]) + completed <- kernelIOResult{n: n + accepted, err: err} + } else { + completed <- read() + } + }() + } else { + select { + case result := <-completed: + test.Fatalf("old deadline still affected I/O: %+v", result) + case <-time.After(time.Until(original.Add(40 * time.Millisecond))): + } + } + if writing { + client.SetReadBuffer(4 << 20) + data := make([]byte, len(payload)) + n, err := io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + select { + case result := <-completed: + test.Logf("writer after deadline change: accepted=%d: %v", result.n, result.err) + default: + } + test.Fatalf("write after deadline change: received=%d/%d accepted_before_change=%d: %v", n, len(payload), accepted, err) + } + } else { + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + } + select { + case result := <-completed: + if result.err != nil || result.n != len(payload) { + test.Fatalf("completed I/O: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("I/O remained blocked after peer resumed") + } + }) + } + } + } +} diff --git a/stack_go_integration_linux_test.go b/stack_go_integration_linux_test.go new file mode 100644 index 00000000..b2626aaf --- /dev/null +++ b/stack_go_integration_linux_test.go @@ -0,0 +1,57 @@ +//go:build linux && !android + +package tun + +import ( + "testing" + "time" + + "github.com/sagernet/netlink" + + "golang.org/x/sys/unix" +) + +const ( + kernelConnectionRefused = unix.ECONNREFUSED + kernelConnectionReset = unix.ECONNRESET +) + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + link, err := netlink.LinkByName(options.Name) + if err != nil { + t.Fatal(err) + } + for _, prefix := range append(options.Inet4Address, options.Inet6Address...) { + address, parseErr := netlink.ParseAddr(prefix.String()) + if parseErr != nil { + t.Fatal(parseErr) + } + address.Flags = unix.IFA_F_NODAD + err = netlink.AddrAdd(link, address) + if err != nil { + t.Fatal(err) + } + } + err = netlink.LinkSetUp(link) + if err != nil { + t.Fatal(err) + } + for _, prefix := range options.Inet6Address { + deadline := time.Now().Add(time.Second) + for { + routes, routeErr := netlink.RouteGet(prefix.Addr().AsSlice()) + if routeErr != nil { + t.Fatal(routeErr) + } + if len(routes) > 0 && routes[0].Type == unix.RTN_LOCAL { + break + } + if time.Now().After(deadline) { + t.Fatalf("IPv6 local route was not installed: %s", prefix.Addr()) + } + t.Log("waiting for IPv6 local route:", prefix.Addr()) + time.Sleep(time.Millisecond) + } + } +} diff --git a/stack_go_integration_packet_flow_test.go b/stack_go_integration_packet_flow_test.go new file mode 100644 index 00000000..1cfe5154 --- /dev/null +++ b/stack_go_integration_packet_flow_test.go @@ -0,0 +1,347 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "net/netip" + "runtime" + "sync" + "testing" + "time" + + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelPacketFragments(t *testing.T) { + configs := []kernelStackConfig{{mtu: 1500}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("gso=%v/mq=%v", config.gso, config.multiQueue), func(configTest *testing.T) { + fixture := newKernelStackFixture(configTest, config) + for _, ipv6 := range []bool{false, true} { + configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + const sessions = 4 + for phase := range 4 { + start := make(chan struct{}) + var workers sync.WaitGroup + for index := range sessions { + client, conn, destination := fixture.packetPair(test, ipv6) + payload := kernelPayload(16385, uint32(phase*sessions+index+1)) + workers.Go(func() { + defer client.Close() + defer conn.Close() + select { + case <-start: + case <-conn.doneChan: + return + } + client.SetReadDeadline(time.Now().Add(time.Second)) + options := N.NewReadWaitOptions(nil, conn) + buffer := options.NewBufferSize(len(payload)) + _, _ = buffer.Write(payload) + options.PostReturn(buffer) + err := conn.WritePacket(buffer, destination) + if err != nil { + test.Error(err) + return + } + storage := make([]byte, 65535) + n, err := client.Read(storage) + if err != nil || !bytes.Equal(storage[:n], payload) { + test.Errorf("phase=%d session=%d: reassembled %d/%d bytes: %v", phase, index, n, len(payload), err) + } + }) + } + close(start) + workers.Wait() + if test.Failed() { + return + } + } + }) + } + }) + } +} + +func TestGoKernelPacketRead(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + for _, batch := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/batch=%v", ipv6, batch), func(test *testing.T) { + test.Parallel() + client, conn, destination := fixture.packetPair(test, ipv6) + waiter, created := bufio.CreatePacketReadWaiter(conn) + if !created { + test.Fatal("missing packet read waiter") + } + batchWaiter, batchCreated := bufio.CreatePacketBatchReadWaiter(conn) + if !batchCreated { + test.Fatal("missing packet batch read waiter") + } + for phase, options := range []N.ReadWaitOptions{ + {BatchSize: 4}, + {FrontHeadroom: 91, RearHeadroom: 73, BatchSize: 4}, + {FrontHeadroom: 17, RearHeadroom: 127, BatchSize: 3}, + } { + expected := make(map[string]bool) + sizes := []int{0, 1, 1452, 1473, 8193, 1024, 13, 64} + for index, size := range sizes { + payload := kernelPayload(size, uint32(193+phase*17+index)) + expected[string(payload)] = true + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + } + time.Sleep(40 * time.Millisecond) + waiter.InitializeReadWaiter(options) + largestBatch := 0 + for received := 0; received < len(sizes); { + var buffers []*buf.Buffer + var destinations []M.Socksaddr + var err error + if batch { + buffers, destinations, err = batchWaiter.WaitReadPackets() + } else { + var buffer *buf.Buffer + var packetDestination M.Socksaddr + buffer, packetDestination, err = waiter.WaitReadPacket() + if buffer != nil { + buffers = []*buf.Buffer{buffer} + destinations = []M.Socksaddr{packetDestination} + } + } + if err != nil { + buf.ReleaseMulti(buffers) + test.Fatal(err) + } + largestBatch = max(largestBatch, len(buffers)) + if len(buffers) == 0 || len(buffers) != len(destinations) || len(buffers) > options.BatchSize { + buf.ReleaseMulti(buffers) + test.Fatal("incorrect packet batch size") + } + for index, buffer := range buffers { + if !expected[string(buffer.Bytes())] || destinations[index] != destination { + buf.ReleaseMulti(buffers[index:]) + test.Fatalf("phase %d: unexpected/duplicate packet or destination", phase) + } + delete(expected, string(buffer.Bytes())) + buffer.ExtendHeader(options.FrontHeadroom) + buffer.Extend(options.RearHeadroom) + buffer.Release() + } + received += len(buffers) + } + if batch && largestBatch <= 1 { + test.Fatal("queued datagrams were never read as a batch") + } + } + options := N.NewReadWaitOptions(nil, conn) + writer := bufio.NewPacketBatchWriter(conn) + var buffers []*buf.Buffer + var destinations []M.Socksaddr + expected := make(map[string]bool) + for index, size := range []int{0, 1, 1453, 8193} { + payload := kernelPayload(size, uint32(211+index)) + expected[string(payload)] = true + buffer := options.NewBufferSize(len(payload)) + _, _ = buffer.Write(payload) + options.PostReturn(buffer) + buffers = append(buffers, buffer) + destinations = append(destinations, destination) + } + err := writer.WritePacketBatch(buffers, destinations) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 65535) + for len(expected) > 0 { + n, readErr := client.Read(storage) + if readErr != nil || !expected[string(storage[:n])] { + var missing []int + for packet := range expected { + missing = append(missing, len(packet)) + } + test.Fatalf("batch write data: length=%d missing=%v: %v", n, missing, readErr) + } + delete(expected, string(storage[:n])) + } + }) + } + } +} + +func TestGoKernelPacketMapping(t *testing.T) { + queues := []bool{false} + if runtime.GOOS == "linux" { + queues = append(queues, true) + } + for _, multiQueue := range queues { + for _, mapping := range []NATMapping{NATMappingEndpointIndependent, NATMappingAddressDependent, NATMappingAddressAndPortDependent} { + for _, filtering := range []NATFiltering{NATFilteringEndpointIndependent, NATFilteringAddressDependent, NATFilteringAddressAndPortDependent} { + t.Run(fmt.Sprintf("mq=%v/mapping=%d/filtering=%d", multiQueue, mapping, filtering), func(configTest *testing.T) { + fixture := newKernelStackFixture(configTest, kernelStackConfig{mtu: 1500, multiQueue: multiQueue, udpMapping: mapping, udpFiltering: filtering}) + for _, ipv6 := range []bool{false, true} { + configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + address := fixture.options.Inet4Address[0].Addr() + if ipv6 { + address = fixture.options.Inet6Address[0].Addr() + } + client, err := net.ListenUDP("udp", &net.UDPAddr{IP: address.AsSlice()}) + if err != nil { + test.Fatal(err) + } + test.Cleanup(func() { client.Close() }) + client.SetDeadline(time.Now().Add(3 * time.Second)) + port := uint16(20000 + 8*fixture.port.Add(1)) + peers := []M.Socksaddr{ + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port+1)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next().Next(), port+2)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port+3)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next().Next().Next(), port+4)), + } + accepted := make(chan N.PacketConn, 4) + fixture.access.Lock() + for _, peer := range peers { + fixture.udp[peer.Port] = accepted + } + fixture.access.Unlock() + var first N.PacketConn + for index, peer := range peers[:3] { + payload := []byte{byte(index + 1)} + _, err = client.WriteToUDPAddrPort(payload, peer.AddrPort()) + if err != nil { + test.Fatal(err) + } + conn := first + if index == 0 || mapping == NATMappingAddressAndPortDependent || (mapping == NATMappingAddressDependent && index == 2) { + select { + case conn = <-accepted: + test.Cleanup(func() { conn.Close() }) + case <-time.After(time.Second): + test.Fatal("missing independent UDP session") + } + } + if index == 0 { + first = conn + } + conn.SetReadDeadline(time.Now().Add(time.Second)) + buffer := buf.NewPacket() + destination, readErr := conn.ReadPacket(buffer) + matches := bytes.Equal(buffer.Bytes(), payload) + buffer.Release() + if readErr != nil || !matches || destination != peer { + test.Fatalf("mapping peer %d: destination=%v: %v", index, destination, readErr) + } + } + for index, peer := range peers { + allowed := filtering == NATFilteringEndpointIndependent || index == 0 + if filtering == NATFilteringAddressDependent && (index == 1 || index == 3) { + allowed = true + } + if index == 1 && mapping != NATMappingAddressAndPortDependent { + allowed = true + } + if index == 2 && mapping == NATMappingEndpointIndependent { + allowed = true + } + payload := []byte{byte(index + 31)} + options := N.NewReadWaitOptions(nil, first) + buffer := options.NewBufferSize(len(payload)) + _, _ = buffer.Write(payload) + options.PostReturn(buffer) + err = first.WritePacket(buffer, peer) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(75 * time.Millisecond)) + if allowed { + client.SetReadDeadline(time.Now().Add(time.Second)) + } + data := make([]byte, 16) + n, source, readErr := client.ReadFromUDPAddrPort(data) + if allowed { + if readErr != nil || !bytes.Equal(data[:n], payload) || source != peer.AddrPort() { + test.Fatalf("allowed return peer %d: source=%v: %v", index, source, readErr) + } + } else { + var timeout net.Error + if !errors.As(readErr, &timeout) || !timeout.Timeout() { + test.Fatalf("uncontacted return peer %d was not filtered: source=%v: %v", index, source, readErr) + } + } + } + }) + } + }) + } + } + } +} + +func TestGoKernelPacketLifetime(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + _, idle, _ := fixture.packetPair(test, ipv6) + client, active, _ := fixture.packetPair(test, ipv6) + for _, conn := range []*UDPNatConn{idle, active} { + if !conn.SetTimeout(180 * time.Millisecond) { + test.Fatal("cannot set UDP timeout") + } + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + } + idleClosed := make(chan error, 1) + go func() { + buffer, _, err := idle.WaitReadPacket() + if buffer != nil { + buffer.Release() + } + idleClosed <- err + }() + buffer := buf.NewPacket() + defer buffer.Release() + for index := range 8 { + time.Sleep(50 * time.Millisecond) + payload := []byte{byte(index + 1)} + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + buffer.Reset() + _, err = active.ReadPacket(buffer) + if err != nil || !bytes.Equal(buffer.Bytes(), payload) { + test.Fatalf("active UDP session expired: %v", err) + } + } + select { + case err := <-idleClosed: + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("independent idle session: %v", err) + } + case <-time.After(300 * time.Millisecond): + test.Fatal("traffic on another session kept idle session alive") + } + active.SetReadDeadline(time.Now().Add(600 * time.Millisecond)) + _, err := active.ReadPacket(buffer) + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("UDP session did not expire after traffic stopped: %v", err) + } + }) + } +} diff --git a/stack_go_integration_packet_offload_linux_test.go b/stack_go_integration_packet_offload_linux_test.go new file mode 100644 index 00000000..c742db78 --- /dev/null +++ b/stack_go_integration_packet_offload_linux_test.go @@ -0,0 +1,182 @@ +//go:build linux && !android + +package tun + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "net/netip" + "testing" + "time" + "unsafe" + + "github.com/sagernet/sing/common/buf" + M "github.com/sagernet/sing/common/metadata" + "github.com/sagernet/sing/protocol/socks" + + "golang.org/x/sys/unix" +) + +func TestGoKernelPacketOffload(t *testing.T) { + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator} + buf.DefaultAllocator = allocator + t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) + for _, multiQueue := range []bool{false, true} { + t.Run(fmt.Sprintf("mq=%v", multiQueue), func(queueTest *testing.T) { + fixture := newKernelStackFixture(queueTest, kernelStackConfig{mtu: 1500, gso: true, multiQueue: multiQueue}) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"direct", "socks", "socks-fqdn"} { + queueTest.Run(fmt.Sprintf("ipv6=%v/%s", ipv6, mode), func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + if ipv6 { + network = "udp6" + address = net.IPv6loopback + } + peer, err := net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + peer.SetReadBuffer(1 << 20) + upstream, err := net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + scenarioTest.Fatal(err) + } + defer upstream.Close() + upstream.SetReadBuffer(1 << 20) + rawUpstream, err := upstream.SyscallConn() + if err != nil { + scenarioTest.Fatal(err) + } + var ( + monitorDescriptor int + monitorError error + ) + err = rawUpstream.Control(func(descriptor uintptr) { + monitorDescriptor, monitorError = unix.FcntlInt(descriptor, unix.F_DUPFD_CLOEXEC, 0) + }) + if err != nil || monitorError != nil { + scenarioTest.Fatalf("duplicate socket for GRO inspection: %v, %v", err, monitorError) + } + defer unix.Close(monitorDescriptor) + for _, socket := range []*net.UDPConn{client, peer} { + rawConn, rawError := socket.SyscallConn() + if rawError != nil { + scenarioTest.Fatal(rawError) + } + controlError := rawConn.Control(func(descriptor uintptr) { + err = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_UDP, unix.UDP_GRO, 1) + }) + if controlError != nil || err != nil { + scenarioTest.Fatalf("enable peer GRO: %v, %v", controlError, err) + } + } + options := SplicePacketOptions{NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}} + var prefix []byte + if mode != "direct" { + target := M.ParseSocksaddr("192.0.2.1:53") + prefix = []byte{0, 0, 0, 1, 192, 0, 2, 1, 0, 53} + if ipv6 { + target = M.ParseSocksaddr("[2001:db8::1]:53") + prefix = append([]byte{0, 0, 0, 4}, netip.MustParseAddr("2001:db8::1").AsSlice()...) + prefix = append(prefix, 0, 53) + } + if mode == "socks-fqdn" { + target = M.ParseSocksaddr("batch.example:53") + prefix = append([]byte{0, 0, 0, 3, byte(len(target.Fqdn))}, target.Fqdn...) + prefix = append(prefix, 0, 53) + } + associate := socks.NewAssociatePacketConn(upstream, target, upstream) + options.Offload, _ = associate.CreatePacketOffload() + options.FrontHeadroom = associate.FrontHeadroom() + options.NAT.Destination = target + } + if !conn.Splice(&kernelPacketSocket{UDPConn: upstream}, options) { + scenarioTest.Fatal("UDP splice refused") + } + for cycle := range 2 { + payloads := make([][]byte, 20) + wirePackets := make([][]byte, len(payloads)) + for index := range payloads { + size := 1200 + if index == len(payloads)-1 { + size = 133 + } + payloads[index] = kernelPayload(size, uint32(index+cycle*31+53)) + wirePackets[index] = append(bytes.Clone(prefix), payloads[index]...) + } + _, _, err = client.WriteMsgUDP(bytes.Join(payloads, nil), kernelUDPSegmentControl(1200), nil) + if err != nil { + scenarioTest.Fatal("kernel GSO upload:", err) + } + source := kernelReadUDPSegments(scenarioTest, peer, wirePackets) + gro, inspectError := unix.GetsockoptInt(monitorDescriptor, unix.IPPROTO_UDP, unix.UDP_GRO) + if inspectError != nil || gro != 1 { + scenarioTest.Fatalf("splice socket GRO disabled: %d, %v", gro, inspectError) + } + _, _, err = peer.WriteMsgUDP(bytes.Join(wirePackets, nil), kernelUDPSegmentControl(1200+len(prefix)), source) + if err != nil { + scenarioTest.Fatal("peer GSO download:", err) + } + kernelReadUDPSegments(scenarioTest, client, payloads) + allocator.waitIdle(scenarioTest) + time.Sleep(10 * time.Millisecond) + } + }) + } + } + }) + } +} + +func kernelUDPSegmentControl(size int) []byte { + control := make([]byte, unix.CmsgSpace(2)) + header := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + header.Level = unix.IPPROTO_UDP + header.Type = unix.UDP_SEGMENT + header.SetLen(unix.CmsgLen(2)) + binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):], uint16(size)) + return control +} + +func kernelReadUDPSegments(t *testing.T, socket *net.UDPConn, expected [][]byte) *net.UDPAddr { + t.Helper() + data := make([]byte, 65535) + control := make([]byte, 128) + var source *net.UDPAddr + segmented := false + for index := 0; index < len(expected); { + length, controlLength, flags, sender, err := socket.ReadMsgUDP(data, control) + if err != nil || flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 { + t.Fatalf("receive segment %d: flags=%d, %v", index, flags, err) + } + source = sender + messages, err := unix.ParseSocketControlMessage(control[:controlLength]) + if err != nil { + t.Fatal(err) + } + segmentSize := length + for _, message := range messages { + if message.Header.Level == unix.IPPROTO_UDP && message.Header.Type == unix.UDP_GRO { + segmentSize = int(binary.NativeEndian.Uint32(message.Data)) + segmented = segmented || segmentSize < length + } + } + for offset := 0; offset < length; offset += segmentSize { + packet := data[offset:min(offset+segmentSize, length)] + if index >= len(expected) || !bytes.Equal(packet, expected[index]) { + t.Fatalf("segment %d corrupted: length=%d", index, len(packet)) + } + index++ + } + } + if !segmented { + t.Fatal("kernel received no UDP GSO/GRO aggregate") + } + return source +} diff --git a/stack_go_integration_packet_test.go b/stack_go_integration_packet_test.go new file mode 100644 index 00000000..0cd29e27 --- /dev/null +++ b/stack_go_integration_packet_test.go @@ -0,0 +1,717 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + std_bufio "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/protocol/socks" +) + +func TestGoKernelPacketTimeout(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, conn, destination := fixture.packetPair(test, ipv6) + if !conn.SetTimeout(100 * time.Millisecond) { + test.Fatal("cannot set live UDP session timeout") + } + buffer, _, err := conn.WaitReadPacket() + if buffer != nil { + buffer.Release() + } + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("idle UDP session did not close: %v", err) + } + payload := kernelPayload(1200, 103) + _, err = client.Write(payload) + if err != nil { + test.Fatal(err) + } + fixture.access.Lock() + accepted := fixture.udp[destination.Port] + fixture.access.Unlock() + select { + case replacement := <-accepted: + defer replacement.Close() + replacement.SetReadDeadline(time.Now().Add(time.Second)) + options := N.NewReadWaitOptions(nil, replacement) + packet := options.NewBufferSize(1500) + _, err = replacement.ReadPacket(packet) + if err != nil || !bytes.Equal(packet.Bytes(), payload) { + packet.Release() + test.Fatalf("UDP session renewal lost its first packet: %v", err) + } + options.PostReturn(packet) + err = replacement.WritePacket(packet, destination) + if err != nil { + test.Fatal(err) + } + response := make([]byte, 1500) + n, readErr := client.Read(response) + if readErr != nil || !bytes.Equal(response[:n], payload) { + test.Fatalf("renewed UDP session response: %v", readErr) + } + case <-time.After(time.Second): + test.Fatal("next UDP packet did not create a new session") + } + }) + } +} + +func TestGoKernelPacketBuffer(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + client, conn, destination := fixture.packetPair(addressTest, ipv6) + for _, size := range []int{0, 1, 1473} { + for _, capacity := range []int{0, 1, 1024, 2048} { + addressTest.Run(fmt.Sprintf("size=%d/capacity=%d", size, capacity), func(packetTest *testing.T) { + payload := kernelPayload(size, uint32(size+71)) + _, err := client.Write(payload) + if err != nil { + packetTest.Fatal(err) + } + prefix := []byte("prefix") + buffer := buf.With(make([]byte, 16+len(prefix)+capacity+32)) + buffer.Resize(16, len(prefix)) + buffer.Reserve(32) + copy(buffer.Bytes(), prefix) + actualDestination, readErr := conn.ReadPacket(buffer) + if capacity == 0 { + if !errors.Is(readErr, io.ErrShortBuffer) { + packetTest.Fatalf("full buffer: %v", readErr) + } + } else if readErr != nil { + packetTest.Fatal(readErr) + } + if actualDestination != destination { + packetTest.Fatalf("destination: %v, want %v", actualDestination, destination) + } + if buffer.Start() != 16 || !bytes.Equal(buffer.Bytes(), append(prefix, payload[:min(size, capacity)]...)) { + packetTest.Fatalf("packet buffer: %x", buffer.Bytes()) + } + marker := kernelPayload(7, 83) + _, err = client.Write(marker) + if err != nil { + packetTest.Fatal(err) + } + buffer.Reset() + _, err = conn.ReadPacket(buffer) + if err != nil || !bytes.Equal(buffer.Bytes(), marker) { + packetTest.Fatalf("next datagram: %x: %v", buffer.Bytes(), err) + } + }) + } + } + }) + } +} + +func (f *kernelStackFixture) packetPair(t *testing.T, ipv6 bool) (*net.UDPConn, *UDPNatConn, M.Socksaddr) { + t.Helper() + port := uint16(20000 + f.port.Add(1)) + accepted := make(chan N.PacketConn, 1) + f.access.Lock() + f.udp[port] = accepted + f.access.Unlock() + destination := M.ParseSocksaddr(f.address(ipv6, port)) + client, err := net.DialUDP("udp", nil, destination.UDPAddr()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { client.Close() }) + err = client.SetWriteBuffer(1 << 20) + if err != nil { + t.Fatal(err) + } + client.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = client.Write([]byte("ready")) + if err != nil { + t.Fatal(err) + } + select { + case conn := <-accepted: + t.Cleanup(func() { conn.Close() }) + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + buffer := buf.NewPacket() + _, err = conn.ReadPacket(buffer) + buffer.Release() + if err != nil { + t.Fatal(err) + } + return client, conn.(*UDPNatConn), destination + case <-time.After(3 * time.Second): + t.Fatal("UDP flow not accepted") + return nil, nil, M.Socksaddr{} + } +} + +type kernelPacketSocket struct { + *net.UDPConn + access sync.Mutex + owner io.Closer + closed bool +} + +func (s *kernelPacketSocket) Attach(owner io.Closer) (io.Closer, bool) { + s.access.Lock() + defer s.access.Unlock() + if s.closed || s.owner != nil { + return nil, false + } + s.owner = owner + return s.UDPConn, true +} + +func (s *kernelPacketSocket) Detach() { + s.access.Lock() + s.owner = nil + closed := s.closed + s.access.Unlock() + if closed { + s.UDPConn.Close() + } +} + +func (s *kernelPacketSocket) Close() error { + s.access.Lock() + s.closed = true + owner := s.owner + s.access.Unlock() + if owner != nil { + return owner.Close() + } + return s.UDPConn.Close() +} + +func TestGoKernelPacket(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + for _, splice := range []string{"none", "direct", "socks", "socks-fqdn"} { + addressTest.Run("splice="+splice, func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + maximumPayload := 65535 - header.IPv4MinimumSize - header.UDPMinimumSize + if ipv6 { + network = "udp6" + address = net.IPv6loopback + maximumPayload = 65535 - header.UDPMinimumSize + } + var ( + peer *net.UDPConn + socksPackets chan kernelSocksPacket + socksDestination M.Socksaddr + headerLength int + ) + switch splice { + case "direct": + var err error + peer, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { peer.Close() }) + upstream, err := net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { upstream.Close() }) + peer.SetWriteBuffer(1 << 20) + upstream.SetWriteBuffer(1 << 20) + peer.SetDeadline(time.Now().Add(3 * time.Second)) + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + SpliceOptions: SpliceOptions{OnClose: func(error) { upstream.Close() }}, + }) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + case "socks", "socks-fqdn": + if splice == "socks" { + socksDestination = M.ParseSocksaddrHostPort("192.0.2.1", 53) + if ipv6 { + socksDestination = M.ParseSocksaddrHostPort("2001:db8::1", 53) + } + } else { + socksDestination = M.ParseSocksaddrHostPort(strings.Repeat(strings.Repeat("a", 63)+".", 3)+strings.Repeat("b", 61), 53) + } + var serverAddress M.Socksaddr + serverAddress, socksPackets = newKernelSocksEchoServer(scenarioTest, address) + socksClient := socks.NewClient(N.SystemDialer, serverAddress, socks.Version5, "", "") + associateConn, err := socksClient.ListenPacket(context.Background(), socksDestination) + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { associateConn.Close() }) + terminal, offload := N.UnwrapPacketOffload(associateConn) + upstream, isUDPConn := terminal.(*net.UDPConn) + if !isUDPConn || offload == nil { + scenarioTest.Fatalf("SOCKS UDP not unwrapped: %T", terminal) + } + upstream.SetWriteBuffer(1 << 20) + _, err = client.Write([]byte("cached")) + if err != nil { + scenarioTest.Fatal(err) + } + cached := N.NewPacketBuffer() + cached.Buffer, cached.Destination, err = conn.WaitReadPacket() + if err != nil { + N.PutPacketBuffer(cached) + scenarioTest.Fatal(err) + } + if !config.multiQueue { + offload = &kernelSocksOffload{ + PacketOffload: offload, + engine: conn.writer.(*GoPacketConn).engine, + cached: 2, + test: scenarioTest, + } + } + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: socksDestination}, + Cached: []*N.PacketBuffer{cached}, + Offload: offload, + FrontHeadroom: N.CalculateFrontHeadroom(associateConn), + RearHeadroom: N.CalculateRearHeadroom(associateConn), + SpliceOptions: SpliceOptions{OnClose: func(error) { associateConn.Close() }}, + }) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + headerLength = 3 + M.SocksaddrSerializer.AddrPortLen(socksDestination) + _, err = client.Write([]byte("ready")) + if err != nil { + scenarioTest.Fatal(err) + } + for _, expected := range []string{"cached", "ready"} { + select { + case packet := <-socksPackets: + if string(packet.payload) != expected || packet.destination != socksDestination { + scenarioTest.Fatalf("SOCKS initial packet: %q to %v", packet.payload, packet.destination) + } + case <-time.After(3 * time.Second): + scenarioTest.Fatal("SOCKS UDP session not established") + } + response := make([]byte, 16) + n, readErr := client.Read(response) + if readErr != nil || string(response[:n]) != expected { + scenarioTest.Fatalf("SOCKS initial response: %q: %v", response[:n], readErr) + } + } + } + sizes := []int{0, 1, 1472, 1473, 8193, 65507} + if ipv6 { + sizes = append(sizes, 1452, 1453, 65527) + } + if headerLength > 0 { + sizes = append(sizes, maximumPayload-headerLength) + } + for _, size := range sizes { + if headerLength+size > maximumPayload { + continue + } + scenarioTest.Run(fmt.Sprintf("size=%d", size), func(packetTest *testing.T) { + payload := kernelPayload(size, uint32(size+37)) + _, err := client.Write(payload) + if err != nil { + packetTest.Fatal("kernel upload:", err) + } + var received []byte + switch splice { + case "none": + options := N.NewReadWaitOptions(nil, conn) + buffer := options.NewBufferSize(65535) + _, err = conn.ReadPacket(buffer) + received = bytes.Clone(buffer.Bytes()) + if err != nil { + buffer.Release() + packetTest.Fatal("Go upload:", err) + } + options.PostReturn(buffer) + err = conn.WritePacket(buffer, destination) + case "direct": + data := make([]byte, 65535) + n, source, readErr := peer.ReadFromUDP(data) + if readErr != nil { + packetTest.Fatal("spliced upload:", readErr) + } + received = data[:n] + _, err = peer.WriteToUDP(payload, source) + default: + select { + case packet := <-socksPackets: + if packet.destination != socksDestination { + packetTest.Fatalf("SOCKS destination: %v, want %v", packet.destination, socksDestination) + } + received = packet.payload + case <-time.After(3 * time.Second): + packetTest.Fatal("spliced SOCKS upload timed out") + } + } + if !bytes.Equal(received, payload) { + packetTest.Fatalf("upload bytes=%d want=%d", len(received), size) + } + if err != nil { + packetTest.Fatal("download write:", err) + } + response := make([]byte, 65535) + n, readErr := client.Read(response) + if readErr != nil { + packetTest.Fatal("kernel download:", readErr) + } + if !bytes.Equal(response[:n], payload) { + packetTest.Fatalf("download bytes=%d want=%d", n, size) + } + }) + } + }) + } + addressTest.Run("tcp_after_udp", func(streamTest *testing.T) { + client, server := fixture.pair(streamTest, ipv6) + err := kernelTransfer(client, server, kernelPayload(64<<10, 41), false) + if err != nil { + streamTest.Fatal("TCP upload after UDP headroom negotiation:", err) + } + err = kernelTransfer(server, client, kernelPayload(64<<10, 43), true) + if err != nil { + streamTest.Fatal("TCP download after UDP headroom negotiation:", err) + } + }) + }) + } + }) + } +} + +type kernelSocksOffload struct { + N.PacketOffload + engine *goEngine + cached int + test *testing.T +} + +func (o *kernelSocksOffload) EncodePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + if o.cached > 0 { + o.cached-- + } else { + borrowed := false + for _, frame := range o.engine.frames { + if buffer == frame.buffer { + borrowed = true + break + } + } + if !borrowed { + for _, entry := range o.engine.reassemblyEntries { + if buffer == entry.buffer { + borrowed = true + break + } + } + } + if !borrowed { + err := E.New("SOCKS splice copied the TUN payload before encoding") + o.test.Error(err) + return err + } + } + return o.PacketOffload.EncodePacket(buffer, destination) +} + +type kernelSocksPacket struct { + payload []byte + destination M.Socksaddr +} + +type kernelSocksEchoServer struct { + packets chan kernelSocksPacket +} + +func newKernelSocksEchoServer(t *testing.T, address net.IP) (M.Socksaddr, chan kernelSocksPacket) { + t.Helper() + listener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: address}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + server := &kernelSocksEchoServer{packets: make(chan kernelSocksPacket, 16)} + go func() { + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + go func() { + handleErr := socks.HandleConnectionEx(context.Background(), conn, std_bufio.NewReader(conn), nil, server, server, 0, M.SocksaddrFromNet(conn.RemoteAddr()), nil) + if handleErr != nil { + conn.Close() + } + }() + } + }() + return M.SocksaddrFromNet(listener.Addr()), server.packets +} + +func (s *kernelSocksEchoServer) ListenPacket(listenConfig net.ListenConfig, ctx context.Context, network string, address string) (net.PacketConn, error) { + conn, err := listenConfig.ListenPacket(ctx, network, address) + if err != nil { + return nil, err + } + udpConn := conn.(*net.UDPConn) + udpConn.SetReadBuffer(1 << 20) + udpConn.SetWriteBuffer(1 << 20) + return conn, nil +} + +func (s *kernelSocksEchoServer) NewConnectionEx(ctx context.Context, conn net.Conn, source M.Socksaddr, destination M.Socksaddr, onClose N.CloseHandlerFunc) { + conn.Close() +} + +func (s *kernelSocksEchoServer) NewPacketConnectionEx(ctx context.Context, conn N.PacketConn, source M.Socksaddr, destination M.Socksaddr, onClose N.CloseHandlerFunc) { + go func() { + defer conn.Close() + frontHeadroom := N.CalculateFrontHeadroom(conn) + for { + buffer := buf.NewSize(frontHeadroom + 65535) + buffer.Resize(frontHeadroom, 0) + packetDestination, err := conn.ReadPacket(buffer) + if err != nil { + buffer.Release() + return + } + s.packets <- kernelSocksPacket{payload: bytes.Clone(buffer.Bytes()), destination: packetDestination} + err = conn.WritePacket(buffer, packetDestination) + if err != nil { + return + } + } + }() +} + +var ( + _ SpliceSocket = (*kernelPacketSocket)(nil) + _ socks.HandlerEx = (*kernelSocksEchoServer)(nil) + _ socks.PacketListener = (*kernelSocksEchoServer)(nil) +) + +type kernelPacketAllocator struct { + buf.Allocator + active atomic.Int64 +} + +func (a *kernelPacketAllocator) Get(size int) []byte { + buffer := a.Allocator.Get(size) + if buffer != nil { + a.active.Add(1) + } + return buffer +} + +func (a *kernelPacketAllocator) Put(buffer []byte) error { + err := a.Allocator.Put(buffer) + if err == nil { + a.active.Add(-1) + } + return err +} + +func (a *kernelPacketAllocator) waitIdle(t *testing.T) { + t.Helper() + deadline := time.Now().Add(time.Second) + for a.active.Load() != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if active := a.active.Load(); active != 0 { + t.Fatalf("idle connections retain %d pooled buffers", active) + } +} + +func TestGoKernelPacketBatchIdle(t *testing.T) { + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator} + buf.DefaultAllocator = allocator + t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"batch", "connected", "unconnected", "socks-fqdn"} { + configurationTest.Run(fmt.Sprintf("ipv6=%v/%s", ipv6, mode), func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + var peer *net.UDPConn + var source *net.UDPAddr + var prefix []byte + if mode != "batch" { + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + if ipv6 { + network = "udp6" + address = net.IPv6loopback + } + var err error + peer, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + defer peer.Close() + peer.SetReadBuffer(1 << 20) + peer.SetWriteBuffer(1 << 20) + peer.SetDeadline(time.Now().Add(5 * time.Second)) + var upstream *net.UDPConn + if mode != "unconnected" { + upstream, err = net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + } else { + upstream, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + } + if err != nil { + scenarioTest.Fatal(err) + } + defer upstream.Close() + upstream.SetReadBuffer(1 << 20) + spliceOptions := SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + } + if mode == "socks-fqdn" { + domain := strings.Repeat(strings.Repeat("a", 63)+".", 3) + strings.Repeat("b", 61) + target := M.ParseSocksaddrHostPort(domain, 53) + prefix = append([]byte{0, 0, 0, 3, byte(len(domain))}, domain...) + prefix = append(prefix, 0, 53) + associate := socks.NewAssociatePacketConn(upstream, target, upstream) + spliceOptions.Offload, _ = associate.CreatePacketOffload() + spliceOptions.FrontHeadroom = associate.FrontHeadroom() + spliceOptions.NAT.Destination = target + } + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, spliceOptions) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + } + for cycle := range 3 { + payloads := make([][]byte, 24) + for index := range payloads { + size := 1200 + if index >= 16 { + size = []int{1, 0, 1453, 8193, 1201, 0, 3, 1200}[index-16] + } + payloads[index] = kernelPayload(size, uint32(cycle*31+index+1)) + _, err := client.Write(payloads[index]) + if err != nil { + scenarioTest.Fatal(err) + } + } + uploaded := make([]bool, len(payloads)) + for index := range payloads { + var actual []byte + if peer != nil { + data := make([]byte, 65535) + length, sender, err := peer.ReadFromUDP(data) + if err != nil { + scenarioTest.Fatalf("upload packet %d: %v", index, err) + } + source = sender + if length < len(prefix) || !bytes.Equal(data[:len(prefix)], prefix) { + scenarioTest.Fatal("SOCKS packet header corrupted") + } + actual = data[len(prefix):length] + } else { + buffer, _, err := conn.WaitReadPacket() + if err != nil { + scenarioTest.Fatal(err) + } + actual = bytes.Clone(buffer.Bytes()) + buffer.Release() + } + kernelCheckBatchPacket(scenarioTest, actual, payloads, uploaded, index, !config.multiQueue) + } + if peer != nil { + for _, payload := range payloads { + _, err := peer.WriteToUDP(append(bytes.Clone(prefix), payload...), source) + if err != nil { + scenarioTest.Fatal(err) + } + } + } else { + options := N.NewReadWaitOptions(nil, conn) + buffers := make([]*buf.Buffer, len(payloads)) + destinations := make([]M.Socksaddr, len(payloads)) + for index, payload := range payloads { + buffers[index] = options.NewBufferSize(len(payload)) + copy(buffers[index].Extend(len(payload)), payload) + options.PostReturn(buffers[index]) + destinations[index] = destination + } + err := bufio.NewPacketBatchWriter(conn).WritePacketBatch(buffers, destinations) + if err != nil { + scenarioTest.Fatal(err) + } + } + downloaded := make([]bool, len(payloads)) + for index := range payloads { + data := make([]byte, 65535) + length, err := client.Read(data) + if err != nil { + scenarioTest.Fatalf("download packet %d: %v", index, err) + } + kernelCheckBatchPacket(scenarioTest, data[:length], payloads, downloaded, index, !config.multiQueue) + } + allocator.waitIdle(scenarioTest) + if conn.isClosed() { + scenarioTest.Fatal("idle test closed the UDP session") + } + time.Sleep(10 * time.Millisecond) + } + }) + } + } + }) + } +} + +func kernelCheckBatchPacket(t *testing.T, packet []byte, expected [][]byte, received []bool, index int, ordered bool) { + t.Helper() + if ordered { + if !bytes.Equal(packet, expected[index]) { + t.Fatalf("packet %d corrupted or reordered: received %d bytes, expected %d", index, len(packet), len(expected[index])) + } + return + } + for candidate, payload := range expected { + if !received[candidate] && bytes.Equal(packet, payload) { + received[candidate] = true + return + } + } + t.Fatalf("packet %d corrupted or duplicated: received %d bytes", index, len(packet)) +} diff --git a/stack_go_integration_splice_test.go b/stack_go_integration_splice_test.go new file mode 100644 index 00000000..1aef4aac --- /dev/null +++ b/stack_go_integration_splice_test.go @@ -0,0 +1,187 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "errors" + "io" + "net" + "testing" + "time" + + M "github.com/sagernet/sing/common/metadata" + "github.com/sagernet/sing/protocol/socks" +) + +func TestGoKernelSpliceOwnership(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + t.Run("tcp", func(streamTest *testing.T) { + client, conn := fixture.pair(streamTest, false) + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + streamTest.Fatal(err) + } + defer listener.Close() + upstream, err := net.DialTCP("tcp4", nil, listener.Addr().(*net.TCPAddr)) + if err != nil { + streamTest.Fatal(err) + } + defer upstream.Close() + peer, err := listener.AcceptTCP() + if err != nil { + streamTest.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + closed := make(chan error, 1) + owner := &kernelSocket{TCPConn: upstream} + accepted := conn.Splice(owner, SpliceOptions{ + OnClose: func(closeError error) { closed <- closeError }, + }) + if !accepted { + streamTest.Fatal("TCP splice refused") + } + payload := kernelPayload(4097, 193) + _, err = client.Write(payload) + if err != nil { + streamTest.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(peer, data) + if err != nil || !bytes.Equal(data, payload) { + streamTest.Fatalf("spliced TCP upload: %v", err) + } + err = upstream.SetDeadline(time.Now().Add(-time.Second)) + if !errors.Is(err, net.ErrClosed) { + streamTest.Fatalf("original TCP connection survived transfer: %v", err) + } + _, err = peer.Write(payload) + if err != nil { + streamTest.Fatal(err) + } + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + streamTest.Fatalf("spliced TCP download after original connection closed: %v", err) + } + owner.Close() + owner.Close() + select { + case <-closed: + case <-time.After(time.Second): + streamTest.Fatal("TCP splice did not close its socket") + } + _, err = peer.Read(data) + var networkError net.Error + if err == nil || errors.As(err, &networkError) && networkError.Timeout() { + streamTest.Fatalf("TCP peer survived splice close: %v", err) + } + }) + t.Run("udp", func(packetTest *testing.T) { + client, conn, destination := fixture.packetPair(packetTest, false) + peer, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + packetTest.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + upstream, err := net.DialUDP("udp4", nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + packetTest.Fatal(err) + } + defer upstream.Close() + address := upstream.LocalAddr().(*net.UDPAddr) + closed := make(chan error, 1) + owner := &kernelPacketSocket{UDPConn: upstream} + accepted := conn.Splice(owner, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + SpliceOptions: SpliceOptions{OnClose: func(closeError error) { closed <- closeError }}, + }) + if !accepted { + packetTest.Fatal("UDP splice refused") + } + payload := kernelPayload(1023, 197) + _, err = client.Write(payload) + if err != nil { + packetTest.Fatal(err) + } + data := make([]byte, len(payload)) + n, source, err := peer.ReadFromUDP(data) + if err != nil || !bytes.Equal(data[:n], payload) { + packetTest.Fatalf("spliced UDP upload: %v", err) + } + err = upstream.SetDeadline(time.Now().Add(-time.Second)) + if !errors.Is(err, net.ErrClosed) { + packetTest.Fatalf("original UDP connection survived transfer: %v", err) + } + _, err = peer.WriteToUDP(payload, source) + if err != nil { + packetTest.Fatal(err) + } + n, err = client.Read(data) + if err != nil || !bytes.Equal(data[:n], payload) { + packetTest.Fatalf("spliced UDP download after original connection closed: %v", err) + } + owner.Close() + owner.Close() + select { + case <-closed: + case <-time.After(time.Second): + packetTest.Fatal("UDP splice did not close its socket") + } + rebound, err := net.ListenUDP("udp4", address) + if err != nil { + packetTest.Fatal("UDP endpoint survived splice close: ", err) + } + rebound.Close() + }) + t.Run("fallback", func(fallbackTest *testing.T) { + _, conn, _ := fixture.packetPair(fallbackTest, false) + upstream, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + fallbackTest.Fatal(err) + } + defer upstream.Close() + address := upstream.LocalAddr().(*net.UDPAddr) + peer, err := net.DialUDP("udp4", nil, upstream.LocalAddr().(*net.UDPAddr)) + if err != nil { + fallbackTest.Fatal(err) + } + defer peer.Close() + offload, _ := (&socks.AssociatePacketConn{}).CreatePacketOffload() + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{Offload: offload}) + if accepted { + fallbackTest.Fatal("unconnected SOCKS socket unexpectedly accepted") + } + upstream.SetReadDeadline(time.Now().Add(time.Second)) + completed := make(chan error, 1) + go func() { + data := make([]byte, 1) + _, readError := upstream.Read(data) + completed <- readError + }() + select { + case err = <-completed: + fallbackTest.Fatalf("fallback reader did not wait: %v", err) + case <-time.After(20 * time.Millisecond): + } + _, err = peer.Write([]byte{1}) + if err != nil { + fallbackTest.Fatal(err) + } + select { + case err = <-completed: + if err != nil { + fallbackTest.Fatal("runtime reader after splice rejection: ", err) + } + case <-time.After(2 * time.Second): + fallbackTest.Fatal("runtime reader was not woken") + } + upstream.Close() + rebound, err := net.ListenUDP("udp4", address) + if err != nil { + fallbackTest.Fatal("rejected splice retained a duplicate socket: ", err) + } + rebound.Close() + }) +} diff --git a/stack_go_integration_test.go b/stack_go_integration_test.go new file mode 100644 index 00000000..41a9f86a --- /dev/null +++ b/stack_go_integration_test.go @@ -0,0 +1,701 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +type kernelStackConfig struct { + stackFactory func(StackOptions) (Stack, error) + mtu uint32 + gso bool + multiQueue bool + prepare func(*testing.T, *Options) + configure func(*testing.T, Options) + pressure func() MemoryPressure + ctx context.Context + handshake func(*GoConn) error + udpMapping NATMapping + udpFiltering NATFiltering + dialer net.Dialer + socketBuffer int + upstreamSocketBuffer int +} + +type kernelAccept struct { + conn *GoConn + err error +} + +type kernelStackFixture struct { + stack *Go + options Options + access sync.Mutex + tcp map[uint16]chan kernelAccept + udp map[uint16]chan N.PacketConn + port atomic.Uint32 + dialer net.Dialer + socketBuffer int + upstreamSocketBuffer int + handshake func(*GoConn) error +} + +var kernelInterfaceSequence atomic.Uint32 + +func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackFixture { + t.Helper() + index := kernelInterfaceSequence.Add(1) + fixture := &kernelStackFixture{ + tcp: make(map[uint16]chan kernelAccept), + udp: make(map[uint16]chan N.PacketConn), + dialer: config.dialer, + socketBuffer: config.socketBuffer, + upstreamSocketBuffer: config.upstreamSocketBuffer, + handshake: config.handshake, + options: Options{ + Name: fmt.Sprintf("gotest%d", index), + MTU: config.mtu, + GSO: config.gso, + MultiQueue: config.multiQueue, + Inet4Address: []netip.Prefix{netip.MustParsePrefix(fmt.Sprintf("198.19.%d.1/24", 200+index%50))}, + Inet6Address: []netip.Prefix{netip.MustParsePrefix(fmt.Sprintf("fd73:ab91:%x::1/64", index))}, + EXP_ExternalConfiguration: true, + EXP_MultiPendingPackets: true, + Logger: logger.NOP(), + }, + } + if fixture.dialer.Timeout == 0 { + fixture.dialer.Timeout = 3 * time.Second + } + if runtime.GOOS == "darwin" { + fixture.options.Name = fmt.Sprintf("utun%d", 200+index) + } + if config.prepare != nil { + config.prepare(t, &fixture.options) + } + device, err := New(fixture.options) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { device.Close() }) + configureKernelInterface(t, device, fixture.options) + if config.configure != nil { + config.configure(t, fixture.options) + } + err = device.Start() + if err != nil { + t.Fatal(err) + } + if config.ctx == nil { + config.ctx = context.Background() + } + stackOptions := StackOptions{ + Context: config.ctx, Tun: device, TunOptions: fixture.options, + Handler: fixture, Logger: logger.NOP(), UDPTimeout: time.Minute, ICMPTimeout: time.Minute, + MemoryPressure: config.pressure, + UDPMapping: config.udpMapping, UDPFiltering: config.udpFiltering, + } + var stack Stack + if config.stackFactory != nil { + stack, err = config.stackFactory(stackOptions) + if err != nil { + t.Fatal(err) + } + } else { + fixture.stack = NewGo(stackOptions) + stack = fixture.stack + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + return fixture +} + +func (f *kernelStackFixture) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionAccept} +} + +func (f *kernelStackFixture) NewDNSPacket([]byte, M.Socksaddr, M.Socksaddr, N.PacketWriter) {} + +func (f *kernelStackFixture) NewConnectionEx(_ context.Context, conn net.Conn, _, destination M.Socksaddr, _ N.CloseHandlerFunc) { + f.access.Lock() + accepted := f.tcp[destination.Port] + f.access.Unlock() + if accepted == nil { + conn.Close() + return + } + goConn := conn.(*GoConn) + goConn.SetDeadline(time.Now().Add(10 * time.Second)) + var err error + if f.handshake != nil { + err = f.handshake(goConn) + } else { + err = goConn.HandshakeSuccess() + } + accepted <- kernelAccept{conn: goConn, err: err} +} + +func (f *kernelStackFixture) NewPacketConnectionEx(_ context.Context, conn N.PacketConn, _, destination M.Socksaddr, _ N.CloseHandlerFunc) { + f.access.Lock() + accepted := f.udp[destination.Port] + f.access.Unlock() + if accepted == nil { + conn.Close() + return + } + accepted <- conn +} + +func (f *kernelStackFixture) address(ipv6 bool, port uint16) string { + address := f.options.Inet4Address[0].Addr().Next() + if ipv6 { + address = f.options.Inet6Address[0].Addr().Next() + } + return netip.AddrPortFrom(address, port).String() +} + +func (f *kernelStackFixture) pair(t *testing.T, ipv6 bool) (*net.TCPConn, *GoConn) { + t.Helper() + port := uint16(20000 + f.port.Add(1)) + accepted := make(chan kernelAccept, 1) + f.access.Lock() + f.tcp[port] = accepted + f.access.Unlock() + conn, err := f.dialer.Dial("tcp", f.address(ipv6, port)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + if f.socketBuffer > 0 { + err = conn.(*net.TCPConn).SetReadBuffer(f.socketBuffer) + if err != nil { + t.Fatal(err) + } + err = conn.(*net.TCPConn).SetWriteBuffer(f.socketBuffer) + if err != nil { + t.Fatal(err) + } + } + conn.SetDeadline(time.Now().Add(10 * time.Second)) + select { + case result := <-accepted: + if result.err != nil { + t.Fatal(result.err) + } + t.Cleanup(func() { result.conn.Close() }) + return conn.(*net.TCPConn), result.conn + case <-time.After(3 * time.Second): + t.Fatal("Go connection handshake did not complete") + return nil, nil + } +} + +type kernelSocket struct { + *net.TCPConn + access sync.Mutex + owner io.Closer + closed bool +} + +func (s *kernelSocket) Attach(owner io.Closer) (io.Closer, bool) { + s.access.Lock() + defer s.access.Unlock() + if s.closed || s.owner != nil { + return nil, false + } + s.owner = owner + return s.TCPConn, true +} + +func (s *kernelSocket) Detach() { + s.access.Lock() + s.owner = nil + closed := s.closed + s.access.Unlock() + if closed { + s.TCPConn.Close() + } +} + +func (s *kernelSocket) Close() error { + s.access.Lock() + s.closed = true + owner := s.owner + s.access.Unlock() + if owner != nil { + return owner.Close() + } + return s.TCPConn.Close() +} + +func (f *kernelStackFixture) splicePair(t *testing.T, ipv6 bool) (*net.TCPConn, *net.TCPConn, *atomic.Int64, *atomic.Int64) { + t.Helper() + client, goConn := f.pair(t, ipv6) + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + remote, err := net.DialTCP("tcp4", nil, listener.Addr().(*net.TCPAddr)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { remote.Close() }) + peer, err := listener.AcceptTCP() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { peer.Close() }) + if f.upstreamSocketBuffer > 0 { + for _, socket := range []*net.TCPConn{remote, peer} { + err = socket.SetReadBuffer(f.upstreamSocketBuffer) + if err != nil { + t.Fatal(err) + } + err = socket.SetWriteBuffer(f.upstreamSocketBuffer) + if err != nil { + t.Fatal(err) + } + } + } + peer.SetDeadline(time.Now().Add(10 * time.Second)) + upload := new(atomic.Int64) + download := new(atomic.Int64) + spliced := goConn.Splice(&kernelSocket{TCPConn: remote}, SpliceOptions{ + ReadCounters: []N.CountFunc{func(n int64) { upload.Add(n) }}, + WriteCounters: []N.CountFunc{func(n int64) { download.Add(n) }}, + OnClose: func(error) { remote.Close() }, + }) + if !spliced { + t.Fatal("splice refused") + } + return client, peer, upload, download +} + +func kernelPayload(size int, seed uint32) []byte { + data := make([]byte, size) + for index := range data { + seed ^= seed << 13 + seed ^= seed >> 17 + seed ^= seed << 5 + data[index] = byte(seed) + } + return data +} + +func kernelTransfer(sender net.Conn, receiver net.Conn, payload []byte, buffered bool) error { + written := make(chan error, 1) + go func() { + var err error + if buffered { + writer := sender.(N.ExtendedWriter) + for offset := 0; offset < len(payload); { + length := min(len(payload)-offset, 32749) + buffer := buf.NewSize(length + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload[offset : offset+length])) + err = writer.WriteBuffer(buffer) + if err != nil { + break + } + offset += length + } + } else { + _, err = sender.Write(payload) + } + if err == nil { + err = N.CloseWrite(sender) + } + written <- err + }() + data, err := io.ReadAll(io.LimitReader(receiver, int64(len(payload)+1))) + if err != nil { + return err + } + if !bytes.Equal(data, payload) { + return E.New("payload mismatch: received ", len(data), " bytes, wanted ", len(payload)) + } + return <-written +} + +func TestGoKernelStream(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 9000}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true}, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + for _, splice := range []bool{false, true} { + addressTest.Run(fmt.Sprintf("duplex/splice=%v", splice), func(scenarioTest *testing.T) { + var client, server net.Conn + var upload, download *atomic.Int64 + if splice { + client, server, upload, download = fixture.splicePair(scenarioTest, ipv6) + } else { + client, server = fixture.pair(scenarioTest, ipv6) + } + payload := kernelPayload(4<<20, 71) + response := kernelPayload(4<<20, 97) + result := make(chan error, 1) + go func() { result <- kernelTransfer(client, server, payload, false) }() + err := kernelTransfer(server, client, response, !splice) + if err != nil { + scenarioTest.Error("download:", err) + } + err = <-result + if err != nil { + scenarioTest.Error("upload:", err) + } + if splice && (upload.Load() != int64(len(payload)) || download.Load() != int64(len(response))) { + scenarioTest.Errorf("counts upload=%d download=%d", upload.Load(), download.Load()) + } + }) + } + addressTest.Run("read_deadline_recovery", func(scenarioTest *testing.T) { + client, server := fixture.pair(scenarioTest, ipv6) + server.SetReadDeadline(time.Now().Add(20 * time.Millisecond)) + data := make([]byte, 7) + _, err := server.Read(data) + if !errors.Is(err, os.ErrDeadlineExceeded) { + scenarioTest.Fatalf("read deadline: %v", err) + } + server.SetReadDeadline(time.Now().Add(time.Second)) + _, err = client.Write([]byte("resumed")) + if err != nil { + scenarioTest.Fatal(err) + } + _, err = io.ReadFull(server, data) + if err != nil || string(data) != "resumed" { + scenarioTest.Fatalf("recovery: %q %v", data, err) + } + }) + addressTest.Run("write_deadline_recovery", func(scenarioTest *testing.T) { + client, server := fixture.pair(scenarioTest, ipv6) + client.SetReadBuffer(4096) + server.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + payload := kernelPayload(8<<20, 31) + n, err := server.Write(payload) + if !errors.Is(err, os.ErrDeadlineExceeded) { + scenarioTest.Fatalf("write %d/%d deadline: %v", n, len(payload), err) + } + server.SetWriteDeadline(time.Now().Add(8 * time.Second)) + client.SetReadBuffer(4 << 20) + written := make(chan error, 1) + go func() { + _, writeErr := server.Write(payload[n:]) + if writeErr == nil { + writeErr = server.CloseWrite() + } + written <- writeErr + }() + data, readErr := io.ReadAll(client) + if readErr != nil { + scenarioTest.Fatalf("initial accepted=%d received=%d: %v; buffered=%d sent=%d transmitted=%d unacked=%d permit=%d", n, len(data), readErr, server.bufferedTail.Load(), server.sentTail.Load(), server.transmittedTail.Load(), server.sendUnacked.Load(), server.sendPermit.Load()) + } + if !bytes.Equal(data, payload) { + scenarioTest.Fatalf("deadline recovery: received %d/%d", len(data), len(payload)) + } + writeErr := <-written + if writeErr != nil { + scenarioTest.Fatal(writeErr) + } + }) + addressTest.Run("wait_read_buffer", func(scenarioTest *testing.T) { + client, server := fixture.pair(scenarioTest, ipv6) + server.InitializeReadWaiter(N.ReadWaitOptions{FrontHeadroom: 91, RearHeadroom: 73}) + payload := kernelPayload(256<<10, 17) + written := make(chan error, 1) + go func() { + _, writeErr := client.Write(payload) + if writeErr == nil { + writeErr = client.CloseWrite() + } + written <- writeErr + }() + var data []byte + for { + buffer, readErr := server.WaitReadBuffer() + if readErr == io.EOF { + break + } + if readErr != nil { + scenarioTest.Fatal(readErr) + } + data = append(data, buffer.Bytes()...) + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + if !bytes.Equal(data, payload) { + scenarioTest.Fatalf("read waiter received %d/%d", len(data), len(payload)) + } + writeErr := <-written + if writeErr != nil { + scenarioTest.Fatal(writeErr) + } + }) + }) + } + }) + } +} + +var ( + _ Handler = (*kernelStackFixture)(nil) + _ SpliceSocket = (*kernelSocket)(nil) +) + +func TestGoKernelFlowControl(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 9000}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}, kernelStackConfig{mtu: 9000, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configTest *testing.T) { + config.socketBuffer = 128 << 10 + config.upstreamSocketBuffer = 128 << 10 + fixture := newKernelStackFixture(configTest, config) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + configTest.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + size := 4 << 20 + if runtime.GOOS == "windows" && mode == "splice" { + size = 16 << 20 + } + kernelBackpressure(test, fixture, ipv6, mode, size) + }) + } + } + }) + } +} + +func kernelBackpressure(t *testing.T, fixture *kernelStackFixture, ipv6 bool, mode string, size int) { + t.Helper() + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(t, ipv6) + } else { + client, server = fixture.pair(t, ipv6) + } + err := client.(*net.TCPConn).SetReadBuffer(4096) + if err != nil { + t.Fatal(err) + } + t.Logf("flow %s -> %s", client.LocalAddr(), client.RemoteAddr()) + deadline := time.Now().Add(10 * time.Second) + client.SetDeadline(deadline) + server.SetDeadline(deadline) + upload := kernelPayload(size, 113) + download := kernelPayload(size, 127) + var uploadAccepted, downloadAccepted atomic.Int64 + written := make(chan error, 2) + go func() { written <- kernelFlowWrite(client, upload, false, &uploadAccepted) }() + go func() { written <- kernelFlowWrite(server, download, mode == "buffer", &downloadAccepted) }() + time.Sleep(100 * time.Millisecond) + if uploadAccepted.Load() == int64(len(upload)) || downloadAccepted.Load() == int64(len(download)) { + t.Fatalf("stalled readers did not backpressure both writers: upload=%d download=%d", uploadAccepted.Load(), downloadAccepted.Load()) + } + read := make(chan error, 2) + go func() { read <- kernelFlowRead(server, upload, mode == "buffer") }() + go func() { read <- kernelFlowRead(client, download, false) }() + for range 2 { + err = <-read + if err != nil { + t.Error(err) + } + } + for range 2 { + err = <-written + if err != nil { + t.Error(err) + } + } +} + +func kernelFlowWrite(conn net.Conn, payload []byte, buffered bool, accepted *atomic.Int64) error { + for offset := 0; offset < len(payload); { + length := min(32749, len(payload)-offset) + var n int + var err error + if buffered { + buffer := buf.NewSize(length + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload[offset : offset+length])) + err = conn.(N.ExtendedWriter).WriteBuffer(buffer) + if err == nil { + n = length + } + } else { + n, err = conn.Write(payload[offset : offset+length]) + } + offset += n + accepted.Add(int64(n)) + if err != nil { + return E.Cause(err, "write flow after ", offset, " bytes") + } + } + return N.CloseWrite(conn) +} + +func kernelFlowRead(conn net.Conn, payload []byte, buffered bool) error { + var waiter N.ReadWaiter + if buffered { + var created bool + waiter, created = bufio.CreateReadWaiter(conn) + if !created { + return E.New("flow connection has no read waiter") + } + waiter.InitializeReadWaiter(N.ReadWaitOptions{MTU: 4093, FrontHeadroom: 91, RearHeadroom: 73}) + } + storage := make([]byte, 16381) + sizes := [...]int{1, 127, 4093, len(storage)} + paused := 0 + for offset, index := 0, 0; ; index++ { + if offset/65536 > paused && paused < 4 && offset < len(payload) { + paused = offset / 65536 + time.Sleep(25 * time.Millisecond) + } + var data []byte + var buffer *buf.Buffer + var err error + if buffered { + buffer, err = waiter.WaitReadBuffer() + if buffer != nil { + data = buffer.Bytes() + } + } else { + var n int + n, err = conn.Read(storage[:sizes[index%len(sizes)]]) + data = storage[:n] + } + end := offset + len(data) + matches := end <= len(payload) && bytes.Equal(data, payload[offset:min(end, len(payload))]) + if buffer != nil { + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + if !matches { + return E.New("flow mismatch at byte ", offset) + } + offset = end + if err == io.EOF && offset == len(payload) { + return nil + } + if err != nil { + return E.Cause(err, "read flow after ", offset, "/", len(payload), " bytes") + } + } +} + +func TestGoKernelSequenceWrap(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, gso: runtime.GOOS == "linux"}) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + t.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + deadline := time.Now().Add(60 * time.Second) + client.SetDeadline(deadline) + server.SetDeadline(deadline) + completed := make(chan error, 1) + go func() { completed <- kernelWrapTransfer(client, server, false) }() + err := kernelWrapTransfer(server, client, mode == "buffer") + if err != nil { + test.Error("download:", err) + } + err = <-completed + if err != nil { + test.Error("upload:", err) + } + }) + } + } +} + +func kernelWrapTransfer(sender net.Conn, receiver net.Conn, buffered bool) error { + const total = uint64(1)<<32 | 123 + const blockSize = 32768 + pattern := kernelPayload(blockSize, 53) + completed := make(chan error, 1) + go func() { + block := bytes.Clone(pattern) + for offset := uint64(0); offset < total; { + length := min(uint64(blockSize), total-offset) + binary.BigEndian.PutUint64(block, offset) + var err error + if buffered { + buffer := buf.NewSize(int(length) + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(block[:length])) + err = sender.(N.ExtendedWriter).WriteBuffer(buffer) + } else { + _, err = sender.Write(block[:length]) + } + if err != nil { + completed <- err + return + } + offset += length + } + completed <- N.CloseWrite(sender) + }() + block := make([]byte, blockSize) + for offset := uint64(0); offset < total; { + length := min(uint64(blockSize), total-offset) + _, err := io.ReadFull(receiver, block[:length]) + if err != nil { + return E.Cause(err, "read sequence offset ", offset) + } + if binary.BigEndian.Uint64(block) != offset || !bytes.Equal(block[8:length], pattern[8:length]) { + return E.New("sequence data mismatch at ", offset) + } + offset += length + } + _, err := receiver.Read(block[:1]) + if err != io.EOF { + return E.New("sequence stream end: ", err) + } + return <-completed +} diff --git a/stack_go_integration_windows_test.go b/stack_go_integration_windows_test.go new file mode 100644 index 00000000..8e6f27b1 --- /dev/null +++ b/stack_go_integration_windows_test.go @@ -0,0 +1,40 @@ +package tun + +import ( + "testing" + + "github.com/sagernet/sing-tun/internal/winipcfg" + + "golang.org/x/sys/windows" +) + +const ( + kernelConnectionRefused = windows.WSAECONNREFUSED + kernelConnectionReset = windows.WSAECONNRESET +) + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + luid := winipcfg.LUID(device.(*NativeTun).adapter.LUID()) + for _, family := range []winipcfg.AddressFamily{windows.AF_INET, windows.AF_INET6} { + addresses := options.Inet4Address + if family == windows.AF_INET6 { + addresses = options.Inet6Address + } + err := luid.SetIPAddressesForFamily(family, addresses) + if err != nil { + t.Fatal(err) + } + row, err := luid.IPInterface(family) + if err != nil { + t.Fatal(err) + } + row.NLMTU = options.MTU + row.DadTransmits = 0 + row.RouterDiscoveryBehavior = winipcfg.RouterDiscoveryDisabled + err = row.Set() + if err != nil { + t.Fatal(err) + } + } +} diff --git a/stack_go_io.go b/stack_go_io.go new file mode 100644 index 00000000..a28549bf --- /dev/null +++ b/stack_go_io.go @@ -0,0 +1,114 @@ +package tun + +import ( + "sync/atomic" + "time" + + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" + N "github.com/sagernet/sing/common/network" +) + +const ( + goTransmitBackoffMin = 250 * time.Microsecond + goTransmitBackoffMax = time.Millisecond + goTransmitBackoffBudget = 3 * time.Millisecond + goDropReportInterval = time.Second +) + +var errGoFrameDropped = E.New("go: frame dropped") + +type goFrame struct { + buffer *buf.Buffer + meta ForwardFrameMeta +} + +type goSocketEvent struct { + token uint32 + readable bool + writable bool +} + +const ( + goInterestRead uint8 = 1 << iota + goInterestWrite +) + +var errGoQueueUnavailable = E.New("go: tun queue unavailable") + +type goPlatformIO interface { + start() error + wait(timeout time.Duration, events []goSocketEvent) (tunReadable bool, count int, err error) + registerSocket(socket *goSocket, token uint32, interest uint8) error + updateSocket(socket *goSocket, interest uint8) error + unregisterSocket(socket *goSocket) + readBurst(frames []goFrame, options N.ReadWaitOptions) (count int, drained bool, err error) + writeFrame(frame [][]byte, meta ForwardFrameMeta) error + writeData(frame [][]byte, meta ForwardFrameMeta) error + writePacketBatch(frames []goUDPFrame) error + releaseReadBuffers() + flush() + transmitPrefix() int + transmitChecksumOffload() bool + transmitSegmentOffload() bool + armTransmitWritable() (bool, error) + takeTransmitWritable() bool + wake() + close() error +} + +type goWriteback struct { + platformIO goPlatformIO +} + +func (w *goWriteback) ReturnHeadroom() int { + return w.platformIO.transmitPrefix() +} + +func (w *goWriteback) WriteReturnPackets(packets [][]byte) error { + prefix := w.platformIO.transmitPrefix() + var writeErr error + for _, packet := range packets { + writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeFrame([][]byte{packet[prefix:]}, ForwardFrameMeta{}))) + } + return writeErr +} + +func newGoReadBuffers(packetSize int, count int, options N.ReadWaitOptions) []*buf.Buffer { + slotSize := options.FrontHeadroom + packetSize + options.RearHeadroom + buffers := make([]*buf.Buffer, count) + for index := range buffers { + buffer := buf.NewSize(slotSize) + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + buffers[index] = buffer + } + return buffers +} + +func goIgnoreDropped(err error) error { + if err == errGoFrameDropped { + return nil + } + return err +} + +type goDropCounter struct { + total atomic.Uint64 + reported atomic.Uint64 + reportedAt atomic.Int64 +} + +func (c *goDropCounter) record(stackLogger logger.Logger, kind string) { + total := c.total.Add(1) + now := time.Now().UnixNano() + previous := c.reportedAt.Load() + if now-previous < int64(goDropReportInterval) { + return + } + if !c.reportedAt.CompareAndSwap(previous, now) { + return + } + stackLogger.Warn("go: dropped ", total-c.reported.Swap(total), " ", kind, ", ", total, " since start") +} diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go new file mode 100644 index 00000000..217b9be2 --- /dev/null +++ b/stack_go_io_darwin.go @@ -0,0 +1,832 @@ +package tun + +import ( + "net" + "os" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/header" + rawfile "github.com/sagernet/sing-tun/internal/rawfile_darwin" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" +) + +const ( + goWakeIdent = 0 + goEngineInlineTransmit = true + + goDarwinTransmitBatch = 64 + goDarwinBatchHeader = 128 + goDarwinBatchIovecs = 8 + // sendit_x builds every packet of the batch before sending any of it, and mbuf_allocpacket + // takes a 16 KB cluster for a packet above MBIGCLBYTES (4096 bytes, prefix included) from a + // pool of a few hundred; a batch of large frames exhausts it and fails with ENOBUFS, so only + // frames that fit a 4 KB cluster are batched and larger ones keep the per-frame writev. + goDarwinBatchFrameLimit = 4096 - PacketOffset + + // utun_pkt_input in netif mode returns ENOSPC once utun_input_chain_count exceeds + // net.utun.max_pending_input (512) and does not free the mbuf, and neither does ctl_send: + // every ENOSPC leaks one mbuf cluster until reboot. The chain only drains when the host side + // syncs the rx ring, which kern_channel_notify triggers per successful input, so the gate keeps + // the estimated chain depth well under the limit and, when nothing moves, submits a small probe + // to trigger a sync rather than wait forever. + goDarwinGateLimit = 512 + goDarwinGateHighWater = 384 + goDarwinGateProbe = 8 + goDarwinGatePoll = 100 * time.Microsecond + goDarwinGateSample = time.Millisecond + goDarwinGateProbeAfter = 16 + goDarwinGateRIBSize = 4096 + goDarwinGateRIBSizeMax = 1 << 20 + + sysprotoControl = 2 + utunOptionEnableNetif = 20 +) + +type goDarwinIO struct { + stack *Go + tunFd int + kqueueFd int + transmitAccess *sync.Mutex + receiveBuffers []*buf.Buffer + readWaitOptions N.ReadWaitOptions + receiveIovecs []unix.Iovec + messageHeaders []rawfile.MsgHdrX + events [goSocketEventBatch + 3]unix.Kevent_t + changes [2]unix.Kevent_t + wakeEvent [1]unix.Kevent_t + transmitWritableEvent [1]unix.Kevent_t + transmitWritable atomic.Bool + receiveBatch int + transmitIovecs []unix.Iovec + socketTokens []uint32 + socketInterests []uint8 + droppedEngineFrames goDropCounter + droppedBatchFrames goDropCounter + leakedClusters goDropCounter + closing atomic.Bool + + batchHeaders []byte + batchIovecs []unix.Iovec + batchMessages []rawfile.MsgHdrX + batchSpill []byte + batchSpilled []bool + batchStart int + batchCount int + packetIovecs [goPacketBatchSize * 3]unix.Iovec + packetMessages [goPacketBatchSize]rawfile.MsgHdrX + + netif bool + interfaceIndex int + epoch time.Time + gateRIB []byte + gateSubmitted int64 + gateSampledAt int64 + gateSampledTime atomic.Int64 + gateInflightSample int64 + gateConsumedBase int64 + gateBlocked atomic.Bool + gateIdlePolls int + gateLastConsumed int64 + gateProbeEnd int64 +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goDarwinIO{stack: stack, epoch: time.Now()}}, nil +} + +func (o *goDarwinIO) start() error { + nativeTun, isNative := o.stack.tun.(*NativeTun) + if !isNative { + return E.New("go: unsupported TUN implementation") + } + err := nativeTun.detachRuntimePoller() + if err != nil { + return E.Cause(err, "go: detach tun from runtime poller") + } + o.tunFd = nativeTun.rawFileDescriptor() + o.transmitAccess = nativeTun.transmitAccess() + err = nativeTun.enableMaxPendingPackets() + if err != nil { + return E.Cause(err, "go: set UTUN_OPT_MAX_PENDING_PACKETS") + } + kqueueFd, err := unix.Kqueue() + if err != nil { + return E.Cause(err, "go: create kqueue") + } + registrations := []unix.Kevent_t{ + {Ident: uint64(o.tunFd), Filter: unix.EVFILT_READ, Flags: unix.EV_ADD | unix.EV_ENABLE}, + {Ident: goWakeIdent, Filter: unix.EVFILT_USER, Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_CLEAR}, + } + _, err = unix.Kevent(kqueueFd, registrations, nil, nil) + if err != nil { + unix.Close(kqueueFd) + return E.Cause(err, "go: register kqueue events") + } + o.kqueueFd = kqueueFd + o.wakeEvent[0] = unix.Kevent_t{Ident: goWakeIdent, Filter: unix.EVFILT_USER, Fflags: unix.NOTE_TRIGGER} + o.transmitWritableEvent[0] = unix.Kevent_t{Ident: uint64(o.tunFd), Filter: unix.EVFILT_WRITE, Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_ONESHOT} + o.receiveIovecs = make([]unix.Iovec, goReadBatch) + o.messageHeaders = make([]rawfile.MsgHdrX, goReadBatch) + o.transmitIovecs = make([]unix.Iovec, 0, 8) + o.receiveBatch = goReceiveBatchMin + o.batchHeaders = make([]byte, goDarwinTransmitBatch*goDarwinBatchHeader) + o.batchIovecs = make([]unix.Iovec, goDarwinTransmitBatch*goDarwinBatchIovecs) + o.batchMessages = make([]rawfile.MsgHdrX, goDarwinTransmitBatch) + o.batchSpilled = make([]bool, goDarwinTransmitBatch) + netif, err := unix.GetsockoptInt(o.tunFd, sysprotoControl, utunOptionEnableNetif) + if err == nil && netif != 0 { + name, nameErr := nativeTun.Name() + if nameErr != nil { + unix.Close(kqueueFd) + return E.Cause(nameErr, "go: utun name") + } + netInterface, lookupErr := net.InterfaceByName(name) + if lookupErr != nil { + unix.Close(kqueueFd) + return E.Cause(lookupErr, "go: utun interface") + } + o.netif = true + o.interfaceIndex = netInterface.Index + consumed, sampleErr := o.readInterfacePackets() + if sampleErr != nil { + unix.Close(kqueueFd) + return E.Cause(sampleErr, "go: utun statistics") + } + o.gateConsumedBase = consumed + } + return nil +} + +// The NET_RT_IFLIST2 route sysctl takes the interface index as its last name component and +// sysctl_iflist2 skips every other interface, so one interface costs a single small transfer. +func (o *goDarwinIO) readInterfacePackets() (int64, error) { + mib := [6]int32{unix.CTL_NET, unix.AF_ROUTE, 0, unix.AF_UNSPEC, unix.NET_RT_IFLIST2, int32(o.interfaceIndex)} + if o.gateRIB == nil { + o.gateRIB = make([]byte, goDarwinGateRIBSize) + } + var rib []byte + for { + length := uintptr(len(o.gateRIB)) + _, _, errno := unix.Syscall6(unix.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), uintptr(len(mib)), uintptr(unsafe.Pointer(&o.gateRIB[0])), uintptr(unsafe.Pointer(&length)), 0, 0) + if errno == unix.ENOMEM && len(o.gateRIB) < goDarwinGateRIBSizeMax { + o.gateRIB = make([]byte, 2*len(o.gateRIB)) + continue + } + if errno != 0 { + return 0, errno + } + rib = o.gateRIB[:length] + break + } + for len(rib) >= int(unsafe.Sizeof(unix.IfMsghdr2{})) { + message := (*unix.IfMsghdr2)(unsafe.Pointer(&rib[0])) + if message.Type == unix.RTM_IFINFO2 && int(message.Index) == o.interfaceIndex { + return int64(message.Data.Ipackets), nil + } + if message.Msglen == 0 { + break + } + rib = rib[message.Msglen:] + } + return 0, unix.ENOENT +} + +func (o *goDarwinIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + if o.gateBlocked.Load() { + poll := goDarwinGatePoll + if o.netif { + poll = max(goDarwinGateSample-(time.Since(o.epoch)-time.Duration(o.gateSampledTime.Load())), 0) + } + if timeout < 0 || timeout > poll { + timeout = poll + } + } + var timeoutSpec *unix.Timespec + if timeout >= 0 { + spec := unix.NsecToTimespec(timeout.Nanoseconds()) + timeoutSpec = &spec + } + var eventCount int + var err error + if timeout == 0 { + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_KEVENT, uintptr(o.kqueueFd), 0, 0, uintptr(unsafe.Pointer(&o.events[0])), uintptr(len(o.events)), uintptr(unsafe.Pointer(timeoutSpec))) + eventCount = int(count) + if errno != 0 { + err = errno + } + } else { + eventCount, err = unix.Kevent(o.kqueueFd, nil, o.events[:], timeoutSpec) + } + if o.gateBlocked.Load() { + o.gatePoll() + } + if err != nil { + if err == unix.EINTR { + return false, 0, nil + } + return false, 0, E.Cause(err, "go: kevent wait") + } + tunReadable := false + socketCount := 0 + for index := range eventCount { + event := &o.events[index] + if event.Filter == unix.EVFILT_USER { + continue + } + if int(event.Ident) == o.tunFd { + if event.Filter == unix.EVFILT_WRITE { + o.transmitWritable.Store(true) + continue + } + if event.Flags&unix.EV_ERROR != 0 && event.Data != 0 { + return false, 0, unix.Errno(event.Data) + } + if event.Flags&unix.EV_EOF != 0 { + return false, 0, unix.ECONNRESET + } + tunReadable = true + continue + } + if int(event.Ident) >= len(o.socketTokens) || socketCount == len(events) { + continue + } + events[socketCount] = goSocketEvent{ + token: o.socketTokens[event.Ident], + readable: event.Filter == unix.EVFILT_READ, + writable: event.Filter == unix.EVFILT_WRITE, + } + socketCount++ + } + return tunReadable, socketCount, nil +} + +func (o *goDarwinIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + if socket.fd >= len(o.socketTokens) { + grown := max(socket.fd+1, 2*len(o.socketTokens), 64) + o.socketTokens = append(o.socketTokens, make([]uint32, grown-len(o.socketTokens))...) + o.socketInterests = append(o.socketInterests, make([]uint8, grown-len(o.socketInterests))...) + } + o.socketTokens[socket.fd] = token + o.socketInterests[socket.fd] = 0 + return o.updateSocket(socket, interest) +} + +func (o *goDarwinIO) updateSocket(socket *goSocket, interest uint8) error { + current := o.socketInterests[socket.fd] + if current == interest { + return nil + } + changes := o.changes[:0] + if current&goInterestRead != interest&goInterestRead { + changes = append(changes, unix.Kevent_t{Ident: uint64(socket.fd), Filter: unix.EVFILT_READ, Flags: goKqueueFlags(interest&goInterestRead != 0)}) + } + if current&goInterestWrite != interest&goInterestWrite { + changes = append(changes, unix.Kevent_t{Ident: uint64(socket.fd), Filter: unix.EVFILT_WRITE, Flags: goKqueueFlags(interest&goInterestWrite != 0)}) + } + o.socketInterests[socket.fd] = interest + _, err := unix.Kevent(o.kqueueFd, changes, nil, nil) + if err != nil { + return E.Cause(err, "go: update socket interest") + } + return nil +} + +func goKqueueFlags(enable bool) uint16 { + if enable { + return unix.EV_ADD | unix.EV_ENABLE + } + return unix.EV_DELETE +} + +func (o *goDarwinIO) unregisterSocket(socket *goSocket) { + o.updateSocket(socket, 0) + o.socketTokens[socket.fd] = 0 +} + +func (o *goDarwinIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + if o.receiveBuffers == nil || o.readWaitOptions != options { + o.releaseReadBuffers() + if o.receiveBuffers == nil { + o.receiveBuffers = make([]*buf.Buffer, goReadBatch) + } + o.readWaitOptions = options + } + // recvmsg_x walks every submitted msghdr slot, whether or not a packet is pending for it. + count := min(o.receiveBatch, len(frames)) + for index := range count { + buffer := o.receiveBuffers[index] + if buffer == nil { + buffer = options.NewBufferSize(o.stack.mtu + PacketOffset) + o.receiveBuffers[index] = buffer + } + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + o.receiveIovecs[index] = rawfile.IovecFromBytes(buffer.FreeBytes()) + // Cannot clear only the length field. Older versions of the darwin kernel will check whether other data is empty. + // https://github.com/Darm64/XNU/blob/xnu-2782.40.9/bsd/kern/uipc_syscalls.c#L2026-L2048 + o.messageHeaders[index] = rawfile.MsgHdrX{} + o.messageHeaders[index].Msg.Iov = &o.receiveIovecs[index] + o.messageHeaders[index].Msg.Iovlen = 1 + } + received, errno := receiveMessageBatch(o.tunFd, o.messageHeaders[:count]) + if errno != 0 { + if errno == unix.EWOULDBLOCK || errno == unix.EINTR { + return 0, true, nil + } + return 0, false, E.Cause(errno, "go: recvmsg_x") + } + if received >= count { + o.receiveBatch = min(count*2, len(o.messageHeaders)) + } else if received*2 < count { + o.receiveBatch = max(count/2, goReceiveBatchMin) + } + frameCount := 0 + for index := range received { + dataLen := int(o.messageHeaders[index].DataLen) + if dataLen <= PacketOffset || dataLen > o.receiveBuffers[index].FreeLen() || o.messageHeaders[index].Msg.Flags&unix.MSG_TRUNC != 0 { + continue + } + buffer := o.receiveBuffers[index] + buffer.Reset() + buffer.Resize(options.FrontHeadroom+PacketOffset, dataLen-PacketOffset) + frames[frameCount] = goFrame{buffer: buffer} + frameCount++ + } + return frameCount, received == 0, nil +} + +func (o *goDarwinIO) releaseReadBuffers() { + clear(o.receiveIovecs) + clear(o.messageHeaders) + buf.ReleaseMulti(o.receiveBuffers) + clear(o.receiveBuffers) + o.transmitAccess.Lock() + clear(o.transmitIovecs) + if o.batchCount == 0 { + clear(o.batchIovecs) + clear(o.batchMessages) + o.batchSpill = nil + } + o.transmitAccess.Unlock() +} + +func receiveMessageBatch(fd int, headers []rawfile.MsgHdrX) (int, unix.Errno) { + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_RECVMSG_X, uintptr(fd), uintptr(unsafe.Pointer(&headers[0])), uintptr(len(headers)), unix.MSG_DONTWAIT, 0, 0) + return int(n), errno +} + +func (o *goDarwinIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + err := o.writePacket(frame) + if err == errGoTransmitBlocked { + o.droppedEngineFrames.record(o.stack.logger, "engine frames") + return errGoFrameDropped + } + return err +} + +func (o *goDarwinIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + frameLength := 0 + for _, segment := range frame { + frameLength += len(segment) + } + if frameLength > goDarwinBatchFrameLimit { + return o.writeUnbatchedLocked(frame) + } + if o.batchCount == goDarwinTransmitBatch { + o.flushLocked() + if o.batchCount == goDarwinTransmitBatch { + if o.batchStart == 0 { + return errGoTransmitBlocked + } + o.compactBatch() + } + } + slot := o.batchCount + iovecs := o.batchIovecs[slot*goDarwinBatchIovecs : slot*goDarwinBatchIovecs : (slot+1)*goDarwinBatchIovecs] + if header.IPVersion(frame[0]) == header.IPv4Version { + iovecs = append(iovecs, packetHeaderVec4) + } else { + iovecs = append(iovecs, packetHeaderVec6) + } + for index, segment := range frame { + if len(segment) == 0 { + continue + } + if index == 0 && len(frame) > 1 { + if len(segment) > goDarwinBatchHeader { + return o.writeUnbatchedLocked(frame) + } + headerCopy := o.batchHeaders[slot*goDarwinBatchHeader : slot*goDarwinBatchHeader+len(segment)] + copy(headerCopy, segment) + segment = headerCopy + } + if len(iovecs) == cap(iovecs) { + return o.writeUnbatchedLocked(frame) + } + iovecs = append(iovecs, rawfile.IovecFromBytes(segment)) + } + o.batchMessages[slot] = rawfile.MsgHdrX{} + o.batchMessages[slot].Msg.Iov = &iovecs[0] + o.batchMessages[slot].Msg.Iovlen = int32(len(iovecs)) + o.batchSpilled[slot] = false + o.batchCount++ + return nil +} + +func (o *goDarwinIO) writeUnbatchedLocked(frame [][]byte) error { + o.flushLocked() + if o.batchCount > o.batchStart { + return errGoTransmitBlocked + } + return o.writePacketLocked(frame) +} + +func (o *goDarwinIO) flush() { + o.transmitAccess.Lock() + o.flushLocked() + o.transmitAccess.Unlock() +} + +func (o *goDarwinIO) gateEstimate() int64 { + return o.gateInflightSample + (o.gateSubmitted - o.gateSampledAt) +} + +func (o *goDarwinIO) gateSample() (int64, bool, bool) { + if !o.netif { + return o.gateEstimate(), false, false + } + now := int64(time.Since(o.epoch)) + if now-o.gateSampledTime.Load() < int64(goDarwinGateSample) { + return o.gateEstimate(), false, false + } + o.gateSampledTime.Store(now) + consumed, err := o.readInterfacePackets() + if err != nil { + return o.gateEstimate(), false, true + } + progressed := consumed != o.gateLastConsumed + if progressed { + o.gateProbeEnd = 0 + } + o.gateLastConsumed = consumed + o.gateInflightSample = max(o.gateSubmitted-(consumed-o.gateConsumedBase), 0) + o.gateSampledAt = o.gateSubmitted + return o.gateInflightSample, progressed, true +} + +func (o *goDarwinIO) gateRoom(pending int) int { + if !o.netif { + return pending + } + estimate := o.gateEstimate() + if estimate+int64(pending) > goDarwinGateHighWater { + estimate, _, _ = o.gateSample() + } + room := goDarwinGateHighWater - estimate + room = max(room, min(o.gateProbeEnd-o.gateSubmitted, goDarwinGateLimit-1-estimate)) + return int(max(min(room, int64(pending)), 0)) +} + +// flushLocked submits the queued frames. sendmsg_x reports no count on failure. ENOBUFS comes from +// sendit_x failing to build the packets before anything is sent, so the batch is kept and retried. +// For the other errors ctl_send_list frees whatever follows the failing packet, so the call +// consumed every frame handed to it: the ones before the failure were delivered, the failing one +// is charged to the kernel (an ENOSPC leaks it), the rest are dropped and left to retransmission. +func (o *goDarwinIO) flushLocked() { + pending := o.batchCount - o.batchStart + if pending == 0 { + o.gateReleaseLocked() + return + } + room := o.gateRoom(pending) + for room > 0 { + messages := o.batchMessages[o.batchStart : o.batchStart+room] + n, errno := rawfile.NonBlockingSendMMsg(o.tunFd, messages) + if errno == unix.EINTR { + continue + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + break + } + if errno != 0 { + o.batchStart += room + o.gateSubmitted += int64(room) + o.recordWriteError(errno) + break + } + o.batchStart += n + o.gateSubmitted += int64(n) + room -= n + } + if o.batchStart == o.batchCount { + o.batchStart = 0 + o.batchCount = 0 + o.gateReleaseLocked() + return + } + o.gateBlockLocked() + o.spillHeld() +} + +func (o *goDarwinIO) gateBlockLocked() { + if !o.gateBlocked.Swap(true) { + o.wake() + } +} + +func (o *goDarwinIO) gateReleaseLocked() { + if !o.gateBlocked.Load() { + return + } + if o.gateRoom(1) == 0 { + return + } + o.gateBlocked.Store(false) + o.gateIdlePolls = 0 + o.transmitWritable.Store(true) + o.wake() +} + +// recordWriteError realigns the estimate after ENOSPC: the chain held more than the limit when +// the kernel refused, and the frames the kernel freed behind the refused one never reach the +// interface counter, so the counter baseline is moved to make the estimate read the limit now. +func (o *goDarwinIO) recordWriteError(errno unix.Errno) { + if errno == unix.ENOSPC { + o.leakedClusters.record(o.stack.logger, "frames into a full utun input queue, each leaking one kernel mbuf") + if !o.netif { + return + } + consumed, err := o.readInterfacePackets() + if err == nil { + o.gateLastConsumed = consumed + o.gateConsumedBase = consumed - (o.gateSubmitted - (goDarwinGateLimit + 1)) + } + o.gateSampledTime.Store(int64(time.Since(o.epoch))) + o.gateInflightSample = goDarwinGateLimit + 1 + o.gateSampledAt = o.gateSubmitted + o.gateProbeEnd = 0 + return + } + o.droppedBatchFrames.record(o.stack.logger, "batched frames") +} + +func (o *goDarwinIO) gatePoll() { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + if !o.gateBlocked.Load() { + return + } + _, progressed, sampled := o.gateSample() + if !sampled { + o.flushLocked() + return + } + if progressed { + o.gateIdlePolls = 0 + } else { + o.gateIdlePolls++ + } + if o.gateIdlePolls >= goDarwinGateProbeAfter { + o.gateProbeEnd = o.gateSubmitted + goDarwinGateProbe + o.gateIdlePolls = 0 + } + o.flushLocked() +} + +func (o *goDarwinIO) spillHeld() { + slotSize := o.stack.mtu + PacketOffset + if o.batchSpill == nil { + o.batchSpill = make([]byte, goDarwinTransmitBatch*slotSize) + } + for slot := o.batchStart; slot < o.batchCount; slot++ { + if o.batchSpilled[slot] { + continue + } + message := &o.batchMessages[slot] + iovecs := unsafe.Slice(message.Msg.Iov, int(message.Msg.Iovlen)) + spill := o.batchSpill[slot*slotSize : (slot+1)*slotSize] + length := 0 + for _, iovec := range iovecs[1:] { + length += copy(spill[length:], bytesFromIovec(iovec)) + } + iovecs[1] = rawfile.IovecFromBytes(spill[:length]) + message.Msg.Iovlen = 2 + o.batchSpilled[slot] = true + } +} + +func bytesFromIovec(iovec unix.Iovec) []byte { + return unsafe.Slice(iovec.Base, int(iovec.Len)) +} + +func (o *goDarwinIO) compactBatch() { + slotSize := o.stack.mtu + PacketOffset + for slot := o.batchStart; slot < o.batchCount; slot++ { + target := slot - o.batchStart + source := &o.batchMessages[slot] + sourceIovecs := unsafe.Slice(source.Msg.Iov, int(source.Msg.Iovlen)) + targetIovecs := o.batchIovecs[target*goDarwinBatchIovecs : target*goDarwinBatchIovecs+len(sourceIovecs)] + copy(targetIovecs, sourceIovecs) + for index := range targetIovecs { + base := uintptr(unsafe.Pointer(targetIovecs[index].Base)) + headerStart := uintptr(unsafe.Pointer(&o.batchHeaders[slot*goDarwinBatchHeader])) + if base >= headerStart && base < headerStart+goDarwinBatchHeader { + copy(o.batchHeaders[target*goDarwinBatchHeader:(target+1)*goDarwinBatchHeader], o.batchHeaders[slot*goDarwinBatchHeader:(slot+1)*goDarwinBatchHeader]) + targetIovecs[index].Base = &o.batchHeaders[target*goDarwinBatchHeader+int(base-headerStart)] + continue + } + if o.batchSpilled[slot] { + spillStart := uintptr(unsafe.Pointer(&o.batchSpill[slot*slotSize])) + if base >= spillStart && base < spillStart+uintptr(slotSize) { + copy(o.batchSpill[target*slotSize:(target+1)*slotSize], o.batchSpill[slot*slotSize:(slot+1)*slotSize]) + targetIovecs[index].Base = &o.batchSpill[target*slotSize+int(base-spillStart)] + } + } + } + o.batchMessages[target] = rawfile.MsgHdrX{} + o.batchMessages[target].Msg.Iov = &targetIovecs[0] + o.batchMessages[target].Msg.Iovlen = int32(len(targetIovecs)) + o.batchSpilled[target] = o.batchSpilled[slot] + } + o.batchCount -= o.batchStart + o.batchStart = 0 +} + +func (o *goDarwinIO) writePacket(frame [][]byte) error { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + return o.writePacketLocked(frame) +} + +func (o *goDarwinIO) writePacketLocked(frame [][]byte) error { + defer func() { clear(o.transmitIovecs) }() + if o.gateRoom(1) == 0 { + o.gateBlockLocked() + return errGoTransmitBlocked + } + headerVec := packetHeaderVec6 + if header.IPVersion(frame[0]) == header.IPv4Version { + headerVec = packetHeaderVec4 + } + iovecs := append(o.transmitIovecs[:0], headerVec) + for _, segment := range frame { + if len(segment) == 0 { + continue + } + iovecs = append(iovecs, rawfile.IovecFromBytes(segment)) + } + o.transmitIovecs = iovecs + for { + errno := rawfile.NonBlockingWriteIovec(o.tunFd, iovecs) + if errno == 0 { + o.gateSubmitted++ + return nil + } + if errno == unix.EINTR { + if o.closing.Load() { + return os.ErrClosed + } + continue + } + if errno == unix.ENOSPC { + o.gateSubmitted++ + o.recordWriteError(errno) + o.gateBlockLocked() + return errGoTransmitBlocked + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + if o.closing.Load() { + return os.ErrClosed + } + o.gateBlockLocked() + return errGoTransmitBlocked + } + return errno + } +} + +func (o *goDarwinIO) transmitPrefix() int { + return PacketOffset +} + +func (o *goDarwinIO) transmitChecksumOffload() bool { + return false +} + +func (o *goDarwinIO) transmitSegmentOffload() bool { + return false +} + +func (o *goDarwinIO) armTransmitWritable() (bool, error) { + if o.gateBlocked.Load() { + return true, nil + } + _, err := unix.Kevent(o.kqueueFd, o.transmitWritableEvent[:], nil, nil) + if err != nil { + return false, E.Cause(err, "go: arm EVFILT_WRITE") + } + return true, nil +} + +func (o *goDarwinIO) takeTransmitWritable() bool { + return o.transmitWritable.Swap(false) +} + +func (o *goDarwinIO) wake() { + _, _ = unix.Kevent(o.kqueueFd, o.wakeEvent[:], nil, nil) +} + +func (o *goDarwinIO) close() error { + o.closing.Store(true) + return unix.Close(o.kqueueFd) +} + +func (o *goDarwinIO) writePacketBatch(frames []goUDPFrame) error { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + defer func() { + clear(o.packetMessages[:]) + clear(o.packetIovecs[:]) + }() + o.flushLocked() + limit := len(frames) + for index := 0; index < len(frames); { + room := o.gateRoom(min(len(frames)-index, limit)) + if room == 0 || o.batchCount > o.batchStart { + o.gateBlockLocked() + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } + frame := &frames[index] + if frame.length+len(frame.payload) > goDarwinBatchFrameLimit { + segments := [2][]byte{frame.header[:frame.length], frame.payload} + err := o.writePacketLocked(segments[:]) + if err == errGoTransmitBlocked { + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } else if err != nil { + return err + } + index++ + continue + } + count := 0 + for count < room { + next := &frames[index+count] + if next.length+len(next.payload) > goDarwinBatchFrameLimit { + break + } + iovecs := o.packetIovecs[count*3 : (count+1)*3] + iovecs[0] = packetHeaderVec6 + if next.length == header.IPv4MinimumSize+header.UDPMinimumSize { + iovecs[0] = packetHeaderVec4 + } + iovecs[1] = rawfile.IovecFromBytes(next.header[:next.length]) + iovecCount := 2 + if len(next.payload) > 0 { + iovecs[2] = rawfile.IovecFromBytes(next.payload) + iovecCount++ + } + o.packetMessages[count] = rawfile.MsgHdrX{} + o.packetMessages[count].Msg.Iov = &iovecs[0] + o.packetMessages[count].Msg.Iovlen = int32(iovecCount) + count++ + } + written, errno := rawfile.NonBlockingSendMMsg(o.tunFd, o.packetMessages[:count]) + if errno == unix.EINTR { + continue + } + if (errno == unix.EMSGSIZE || errno == unix.ENOBUFS) && count > 1 { + limit = (count + 1) / 2 + continue + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + o.gateBlockLocked() + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } + if errno != 0 { + o.gateSubmitted += int64(count) + o.recordWriteError(errno) + return errGoFrameDropped + } + if written == 0 { + return unix.EIO + } + o.gateSubmitted += int64(written) + index += written + } + return nil +} diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go new file mode 100644 index 00000000..1a1d276c --- /dev/null +++ b/stack_go_io_linux.go @@ -0,0 +1,568 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "os" + "runtime" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" +) + +const goEngineInlineTransmit = false + +var goEmptyVirtioHeader [virtioNetHdrLen]byte + +type goLinuxDevice struct { + stack *Go + vnetHeader bool + offloadProbePending atomic.Bool + transmitOffload atomic.Bool + udpTransmitDisabled atomic.Bool + droppedEngineFrames goDropCounter + droppedDataFrames goDropCounter + closing atomic.Bool +} + +type goLinuxIO struct { + device *goLinuxDevice + tun *NativeTun + tunFd int + ownsFd bool + epollFd int + eventFd int + receiveBuffers []*buf.Buffer + readWaitOptions N.ReadWaitOptions + events [goSocketEventBatch + 2]unix.EpollEvent +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + nativeTun, isNative := stack.tun.(*NativeTun) + if !isNative { + return nil, E.New("go: unsupported TUN implementation") + } + err := nativeTun.detachRuntimePoller() + if err != nil { + return nil, E.Cause(err, "go: detach tun from runtime poller") + } + device := &goLinuxDevice{stack: stack} + vnetHeader, err := nativeTun.vnetHeaderEnabled() + if err != nil { + stack.logger.Warn(E.Cause(err, "go: check IFF_VNET_HDR")) + vnetHeader = false + } + device.vnetHeader = vnetHeader + if vnetHeader { + offloadErr := setUDPOffload(nativeTun.rawFileDescriptor()) + if offloadErr != nil { + offloadErr = setTCPOffload(nativeTun.rawFileDescriptor()) + } + if offloadErr != nil { + stack.logger.Warn(E.Cause(offloadErr, "go: set TSO offload")) + } else { + device.offloadProbePending.Store(true) + } + } + queueCount := 1 + if nativeTun.multiQueue { + queueCount = runtime.GOMAXPROCS(0) + } else if nativeTun.options.MultiQueue { + stack.logger.Warn("go: multi-queue requested but the tun device is single-queue") + } + queues := make([]goPlatformIO, queueCount) + queues[0] = &goLinuxIO{device: device, tun: nativeTun, tunFd: nativeTun.rawFileDescriptor()} + for index := 1; index < len(queues); index++ { + queues[index] = &goLinuxIO{device: device, tun: nativeTun, tunFd: -1} + } + return queues, nil +} + +func (o *goLinuxIO) start() error { + if o.tunFd < 0 { + queueFd, err := o.tun.openQueue() + if err != nil { + o.device.stack.logger.Warn(E.Cause(err, "go: attach tun queue")) + return errGoQueueUnavailable + } + o.tunFd = queueFd + o.ownsFd = true + } + err := o.startPoller() + if err != nil && o.ownsFd { + err = E.Errors(err, unix.Close(o.tunFd)) + } + return err +} + +func (o *goLinuxIO) startPoller() error { + epollFd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC) + if err != nil { + return E.Cause(err, "go: create epoll") + } + eventFd, err := unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) + if err != nil { + unix.Close(epollFd) + return E.Cause(err, "go: create eventfd") + } + err = unix.EpollCtl(epollFd, unix.EPOLL_CTL_ADD, o.tunFd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: goEpollDataTun}) + if err == nil { + err = unix.EpollCtl(epollFd, unix.EPOLL_CTL_ADD, eventFd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: goEpollDataWake}) + } + if err != nil { + return E.Errors(E.Cause(err, "go: register epoll events"), unix.Close(epollFd), unix.Close(eventFd)) + } + o.epollFd = epollFd + o.eventFd = eventFd + return nil +} + +func (o *goLinuxIO) probeTransmitOffload() error { + probeAddr := netip.AddrFrom4([4]byte{127, 0, 0, 1}) + fingerprint := []byte("sing-tun-probe-tun-gro") + segmentSize := len(fingerprint) + const ipHeaderLength = 20 + const tcpHeaderLength = 20 + totalLength := ipHeaderLength + tcpHeaderLength + 2*segmentSize + packet := make([]byte, totalLength) + probeIPv4 := header.IPv4(packet) + probeIPv4.Encode(&header.IPv4Fields{ + SrcAddr: probeAddr, + DstAddr: probeAddr, + Protocol: unix.IPPROTO_TCP, + TTL: 0, + TotalLength: uint16(totalLength), + }) + probeTCP := header.TCP(packet[ipHeaderLength:]) + probeTCP.Encode(&header.TCPFields{ + SrcPort: 0, + DstPort: 0, + SeqNum: 1, + AckNum: 1, + DataOffset: tcpHeaderLength, + Flags: header.TCPFlagAck, + WindowSize: 3000, + }) + copy(packet[ipHeaderLength+tcpHeaderLength:], fingerprint) + copy(packet[ipHeaderLength+tcpHeaderLength+segmentSize:], fingerprint) + probeIPv4.SetChecksum(^probeIPv4.CalculateChecksum()) + pseudoSum := header.PseudoHeaderChecksum(unix.IPPROTO_TCP, probeIPv4.SourceAddressSlice(), probeIPv4.DestinationAddressSlice(), uint16(tcpHeaderLength+2*segmentSize)) + probeTCP.SetChecksum(pseudoSum) + errno := o.transmitFrame([][]byte{packet}, ForwardFrameMeta{ + needsChecksum: true, + checksumStart: ipHeaderLength, + checksumOffset: header.TCPChecksumOffset, + gsoType: unix.VIRTIO_NET_HDR_GSO_TCPV4, + gsoSize: uint16(segmentSize), + }) + if errno != 0 { + return E.Cause(errno, "go: tso probe write") + } + return nil +} + +const ( + goEpollDataTun int32 = -1 + goEpollDataWake int32 = -2 +) + +func (o *goLinuxIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + milliseconds := -1 + if timeout >= 0 { + milliseconds = int((timeout + time.Millisecond - 1) / time.Millisecond) + } + var eventCount int + var err error + if timeout == 0 { + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_EPOLL_PWAIT, uintptr(o.epollFd), uintptr(unsafe.Pointer(&o.events[0])), uintptr(len(o.events)), 0, 0, 0) + eventCount = int(count) + if errno != 0 { + err = errno + } + } else { + eventCount, err = unix.EpollWait(o.epollFd, o.events[:], milliseconds) + } + if err != nil { + if err == unix.EINTR { + return false, 0, nil + } + return false, 0, E.Cause(err, "go: epoll wait") + } + tunReadable := false + socketCount := 0 + for index := range eventCount { + event := &o.events[index] + switch event.Fd { + case goEpollDataWake: + var value [8]byte + _, _ = unix.Read(o.eventFd, value[:]) + case goEpollDataTun: + if event.Events&(unix.EPOLLERR|unix.EPOLLHUP) != 0 { + return false, 0, unix.ECONNRESET + } + tunReadable = true + default: + if socketCount == len(events) { + continue + } + events[socketCount] = goSocketEvent{ + token: uint32(event.Fd), + readable: event.Events&(unix.EPOLLIN|unix.EPOLLERR|unix.EPOLLHUP|unix.EPOLLRDHUP) != 0, + writable: event.Events&unix.EPOLLOUT != 0, + } + socketCount++ + } + } + return tunReadable, socketCount, nil +} + +func (o *goLinuxIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + socket.token = token + err := unix.EpollCtl(o.epollFd, unix.EPOLL_CTL_ADD, socket.fd, &unix.EpollEvent{Events: goEpollEvents(interest), Fd: int32(token)}) + if err != nil { + return E.Cause(err, "go: register socket") + } + socket.registered = true + return nil +} + +func (o *goLinuxIO) updateSocket(socket *goSocket, interest uint8) error { + operation := unix.EPOLL_CTL_MOD + if interest == 0 { + operation = unix.EPOLL_CTL_DEL + } else if !socket.registered { + operation = unix.EPOLL_CTL_ADD + } + err := unix.EpollCtl(o.epollFd, operation, socket.fd, &unix.EpollEvent{Events: goEpollEvents(interest), Fd: int32(socket.token)}) + if err != nil { + return E.Cause(err, "go: update socket interest") + } + socket.registered = interest != 0 + return nil +} + +func (o *goLinuxIO) unregisterSocket(socket *goSocket) { + if !socket.registered { + return + } + unix.EpollCtl(o.epollFd, unix.EPOLL_CTL_DEL, socket.fd, nil) + socket.registered = false +} + +func goEpollEvents(interest uint8) uint32 { + var events uint32 + if interest&goInterestRead != 0 { + events |= unix.EPOLLIN | unix.EPOLLRDHUP + } + if interest&goInterestWrite != 0 { + events |= unix.EPOLLOUT + } + return events +} + +func (o *goLinuxIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + if o.receiveBuffers == nil || o.readWaitOptions != options { + o.releaseReadBuffers() + if o.receiveBuffers == nil { + o.receiveBuffers = make([]*buf.Buffer, goPacketBatchSize) + } + o.readWaitOptions = options + } + count := 0 + for count < min(len(frames), len(o.receiveBuffers)) { + buffer := o.receiveBuffers[count] + if buffer == nil { + // Linux hands pre-segmentation TSO aggregates to the TUN fd even with IFF_VNET_HDR off + // (observed on 6.x kernels). + buffer = options.NewBufferSize(virtioNetHdrLen + gsoMaxSize) + o.receiveBuffers[count] = buffer + } + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + scratch := buffer.FreeBytes() + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_READ, uintptr(o.tunFd), uintptr(unsafe.Pointer(&scratch[0])), uintptr(len(scratch))) + if errno != 0 { + if errno == unix.EAGAIN { + return count, true, nil + } + if errno == unix.EINTR { + continue + } + return 0, false, E.Cause(errno, "go: read tun") + } + if n == 0 { + return 0, false, os.ErrClosed + } + // The kernel rejects writes with EIO while the device is not up (tun_get_user). + if o.device.offloadProbePending.CompareAndSwap(true, false) { + probeErr := o.probeTransmitOffload() + if probeErr != nil { + o.device.stack.logger.Warn(E.Cause(probeErr, "go: TSO write probe")) + } else { + o.device.transmitOffload.Store(true) + } + } + packet := scratch[:n] + if !o.device.vnetHeader { + buffer.Truncate(int(n)) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer} + count++ + continue + } + payload, virtioOptions, parseErr := parseVirtioRead(packet) + if parseErr != nil { + continue + } + var meta ForwardFrameMeta + switch virtioOptions.GSOType { + case GSONone: + case GSOTCPv4: + meta.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 + meta.gsoSize = virtioOptions.GSOSize + case GSOTCPv6: + meta.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6 + meta.gsoSize = virtioOptions.GSOSize + case GSOUDPL4: + meta.gsoType = goUDPGSOType + meta.gsoSize = virtioOptions.GSOSize + } + meta.needsChecksum = virtioOptions.NeedsCsum + meta.checksumStart = virtioOptions.CsumStart + meta.checksumOffset = virtioOptions.CsumOffset + buffer.Resize(options.FrontHeadroom+len(packet)-len(payload), len(payload)) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer, meta: meta} + count++ + } + return count, false, nil +} + +func (o *goLinuxIO) releaseReadBuffers() { + buf.ReleaseMulti(o.receiveBuffers) + clear(o.receiveBuffers) +} + +func (o *goLinuxIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + for { + errno := o.transmitFrame(frame, meta) + switch errno { + case 0: + return nil + case unix.EINTR: + if o.device.closing.Load() { + return os.ErrClosed + } + case unix.EAGAIN: + o.device.droppedEngineFrames.record(o.device.stack.logger, "engine frames") + return errGoFrameDropped + default: + return E.Cause(errno, "go: write tun") + } + } +} + +func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { + delay := goTransmitBackoffMin + waited := time.Duration(0) + for { + errno := o.transmitFrame(frame, meta) + switch errno { + case 0: + return nil + case unix.EINTR: + if o.device.closing.Load() { + return os.ErrClosed + } + case unix.EAGAIN: + // tun chardev writes complete into netif_rx, which drops on backlog overflow + // instead of reporting EAGAIN, and tun_chr_poll reports the fd always writable. + if o.device.closing.Load() { + return os.ErrClosed + } + if waited >= goTransmitBackoffBudget { + o.device.droppedDataFrames.record(o.device.stack.logger, "data frames") + return errGoFrameDropped + } + time.Sleep(delay) + if o.device.closing.Load() { + return os.ErrClosed + } + waited += delay + delay = min(2*delay, goTransmitBackoffMax) + default: + return E.Cause(errno, "go: write tun") + } + } +} + +func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Errno { + var headerStorage [virtioNetHdrLen]byte + var iovecStorage [goPacketBatchSize + 2]unix.Iovec + iovecs := iovecStorage[:0] + if o.device.vnetHeader { + prefix := goEmptyVirtioHeader[:] + if meta != (ForwardFrameMeta{}) { + virtioHeader := virtioNetHdr{gsoType: meta.gsoType, gsoSize: meta.gsoSize} + if meta.needsChecksum { + virtioHeader.flags = unix.VIRTIO_NET_HDR_F_NEEDS_CSUM + virtioHeader.csumStart = meta.checksumStart + virtioHeader.csumOffset = meta.checksumOffset + } + if meta.gsoType == goUDPGSOType { + virtioHeader.hdrLen = meta.checksumStart + header.UDPMinimumSize + } else if meta.gsoType != unix.VIRTIO_NET_HDR_GSO_NONE { + virtioHeader.hdrLen = meta.checksumStart + uint16(header.TCP(frame[0][meta.checksumStart:]).DataOffset()) + } + common.Must(virtioHeader.encode(headerStorage[:])) + prefix = headerStorage[:] + } + vector := unix.Iovec{Base: &prefix[0]} + vector.SetLen(len(prefix)) + iovecs = append(iovecs, vector) + } + for _, segment := range frame { + if len(segment) == 0 { + continue + } + vector := unix.Iovec{Base: &segment[0]} + vector.SetLen(len(segment)) + iovecs = append(iovecs, vector) + } + //nolint:staticcheck + _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) + return errno +} + +func (o *goLinuxIO) transmitPrefix() int { + if o.device.vnetHeader { + return virtioNetHdrLen + } + return 0 +} + +func (o *goLinuxIO) transmitChecksumOffload() bool { + return o.device.vnetHeader +} + +func (o *goLinuxIO) transmitSegmentOffload() bool { + return o.device.transmitOffload.Load() +} + +func (o *goLinuxIO) armTransmitWritable() (bool, error) { + return false, nil +} + +func (o *goLinuxIO) takeTransmitWritable() bool { + return false +} + +func (o *goLinuxIO) wake() { + var value [8]byte + binary.NativeEndian.PutUint64(value[:], 1) + _, _ = unix.Write(o.eventFd, value[:]) +} + +func (o *goLinuxIO) close() error { + o.device.closing.Store(true) + err := E.Errors(unix.Close(o.epollFd), unix.Close(o.eventFd)) + if o.ownsFd { + err = E.Errors(err, unix.Close(o.tunFd)) + } + return err +} + +func (o *goLinuxIO) flush() { +} + +func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { + var writeError error + var segments [goPacketBatchSize + 1][]byte + for index := 0; index < len(frames); { + frame := &frames[index] + end := index + 1 + payloadLength := len(frame.payload) + if o.device.vnetHeader && !o.device.udpTransmitDisabled.Load() && payloadLength > 0 { + for end < len(frames) { + next := &frames[end] + if next.length != frame.length || len(next.payload) == 0 || len(next.payload) > len(frame.payload) || frame.length+payloadLength+len(next.payload) > 65535 { + break + } + udpOffset := frame.length - header.UDPMinimumSize + if binary.BigEndian.Uint32(next.header[udpOffset:]) != binary.BigEndian.Uint32(frame.header[udpOffset:]) { + break + } + if frame.length == header.IPv4MinimumSize+header.UDPMinimumSize { + if [8]byte(next.header[12:20]) != [8]byte(frame.header[12:20]) { + break + } + } else if [32]byte(next.header[8:40]) != [32]byte(frame.header[8:40]) { + break + } + payloadLength += len(next.payload) + end++ + if len(next.payload) < len(frame.payload) { + break + } + } + } + packetHeader := frame.header + meta := frame.meta + if end-index > 1 { + udpLength := uint16(header.UDPMinimumSize + payloadLength) + var sourceAddress, destinationAddress []byte + if frame.length == header.IPv4MinimumSize+header.UDPMinimumSize { + ipHdr := header.IPv4(packetHeader[:header.IPv4MinimumSize]) + ipHdr.SetTotalLength(uint16(frame.length + payloadLength)) + ipHdr.SetChecksum(0) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } else { + ipHdr := header.IPv6(packetHeader[:header.IPv6MinimumSize]) + ipHdr.SetPayloadLength(udpLength) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } + udpHdr := header.UDP(packetHeader[frame.length-header.UDPMinimumSize : frame.length]) + udpHdr.SetLength(udpLength) + udpHdr.SetChecksum(header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpLength)) + meta.gsoType = goUDPGSOType + meta.gsoSize = uint16(len(frame.payload)) + } + segments[0] = packetHeader[:frame.length] + for packetIndex := index; packetIndex < end; packetIndex++ { + segments[packetIndex-index+1] = frames[packetIndex].payload + } + errno := o.transmitFrame(segments[:end-index+1], meta) + if errno == unix.EINTR { + continue + } + if errno != 0 && end-index > 1 { + switch errno { + case unix.EINVAL, unix.EIO, unix.EOPNOTSUPP: + o.device.udpTransmitDisabled.Store(true) + continue + } + } + if errno == unix.EAGAIN { + o.device.droppedEngineFrames.record(o.device.stack.logger, "UDP frames") + writeError = E.Errors(writeError, errGoFrameDropped) + } else if errno != 0 { + writeError = E.Errors(writeError, E.Cause(errno, "go: write UDP batch")) + } + index = end + } + return writeError +} diff --git a/stack_go_io_other.go b/stack_go_io_other.go new file mode 100644 index 00000000..146fffc3 --- /dev/null +++ b/stack_go_io_other.go @@ -0,0 +1,76 @@ +//go:build !(linux || windows || darwin) + +package tun + +import ( + "net/netip" + "syscall" + + E "github.com/sagernet/sing/common/exceptions" +) + +const goEngineInlineTransmit = false + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return nil, E.New("go: unsupported platform") +} + +func goFatalReadError(err error) bool { + return true +} + +const goSpliceDuplicatesSocket = false + +type goIOVector struct{} + +type goSocket struct{} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + return goSocket{}, E.New("go: splice not supported") +} + +func (s *goSocket) close() { +} + +func (s *goSocket) readVector(iovecs []goIOVector) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) writeVector(iovecs []goIOVector) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + return syscall.ENOSYS +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + return 0, netip.AddrPort{}, syscall.ENOSYS +} + +func (s *goSocket) shutdownWrite() { +} + +func (s *goSocket) family() (uint8, error) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + return netip.AddrPort{}, false +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + return iovecs[:0] +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK || errno == syscall.EINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == syscall.EMSGSIZE || errno == syscall.EAFNOSUPPORT || errno == syscall.ENOBUFS +} diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go new file mode 100644 index 00000000..5dc8472d --- /dev/null +++ b/stack_go_io_windows.go @@ -0,0 +1,431 @@ +package tun + +import ( + "os" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/internal/afd" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/windows" +) + +const goEngineInlineTransmit = false + +const goSessionRingCapacity = 0x800000 + +const ( + goCompletionKeyTun uintptr = iota + 1 + goCompletionKeyWake + goCompletionKeySocket +) + +const goAFDReadEvents = afd.POLL_RECEIVE | afd.POLL_DISCONNECT | afd.POLL_ABORT | afd.POLL_LOCAL_CLOSE | afd.POLL_CONNECT_FAIL + +type goWindowsIO struct { + stack *Go + tun *NativeTun + iocp windows.Handle + afd *afd.Device + waitPacket *afd.WaitCompletionPacket + waitArmed bool + bridgeArm windows.Handle + bridgeClose windows.Handle + bridgeDone chan struct{} + entries map[*goAFDEntry]struct{} + completions [goSocketEventBatch + 2]afd.OverlappedEntry + receiveBuffers []*buf.Buffer + readWaitOptions N.ReadWaitOptions + // wintun's read-wait event is auto-reset and only set by the driver when it appends to + // the ring; WintunReceivePacket neither re-signals nor resets it (wintun api/session.c: + // CreateEventW(&SecurityAttributes, FALSE, FALSE, NULL)). + ringDrained bool + droppedEngineFrames goDropCounter + closing atomic.Bool + droppedDataFrames goDropCounter +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goWindowsIO{stack: stack}}, nil +} + +func (o *goWindowsIO) start() error { + nativeTun, isNative := o.stack.tun.(*NativeTun) + if !isNative { + return E.New("go: unsupported TUN implementation") + } + o.tun = nativeTun + err := nativeTun.resizeSessionRing(goSessionRingCapacity) + if err != nil { + return E.Cause(err, "go: resize wintun session ring") + } + iocp, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 1) + if err != nil { + return E.Cause(err, "go: create completion port") + } + o.iocp = iocp + device, err := afd.Open(iocp, "sing-tun") + if err != nil { + windows.CloseHandle(iocp) + return E.Cause(err, "go: open afd device") + } + o.afd = device + if afd.WaitCompletionPacketSupported() { + waitPacket, packetErr := afd.NewWaitCompletionPacket() + if packetErr != nil { + device.Close() + windows.CloseHandle(iocp) + return E.Cause(packetErr, "go: create wait completion packet") + } + o.waitPacket = waitPacket + } else { + armEvent, eventErr := windows.CreateEvent(nil, 0, 0, nil) + if eventErr != nil { + device.Close() + windows.CloseHandle(iocp) + return E.Cause(eventErr, "go: create read wait event") + } + closeEvent, eventErr := windows.CreateEvent(nil, 1, 0, nil) + if eventErr != nil { + windows.CloseHandle(armEvent) + device.Close() + windows.CloseHandle(iocp) + return E.Cause(eventErr, "go: create read wait close event") + } + o.bridgeArm = armEvent + o.bridgeClose = closeEvent + o.bridgeDone = make(chan struct{}) + go o.bridgeReadWait() + } + o.entries = make(map[*goAFDEntry]struct{}) + o.ringDrained = true + return nil +} + +func (o *goWindowsIO) bridgeReadWait() { + defer close(o.bridgeDone) + armHandles := []windows.Handle{o.bridgeArm, o.bridgeClose} + readHandles := []windows.Handle{o.tun.readWaitHandle(), o.bridgeClose} + for { + signaled, err := windows.WaitForMultipleObjects(armHandles, false, windows.INFINITE) + if err != nil || signaled != windows.WAIT_OBJECT_0 { + return + } + signaled, err = windows.WaitForMultipleObjects(readHandles, false, windows.INFINITE) + if err != nil || signaled != windows.WAIT_OBJECT_0 { + return + } + windows.PostQueuedCompletionStatus(o.iocp, 0, goCompletionKeyTun, nil) + } +} + +func (o *goWindowsIO) armReadWait() (bool, error) { + if o.waitArmed { + return false, nil + } + o.waitArmed = true + if o.waitPacket == nil { + windows.SetEvent(o.bridgeArm) + return false, nil + } + alreadySignaled, err := o.waitPacket.Associate(o.iocp, o.tun.readWaitHandle(), goCompletionKeyTun) + if err != nil { + o.waitArmed = false + return false, E.Cause(err, "go: associate wintun read event") + } + if alreadySignaled { + o.waitArmed = false + return true, nil + } + return false, nil +} + +func (o *goWindowsIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + tunReadable := !o.ringDrained + if o.ringDrained { + signaled, err := o.armReadWait() + if err != nil { + return false, 0, err + } + if signaled { + tunReadable = true + o.waitPacket.Cancel() + } + } + var waitMillis uint32 + if tunReadable || timeout == 0 { + waitMillis = 0 + } else if timeout < 0 { + waitMillis = windows.INFINITE + } else { + waitMillis = uint32((timeout + time.Millisecond - 1) / time.Millisecond) + } + var removed uint32 + err := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) + if err != nil { + if err == windows.WAIT_TIMEOUT { + return tunReadable, 0, nil + } + return false, 0, E.Cause(err, "go: wait for completion") + } + socketCount := 0 + for index := range removed { + completion := &o.completions[index] + switch completion.CompletionKey { + case goCompletionKeyTun: + o.waitArmed = false + tunReadable = true + case goCompletionKeyWake: + default: + entry := (*goAFDEntry)(unsafe.Pointer(completion.Overlapped)) + entry.armed = false + if entry.cancelled { + o.releaseEntry(entry) + continue + } + var pollEvents uint32 + if entry.pollInfo.NumberOfHandles > 0 { + pollEvents = entry.pollInfo.Handles[0].Events + } + if uint32(entry.ioStatusBlock.Status) != afd.STATUS_CANCELLED && socketCount < len(events) { + readable := pollEvents&goAFDReadEvents != 0 || entry.ioStatusBlock.Status != 0 + writable := pollEvents&afd.POLL_SEND != 0 + if readable || writable { + events[socketCount] = goSocketEvent{token: entry.token, readable: readable, writable: writable} + socketCount++ + } + } + if entry.interest != 0 { + armErr := o.armEntry(entry) + if armErr != nil && socketCount < len(events) { + events[socketCount] = goSocketEvent{token: entry.token, readable: true} + socketCount++ + } + } + } + } + return tunReadable, socketCount, nil +} + +func (o *goWindowsIO) armEntry(entry *goAFDEntry) error { + var pollEvents uint32 + if entry.interest&goInterestRead != 0 { + pollEvents |= goAFDReadEvents + } + if entry.interest&goInterestWrite != 0 { + pollEvents |= afd.POLL_SEND + } + err := o.afd.Poll(entry.baseHandle, pollEvents, &entry.ioStatusBlock, &entry.pollInfo) + if err != nil { + return err + } + entry.armed = true + return nil +} + +func (o *goWindowsIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + baseHandle, err := afd.BaseSocket(socket.handle) + if err != nil { + return E.Cause(err, "go: query base socket") + } + entry := &goAFDEntry{baseHandle: baseHandle, token: token, interest: interest} + entry.pinner.Pin(entry) + if o.entries == nil { + o.entries = make(map[*goAFDEntry]struct{}) + } + o.entries[entry] = struct{}{} + socket.entry = entry + if interest == 0 { + return nil + } + err = o.armEntry(entry) + if err != nil { + o.releaseEntry(entry) + socket.entry = nil + return E.Cause(err, "go: poll socket") + } + return nil +} + +func (o *goWindowsIO) releaseEntry(entry *goAFDEntry) { + delete(o.entries, entry) + entry.pinner.Unpin() +} + +func (o *goWindowsIO) updateSocket(socket *goSocket, interest uint8) error { + entry := socket.entry + if entry == nil || entry.interest == interest { + return nil + } + entry.interest = interest + if entry.armed { + return o.afd.Cancel(&entry.ioStatusBlock) + } + if interest == 0 { + return nil + } + err := o.armEntry(entry) + if err != nil { + return E.Cause(err, "go: poll socket") + } + return nil +} + +func (o *goWindowsIO) unregisterSocket(socket *goSocket) { + entry := socket.entry + if entry == nil { + return + } + socket.entry = nil + entry.interest = 0 + entry.cancelled = true + if !entry.armed { + o.releaseEntry(entry) + return + } + o.afd.Cancel(&entry.ioStatusBlock) +} + +func (o *goWindowsIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + if o.receiveBuffers == nil || o.readWaitOptions != options { + o.releaseReadBuffers() + if o.receiveBuffers == nil { + o.receiveBuffers = make([]*buf.Buffer, goReadBatch) + } + o.readWaitOptions = options + } + limit := min(len(frames), len(o.receiveBuffers)) + count := 0 + for count < limit { + buffer := o.receiveBuffers[count] + if buffer == nil { + buffer = options.NewBufferSize(o.stack.mtu) + o.receiveBuffers[count] = buffer + } + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + n, err := o.tun.receiveInto(buffer.FreeBytes()) + if err != nil { + return 0, false, err + } + if n == 0 { + o.ringDrained = true + return count, true, nil + } + buffer.Truncate(n) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer} + count++ + } + o.ringDrained = false + return count, false, nil +} + +func goFatalReadError(err error) bool { + return true +} + +func (o *goWindowsIO) releaseReadBuffers() { + buf.ReleaseMulti(o.receiveBuffers) + clear(o.receiveBuffers) +} + +func (o *goWindowsIO) writePacketBatch(frames []goUDPFrame) error { + var writeError error + var segments [2][]byte + for index := range frames { + frame := &frames[index] + segments[0] = frame.header[:frame.length] + segments[1] = frame.payload + writeError = E.Errors(writeError, o.writeFrame(segments[:], frame.meta)) + } + return writeError +} + +func (o *goWindowsIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + err := o.transmitFrame(frame) + if err == windows.ERROR_BUFFER_OVERFLOW { + o.droppedEngineFrames.record(o.stack.logger, "engine frames") + return errGoFrameDropped + } + return err +} + +func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { + backoff := goTransmitBackoffMin + waited := time.Duration(0) + for { + err := o.transmitFrame(frame) + if err != windows.ERROR_BUFFER_OVERFLOW { + return err + } + if o.closing.Load() { + return os.ErrClosed + } + if waited >= goTransmitBackoffBudget { + o.droppedDataFrames.record(o.stack.logger, "data frames") + return errGoFrameDropped + } + time.Sleep(backoff) + waited += backoff + backoff = min(backoff*2, goTransmitBackoffMax) + } +} + +func (o *goWindowsIO) transmitFrame(frame [][]byte) error { + err := o.tun.transmitGather(frame) + if err == nil || err == windows.ERROR_BUFFER_OVERFLOW || err == os.ErrClosed { + return err + } + return E.Cause(err, "go: wintun send") +} + +func (o *goWindowsIO) transmitPrefix() int { + return 0 +} + +func (o *goWindowsIO) transmitChecksumOffload() bool { + return false +} + +func (o *goWindowsIO) transmitSegmentOffload() bool { + return false +} + +func (o *goWindowsIO) armTransmitWritable() (bool, error) { + return false, nil +} + +func (o *goWindowsIO) takeTransmitWritable() bool { + return false +} + +func (o *goWindowsIO) wake() { + _ = windows.PostQueuedCompletionStatus(o.iocp, 0, goCompletionKeyWake, nil) +} + +func (o *goWindowsIO) close() error { + o.closing.Store(true) + var err error + if o.waitPacket != nil { + err = E.Errors(o.waitPacket.Cancel(), o.waitPacket.Close()) + } else { + windows.SetEvent(o.bridgeClose) + <-o.bridgeDone + err = E.Errors(windows.CloseHandle(o.bridgeArm), windows.CloseHandle(o.bridgeClose)) + } + err = E.Errors(err, o.afd.Close(), windows.CloseHandle(o.iocp)) + for entry := range o.entries { + entry.pinner.Unpin() + } + clear(o.entries) + return err +} + +func (o *goWindowsIO) flush() { +} diff --git a/stack_go_profile_default.go b/stack_go_profile_default.go new file mode 100644 index 00000000..ca476e3d --- /dev/null +++ b/stack_go_profile_default.go @@ -0,0 +1,17 @@ +//go:build !ios + +package tun + +const ( + goReceiveCapacityBase = 64 << 10 + goReceiveCapacityMax = 4 << 20 + goTransmitCapacityMax = 2 << 20 + goEngineBurstBytes = 1 << 20 + goFlowCapacity = 16384 + goReceiveBatchMin = 8 + goReadBatch = 64 + goDescriptorRingCapacity = 2048 + goReassemblyEntries = 8 + goSlabPoolLowWater = 128 + goDescriptorPoolLowWater = 64 +) diff --git a/stack_go_profile_ios.go b/stack_go_profile_ios.go new file mode 100644 index 00000000..e6732bac --- /dev/null +++ b/stack_go_profile_ios.go @@ -0,0 +1,15 @@ +package tun + +const ( + goReceiveCapacityBase = 32 << 10 + goReceiveCapacityMax = 256 << 10 + goTransmitCapacityMax = 512 << 10 + goEngineBurstBytes = 256 << 10 + goFlowCapacity = 1024 + goReceiveBatchMin = 4 + goReadBatch = 32 + goDescriptorRingCapacity = 512 + goReassemblyEntries = 4 + goSlabPoolLowWater = 8 + goDescriptorPoolLowWater = 16 +) diff --git a/stack_go_ring.go b/stack_go_ring.go new file mode 100644 index 00000000..769ccf03 --- /dev/null +++ b/stack_go_ring.go @@ -0,0 +1,768 @@ +package tun + +import ( + "slices" + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common/buf" +) + +const ( + goSlabSize = 32 << 10 + goSlabShareMin = 4 * goSlabSize +) + +type goSlab [goSlabSize]byte + +type goSlabPool struct { + access sync.Mutex + free []*goSlab + inUse atomic.Int32 + holders atomic.Int32 + pressure func() MemoryPressure + lowWater int + closed bool +} + +func newGoSlabPool(pressure func() MemoryPressure, lowWater int) *goSlabPool { + pool := &goSlabPool{ + free: make([]*goSlab, 0, lowWater), + pressure: pressure, + lowWater: lowWater, + } + for range lowWater { + pool.free = append(pool.free, goAllocateSlab()) + } + return pool +} + +func (p *goSlabPool) acquire() *goSlab { + var slab *goSlab + p.access.Lock() + index := len(p.free) - 1 + if index >= 0 { + slab = p.free[index] + p.free[index] = nil + p.free = p.free[:index] + } + p.access.Unlock() + if slab == nil { + slab = goAllocateSlab() + } + p.inUse.Add(1) + return slab +} + +func (p *goSlabPool) release(slab *goSlab) { + p.access.Lock() + if !p.closed && p.pressureLevel() == MemoryPressureNone { + p.free = append(p.free, slab) + p.access.Unlock() + } else { + p.access.Unlock() + goFreeSlab(slab) + } + p.inUse.Add(-1) +} + +func (p *goSlabPool) close() { + p.access.Lock() + p.closed = true + free := p.free + p.free = nil + p.access.Unlock() + for _, slab := range free { + goFreeSlab(slab) + } +} + +func (p *goSlabPool) trim() { + p.access.Lock() + defer p.access.Unlock() + if len(p.free) <= p.lowWater { + return + } + target := p.lowWater + if p.pressureLevel() == MemoryPressureNone { + target = max(p.lowWater, len(p.free)/2) + } + for _, slab := range p.free[target:] { + goFreeSlab(slab) + } + clear(p.free[target:]) + p.free = p.free[:target] +} + +func (p *goSlabPool) pressureLevel() MemoryPressure { + if p.pressure == nil { + return MemoryPressureNone + } + return p.pressure() +} + +func (p *goSlabPool) shareBytes() uint64 { + var share int + switch p.pressureLevel() { + case MemoryPressureNone: + return 0 + case MemoryPressureWarning: + share = int(p.inUse.Load()) / max(int(p.holders.Load()), 1) + } + return uint64(max(share, goSlabShareMin/goSlabSize)) * goSlabSize +} + +type goSlabHolder struct { + held atomic.Int32 +} + +func (h *goSlabHolder) heldBytes() uint64 { + return uint64(h.held.Load()) * goSlabSize +} + +const goSlabChainMinSlots = 4 + +type goSlabChain struct { + slots atomic.Pointer[[]*goSlab] + pool *goSlabPool + holder *goSlabHolder + held atomic.Int32 + maxSlots uint32 + releasedIndex uint64 +} + +func (c *goSlabChain) acquire() *goSlab { + slab := c.pool.acquire() + c.held.Add(1) + if c.holder.held.Add(1) == 1 { + c.pool.holders.Add(1) + } + return slab +} + +func (c *goSlabChain) release(slab *goSlab) { + c.held.Add(-1) + if c.holder.held.Add(-1) == 0 { + c.pool.holders.Add(-1) + } + c.pool.release(slab) +} + +func (c *goSlabChain) heldBytes() uint64 { + return uint64(c.held.Load()) * goSlabSize +} + +func (c *goSlabChain) loadSlots() []*goSlab { + slots := c.slots.Load() + if slots == nil { + return nil + } + return *slots +} + +func (c *goSlabChain) ensure(last uint64) []*goSlab { + slots := c.loadSlots() + required := last - c.releasedIndex + 1 + if required <= uint64(len(slots)) { + return slots + } + grown := make([]*goSlab, max(required, min(max(2*uint64(len(slots)), goSlabChainMinSlots), uint64(c.maxSlots)))) + for index := c.releasedIndex; index < c.releasedIndex+uint64(len(slots)); index++ { + grown[index%uint64(len(grown))] = slots[index%uint64(len(slots))] + } + c.slots.Store(&grown) + return grown +} + +func goSlotOf(slots []*goSlab, offset uint64) int { + return int((offset / goSlabSize) % uint64(len(slots))) +} + +func (c *goSlabChain) reserve(offset uint64, length int) { + first := offset / goSlabSize + last := (offset + uint64(length) - 1) / goSlabSize + slots := c.ensure(last) + for index := first; index <= last; index++ { + slot := int(index % uint64(len(slots))) + if slots[slot] != nil { + continue + } + slots[slot] = c.acquire() + } +} + +func (c *goSlabChain) writeAt(offset uint64, data []byte) { + slots := c.loadSlots() + for len(data) > 0 { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := copy(slab[start:], data) + data = data[n:] + offset += uint64(n) + } +} + +func (c *goSlabChain) readAt(offset uint64, target []byte) { + slots := c.loadSlots() + copied := 0 + for copied < len(target) { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := copy(target[copied:], slab[start:]) + copied += n + offset += uint64(n) + } +} + +func (c *goSlabChain) appendRuns(segments [][]byte, offset uint64, length int) [][]byte { + slots := c.loadSlots() + for length > 0 { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := min(goSlabSize-start, length) + segments = append(segments, slab[start:start+n]) + offset += uint64(n) + length -= n + } + return segments +} + +func (c *goSlabChain) releaseRange(first uint64, last uint64) { + slots := c.loadSlots() + if len(slots) == 0 { + return + } + for index := first; index <= last; index++ { + slot := int(index % uint64(len(slots))) + slab := slots[slot] + if slab == nil { + continue + } + slots[slot] = nil + c.release(slab) + } +} + +func (c *goSlabChain) releaseBelow(offset uint64) { + target := offset / goSlabSize + if target <= c.releasedIndex { + return + } + slots := c.loadSlots() + end := min(target, c.releasedIndex+uint64(len(slots))) + if end > c.releasedIndex { + c.releaseRange(c.releasedIndex, end-1) + } + c.releasedIndex = target +} + +func (c *goSlabChain) releaseDrained(offset uint64) { + c.releaseBelow(offset) + c.releaseRange(offset/goSlabSize, offset/goSlabSize) +} + +func (c *goSlabChain) releaseAll() { + slots := c.loadSlots() + for slot, slab := range slots { + if slab == nil { + continue + } + slots[slot] = nil + c.release(slab) + } +} + +const goTransmitRunCapacity = 32 + +type goTransmitRun struct { + start uint64 + end uint64 + buffer *buf.Buffer + data []byte +} + +type goTransmitStore struct { + access sync.Mutex + chain goSlabChain + runs []goTransmitRun + reserveTail uint64 +} + +func (s *goTransmitStore) reserve(offset uint64, length int) { + s.access.Lock() + defer s.access.Unlock() + s.chain.reserve(offset, length) + s.reserveTail = offset + uint64(length) +} + +func (s *goTransmitStore) shrinkReserve(tail uint64) { + s.access.Lock() + defer s.access.Unlock() + if tail >= s.reserveTail { + return + } + first := (tail + goSlabSize - 1) / goSlabSize + last := (s.reserveTail - 1) / goSlabSize + if first <= last { + s.chain.releaseRange(first, last) + } + s.reserveTail = tail +} + +func (s *goTransmitStore) writeAt(offset uint64, data []byte) { + for len(data) > 0 { + s.access.Lock() + slots := s.chain.loadSlots() + slab := slots[goSlotOf(slots, offset)] + s.access.Unlock() + start := int(offset % goSlabSize) + n := copy(slab[start:], data) + data = data[n:] + offset += uint64(n) + } +} + +func (s *goTransmitStore) adopt(offset uint64, buffer *buf.Buffer) bool { + s.access.Lock() + defer s.access.Unlock() + if len(s.runs) == goTransmitRunCapacity { + return false + } + data := buffer.Bytes() + s.runs = append(s.runs, goTransmitRun{ + start: offset, + end: offset + uint64(len(data)), + buffer: buffer, + data: data, + }) + return true +} + +func (s *goTransmitStore) appendRuns(segments [][]byte, offset uint64, length int) [][]byte { + s.access.Lock() + defer s.access.Unlock() + end := offset + uint64(length) + for index := range s.runs { + run := &s.runs[index] + if run.end <= offset { + continue + } + if run.start >= end { + break + } + if run.start > offset { + segments = s.chain.appendRuns(segments, offset, int(run.start-offset)) + offset = run.start + } + stop := min(run.end, end) + segments = append(segments, run.data[offset-run.start:stop-run.start]) + offset = stop + if offset == end { + return segments + } + } + if offset < end { + segments = s.chain.appendRuns(segments, offset, int(end-offset)) + } + return segments +} + +func (s *goTransmitStore) leadingRun(offset uint64, length int) (*buf.Buffer, []byte, bool) { + s.access.Lock() + defer s.access.Unlock() + for index := range s.runs { + run := &s.runs[index] + if run.end <= offset { + continue + } + if run.start != offset || run.end < offset+uint64(length) { + return nil, nil, false + } + return run.buffer, run.data[:length], true + } + return nil, nil, false +} + +func (s *goTransmitStore) releaseBelow(offset uint64) { + s.access.Lock() + defer s.access.Unlock() + s.chain.releaseBelow(offset) + released := 0 + for released < len(s.runs) && s.runs[released].end <= offset { + s.runs[released].buffer.Release() + released++ + } + if released > 0 { + remaining := copy(s.runs, s.runs[released:]) + clear(s.runs[remaining:]) + s.runs = s.runs[:remaining] + } +} + +func (s *goTransmitStore) releaseDrained(offset uint64) { + s.access.Lock() + defer s.access.Unlock() + if len(s.runs) > 0 || s.reserveTail > offset { + return + } + s.chain.releaseDrained(offset) +} + +func (s *goTransmitStore) releaseAll() { + s.access.Lock() + defer s.access.Unlock() + s.chain.releaseAll() + for index := range s.runs { + s.runs[index].buffer.Release() + } + clear(s.runs) + s.runs = s.runs[:0] +} + +const ( + goDescriptorRetransmitted uint8 = 1 << iota + goDescriptorNoSample + goDescriptorSacked + goDescriptorDropped + goDescriptorAmend +) + +type goSentDescriptor struct { + endOffset uint64 + sentAt int32 + flags uint8 + _ [3]byte +} + +const ( + goDescriptorBlockCapacity = 64 + goScoreboardInlineCapacity = 16 +) + +type goDescriptorBlock [goDescriptorBlockCapacity]goSentDescriptor + +// The producer publishes entries through tail; the engine consumes them through +// head. Blocks are allocated on demand and returned after their final entry is +// consumed. The directory stays fixed so neither side has to resize shared state. +type goDescriptorRing struct { + blocks [goDescriptorRingCapacity / goDescriptorBlockCapacity]atomic.Pointer[goDescriptorBlock] + pool *goDescriptorPool + head atomic.Uint32 + tail atomic.Uint32 + pending goSentDescriptor + pendingActive bool +} + +type goDescriptorPool struct { + access sync.Mutex + free []*goDescriptorBlock + lowWater int +} + +func (p *goDescriptorPool) acquire() *goDescriptorBlock { + p.access.Lock() + index := len(p.free) - 1 + if index >= 0 { + block := p.free[index] + p.free[index] = nil + p.free = p.free[:index] + p.access.Unlock() + return block + } + p.access.Unlock() + return new(goDescriptorBlock) +} + +func (p *goDescriptorPool) release(block *goDescriptorBlock) { + p.access.Lock() + p.free = append(p.free, block) + p.access.Unlock() +} + +func (p *goDescriptorPool) trim() { + p.access.Lock() + defer p.access.Unlock() + if len(p.free) <= p.lowWater { + return + } + target := max(p.lowWater, len(p.free)/2) + clear(p.free[target:]) + p.free = p.free[:target] +} + +// release requires exclusive access to both sides of the ring. For a live +// connection the engine holds transmitOwner and checks that all sent data is ACKed. +func (r *goDescriptorRing) release() { + for index := range r.blocks { + if block := r.blocks[index].Swap(nil); block != nil { + r.pool.release(block) + } + } + r.head.Store(0) + r.tail.Store(0) + r.pending = goSentDescriptor{} + r.pendingActive = false +} + +func (r *goDescriptorRing) push(endOffset uint64, sentAt int32, flags uint8) { + if flags&goDescriptorAmend != 0 { + r.pushAmendment(endOffset, sentAt, flags) + return + } + if r.pendingActive { + r.pending.endOffset = endOffset + r.pending.sentAt = sentAt + r.pending.flags |= flags + } else { + r.pending = goSentDescriptor{endOffset: endOffset, sentAt: sentAt, flags: flags} + r.pendingActive = true + } + if r.store(r.pending) { + r.pendingActive = false + } +} + +func (r *goDescriptorRing) pushAmendment(endOffset uint64, length int32, flags uint8) { + if r.pendingActive { + if !r.store(r.pending) { + return + } + r.pendingActive = false + } + r.store(goSentDescriptor{endOffset: endOffset, sentAt: length, flags: flags}) +} + +func (r *goDescriptorRing) store(entry goSentDescriptor) bool { + tail := r.tail.Load() + if tail-r.head.Load() >= goDescriptorRingCapacity { + return false + } + slot := &r.blocks[tail/goDescriptorBlockCapacity%uint32(len(r.blocks))] + block := slot.Load() + // A wrapped producer must not reuse the consumed prefix of a block that + // the consumer still owns; the consumer releases whole blocks. + if tail%goDescriptorBlockCapacity == 0 && block != nil { + return false + } + if block == nil { + block = r.pool.acquire() + slot.Store(block) + } + block[tail%goDescriptorBlockCapacity] = entry + r.tail.Store(tail + 1) + return true +} + +func (r *goDescriptorRing) take(index uint32) goSentDescriptor { + slot := &r.blocks[index/goDescriptorBlockCapacity%uint32(len(r.blocks))] + block := slot.Load() + entry := block[index%goDescriptorBlockCapacity] + if (index+1)%goDescriptorBlockCapacity == 0 { + slot.Store(nil) + r.pool.release(block) + } + return entry +} + +type goScoreboard struct { + entries []goSentDescriptor + inline [goScoreboardInlineCapacity]goSentDescriptor +} + +func (s *goScoreboard) append(entry goSentDescriptor) { + if s.entries == nil { + s.entries = s.inline[:0] + } + s.entries = append(s.entries, entry) +} + +func (s *goScoreboard) drain(ring *goDescriptorRing, unacked uint64) { + head := ring.head.Load() + tail := ring.tail.Load() + for head != tail { + entry := ring.take(head) + head++ + if entry.flags&goDescriptorAmend != 0 { + s.amend(entry.endOffset-uint64(entry.sentAt), entry.endOffset, entry.flags&^goDescriptorAmend) + continue + } + if entry.endOffset > unacked { + s.append(entry) + } + } + ring.head.Store(head) +} + +func (s *goScoreboard) amend(start uint64, end uint64, flags uint8) { + for index := range s.entries { + entry := &s.entries[index] + if entry.endOffset > start && entry.endOffset <= end { + entry.flags |= flags + } + } +} + +func (s *goScoreboard) advance(acked uint64) { + dropped := 0 + for dropped < len(s.entries) && s.entries[dropped].endOffset <= acked { + dropped++ + } + if dropped == 0 { + return + } + remaining := copy(s.entries, s.entries[dropped:]) + s.entries = s.entries[:remaining] + if remaining == 0 { + s.reset() + } +} + +func (s *goScoreboard) startOf(index int, unacked uint64) uint64 { + if index == 0 { + return unacked + } + return s.entries[index-1].endOffset +} + +func (s *goScoreboard) split(index int, offset uint64) { + entry := s.entries[index] + if entry.endOffset == offset { + return + } + s.append(goSentDescriptor{}) + copy(s.entries[index+1:], s.entries[index:len(s.entries)-1]) + s.entries[index].endOffset = offset +} + +func (s *goScoreboard) reset() { + s.entries = nil +} + +const goMaxSackBlocks = 4 + +const goMaxSackBlocksSent = 3 + +type goSackBlock struct { + start uint64 + end uint64 +} + +const goMaxOOORanges = 32 + +type goRange struct { + start uint64 + end uint64 + fin bool +} + +type goRangeSet struct { + ranges [goMaxOOORanges]goRange + count int + recent [3]uint64 +} + +func (s *goRangeSet) insert(start uint64, end uint64, fin bool) bool { + index := 0 + for index < s.count && s.ranges[index].end < start { + index++ + } + if index == s.count || s.ranges[index].start > end { + if s.count == goMaxOOORanges { + return false + } + copy(s.ranges[index+1:s.count+1], s.ranges[index:s.count]) + s.ranges[index] = goRange{start: start, end: end, fin: fin} + s.count++ + s.touch(start) + return true + } + mergedStart := min(start, s.ranges[index].start) + mergedEnd := end + mergedFin := fin + mergeEnd := index + for mergeEnd < s.count && s.ranges[mergeEnd].start <= end { + mergedEnd = max(mergedEnd, s.ranges[mergeEnd].end) + mergedFin = mergedFin || s.ranges[mergeEnd].fin + mergeEnd++ + } + s.ranges[index] = goRange{start: mergedStart, end: mergedEnd, fin: mergedFin} + copy(s.ranges[index+1:], s.ranges[mergeEnd:s.count]) + s.count -= mergeEnd - index - 1 + s.touch(mergedStart) + return true +} + +func (s *goRangeSet) touch(start uint64) { + if s.recent[0] == start { + return + } + s.recent[2] = s.recent[1] + s.recent[1] = s.recent[0] + s.recent[0] = start +} + +func (s *goRangeSet) first() (goRange, bool) { + if s.count == 0 { + return goRange{}, false + } + return s.ranges[0], true +} + +func (s *goRangeSet) removeBelow(offset uint64) { + kept := 0 + for index := range s.count { + if s.ranges[index].end <= offset { + continue + } + s.ranges[kept] = s.ranges[index] + if s.ranges[kept].start < offset { + s.ranges[kept].start = offset + } + kept++ + } + s.count = kept +} + +func (s *goRangeSet) covers(start uint64, end uint64) bool { + for index := range s.count { + if s.ranges[index].start <= start && end <= s.ranges[index].end { + return true + } + } + return false +} + +func (s *goRangeSet) bytes() uint64 { + var total uint64 + for index := range s.count { + total += s.ranges[index].end - s.ranges[index].start + } + return total +} + +func (s *goRangeSet) blocks(out *[goMaxSackBlocks]goSackBlock, base int) int { + emitted := base + for _, start := range s.recent { + if emitted == goMaxSackBlocksSent { + break + } + if start == 0 { + continue + } + found := slices.IndexFunc(s.ranges[:s.count], func(candidate goRange) bool { return candidate.start == start }) + if found < 0 { + continue + } + out[emitted] = goSackBlock{start: s.ranges[found].start, end: s.ranges[found].end} + emitted++ + } + for index := 0; index < s.count && emitted < goMaxSackBlocksSent; index++ { + start := s.ranges[index].start + if slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { + continue + } + out[emitted] = goSackBlock{start: s.ranges[index].start, end: s.ranges[index].end} + emitted++ + } + return emitted +} diff --git a/stack_go_slab_other.go b/stack_go_slab_other.go new file mode 100644 index 00000000..e9b9c1a7 --- /dev/null +++ b/stack_go_slab_other.go @@ -0,0 +1,10 @@ +//go:build !(darwin || linux) + +package tun + +func goAllocateSlab() *goSlab { + return new(goSlab) +} + +func goFreeSlab(slab *goSlab) { +} diff --git a/stack_go_slab_unix.go b/stack_go_slab_unix.go new file mode 100644 index 00000000..aab241df --- /dev/null +++ b/stack_go_slab_unix.go @@ -0,0 +1,23 @@ +//go:build darwin || linux + +package tun + +import ( + "unsafe" + + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +func goAllocateSlab() *goSlab { + memory, err := unix.Mmap(-1, 0, goSlabSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_ANON|unix.MAP_PRIVATE) + if err != nil { + panic(E.Cause(err, "go: allocate slab")) + } + return (*goSlab)(unsafe.Pointer(&memory[0])) +} + +func goFreeSlab(slab *goSlab) { + unix.Munmap(unsafe.Slice(&slab[0], goSlabSize)) +} diff --git a/stack_go_socket_darwin.go b/stack_go_socket_darwin.go new file mode 100644 index 00000000..2cd6ee14 --- /dev/null +++ b/stack_go_socket_darwin.go @@ -0,0 +1,170 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "syscall" + "unsafe" + + rawfile "github.com/sagernet/sing-tun/internal/rawfile_darwin" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +func (s *goSocket) writeVector(iovecs []unix.Iovec) (int, syscall.Errno) { + var message unix.Msghdr + message.Iov = &iovecs[0] + message.SetIovlen(len(iovecs)) + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_SENDMSG, uintptr(s.fd), uintptr(unsafe.Pointer(&message)), goSocketSendFlags) + return int(n), errno +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + var base *byte + if len(data) > 0 { + base = &data[0] + } + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_SENDTO, uintptr(s.fd), uintptr(unsafe.Pointer(base)), uintptr(len(data)), goSocketSendFlags, 0, 0) + return int(n), errno +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + sockaddr unsafe.Pointer + sockaddrLength uintptr + inet4 unix.RawSockaddrInet4 + inet6 unix.RawSockaddrInet6 + ) + address := destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return unix.EAFNOSUPPORT + } + inet4.Len = unix.SizeofSockaddrInet4 + inet4.Family = unix.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], destination.Port()) + sockaddr = unsafe.Pointer(&inet4) + sockaddrLength = unix.SizeofSockaddrInet4 + } else { + inet6.Len = unix.SizeofSockaddrInet6 + inet6.Family = unix.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], destination.Port()) + sockaddr = unsafe.Pointer(&inet6) + sockaddrLength = unix.SizeofSockaddrInet6 + } + var base *byte + if len(data) > 0 { + base = &data[0] + } + //nolint:staticcheck + _, _, errno := unix.RawSyscall6(unix.SYS_SENDTO, uintptr(s.fd), uintptr(unsafe.Pointer(base)), uintptr(len(data)), goSocketSendFlags, uintptr(sockaddr), sockaddrLength) + return errno +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + var storage unix.RawSockaddrInet6 + length := uint32(unsafe.Sizeof(storage)) + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_RECVFROM, uintptr(s.fd), uintptr(unsafe.Pointer(&buffer[0])), uintptr(len(buffer)), unix.MSG_DONTWAIT, uintptr(unsafe.Pointer(&storage)), uintptr(unsafe.Pointer(&length))) + if errno != 0 { + return 0, netip.AddrPort{}, errno + } + return int(n), M.SocksaddrFromNetIP(M.AddrPortFromRawSockaddr((*unix.RawSockaddr)(unsafe.Pointer(&storage)))).Unwrap().AddrPort(), 0 +} + +type goPacketBatchIO struct { + headers [goPacketBatchSize]rawfile.MsgHdrX + vectors [goPacketBatchSize]unix.Iovec + names [goPacketBatchSize]unix.RawSockaddrAny +} + +func (b *goPacketBatchIO) reset() { + clear(b.headers[:]) + clear(b.vectors[:]) +} + +func (s *goSocket) enablePacketOffload() {} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + b.vectors[index] = rawfile.IovecFromBytes(packets[index].data) + message := &b.headers[index] + *message = rawfile.MsgHdrX{} + message.Msg.Iov = &b.vectors[index] + message.Msg.Iovlen = 1 + if !connected { + b.names[index] = unix.RawSockaddrAny{} + message.Msg.Name = (*byte)(unsafe.Pointer(&b.names[index])) + message.Msg.Namelen = unix.SizeofSockaddrAny + } + } + count, errno := receiveMessageBatch(socket.fd, b.headers[:len(packets)]) + if errno != 0 { + return 0, errno + } + for index := range count { + message := &b.headers[index] + packet := &packets[index] + packet.truncated = message.Msg.Flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 || int(message.DataLen) > len(packet.data) + packet.data = packet.data[:min(int(message.DataLen), len(packet.data))] + if !connected { + packet.destination = M.SocksaddrFromRawSockaddrAny(&b.names[index]).AddrPort() + } + } + return count, 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + sent := 0 + limit := len(packets) + for sent < len(packets) { + if !connected || len(packets[sent].data) == 0 { + var errno syscall.Errno + if connected { + _, errno = socket.write(packets[sent].data) + } else { + errno = socket.sendTo(packets[sent].data, packets[sent].destination, family) + } + if errno == unix.EINTR { + continue + } + if errno != 0 { + return sent, errno + } + sent++ + continue + } + count := 0 + for index := sent; index < len(packets) && count < limit; index++ { + if len(packets[index].data) == 0 { + break + } + b.vectors[count] = rawfile.IovecFromBytes(packets[index].data) + b.headers[count] = rawfile.MsgHdrX{} + b.headers[count].Msg.Iov = &b.vectors[count] + b.headers[count].Msg.Iovlen = 1 + count++ + } + written, errno := rawfile.NonBlockingSendMMsg(socket.fd, b.headers[:count]) + if errno == unix.EINTR { + continue + } + if errno == unix.EMSGSIZE && count > 1 { + limit = (count + 1) / 2 + continue + } + if errno != 0 { + return sent, errno + } + if written == 0 { + return sent, unix.EIO + } + sent += written + } + return sent, 0 +} diff --git a/stack_go_socket_linux.go b/stack_go_socket_linux.go new file mode 100644 index 00000000..0deaca11 --- /dev/null +++ b/stack_go_socket_linux.go @@ -0,0 +1,256 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "syscall" + "unsafe" + + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +func (s *goSocket) writeVector(iovecs []unix.Iovec) (int, syscall.Errno) { + var message unix.Msghdr + message.Iov = &iovecs[0] + message.SetIovlen(len(iovecs)) + return goSendmsg(s.fd, &message, goSocketSendFlags) +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + var vector unix.Iovec + if len(data) > 0 { + vector.Base = &data[0] + } + vector.SetLen(len(data)) + var message unix.Msghdr + message.Iov = &vector + message.SetIovlen(1) + return goSendmsg(s.fd, &message, goSocketSendFlags) +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + inet4 unix.SockaddrInet4 + inet6 unix.SockaddrInet6 + to unix.Sockaddr + ) + address := destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return unix.EAFNOSUPPORT + } + inet4.Addr = address.As4() + inet4.Port = int(destination.Port()) + to = &inet4 + } else { + inet6.Addr = address.As16() + inet6.Port = int(destination.Port()) + to = &inet6 + } + err := unix.Sendto(s.fd, data, goSocketSendFlags, to) + if err != nil { + return err.(syscall.Errno) + } + return 0 +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + n, from, err := unix.Recvfrom(s.fd, buffer, unix.MSG_DONTWAIT) + if err != nil { + return 0, netip.AddrPort{}, err.(syscall.Errno) + } + return n, M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(from)).Unwrap().AddrPort(), 0 +} + +func goSendmsg(fd int, message *unix.Msghdr, flags int) (int, syscall.Errno) { + n, _, errno := unix.Syscall(unix.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(message)), uintptr(flags)) + if errno != 0 { + return 0, errno + } + return int(n), 0 +} + +type goPacketMessageHeader struct { + header unix.Msghdr + length uint32 +} + +type goPacketBatchIO struct { + headers [goPacketBatchSize]goPacketMessageHeader + vectors [goPacketBatchSize]unix.Iovec + names [goPacketBatchSize]unix.RawSockaddrAny + controls [goPacketBatchSize][64]byte + counts [goPacketBatchSize]int +} + +func (b *goPacketBatchIO) reset() { + clear(b.headers[:]) + clear(b.vectors[:]) +} + +func (s *goSocket) enablePacketOffload() { + err := unix.SetsockoptInt(s.fd, unix.IPPROTO_UDP, unix.UDP_GRO, 1) + s.packetGRO = err == nil +} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + vector := &b.vectors[index] + *vector = unix.Iovec{Base: &packets[index].data[0]} + vector.SetLen(len(packets[index].data)) + message := &b.headers[index] + *message = goPacketMessageHeader{} + message.header.Iov = vector + message.header.SetIovlen(1) + if !connected { + b.names[index] = unix.RawSockaddrAny{} + message.header.Name = (*byte)(unsafe.Pointer(&b.names[index])) + message.header.Namelen = unix.SizeofSockaddrAny + } + if socket.packetGRO { + message.header.Control = &b.controls[index][0] + message.header.SetControllen(len(b.controls[index])) + } + } + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_RECVMMSG, uintptr(socket.fd), uintptr(unsafe.Pointer(&b.headers[0])), uintptr(len(packets)), unix.MSG_DONTWAIT, 0, 0) + if errno != 0 { + return 0, errno + } + for index := range int(count) { + message := &b.headers[index] + packet := &packets[index] + packet.truncated = message.header.Flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 || int(message.length) > len(packet.data) + packet.data = packet.data[:min(int(message.length), len(packet.data))] + if !connected { + packet.destination = M.SocksaddrFromRawSockaddrAny(&b.names[index]).AddrPort() + } + if socket.packetGRO { + control := b.controls[index][:message.header.Controllen] + for len(control) >= unix.CmsgLen(0) { + controlHeader := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + length := int(controlHeader.Len) + if length < unix.CmsgLen(0) || length > len(control) { + packet.truncated = true + break + } + if controlHeader.Level == unix.IPPROTO_UDP && controlHeader.Type == unix.UDP_GRO { + if length < unix.CmsgLen(4) { + packet.truncated = true + } else { + packet.segmentSize = int(binary.NativeEndian.Uint32(control[unix.CmsgLen(0):])) + packet.truncated = packet.truncated || packet.segmentSize <= 0 || packet.segmentSize > 65535 + } + } + control = control[min(unix.CmsgSpace(length-unix.CmsgLen(0)), len(control)):] + } + } + } + return int(count), 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + coalesce := !socket.packetGSODisabled + sent := 0 + for sent < len(packets) { + messageCount := 0 + hasSegments := false + for index := sent; index < len(packets); { + packet := &packets[index] + segmentSize := len(packet.data) + end := index + 1 + length := segmentSize + if coalesce && segmentSize > 0 { + for end < len(packets) && end-index < 64 { + next := &packets[end] + if !connected && next.destination != packet.destination || len(next.data) == 0 || len(next.data) > segmentSize || length+len(next.data) > 65507 { + break + } + length += len(next.data) + end++ + if len(next.data) < segmentSize { + break + } + } + } + message := &b.headers[messageCount] + *message = goPacketMessageHeader{} + for vectorIndex := index; vectorIndex < end; vectorIndex++ { + data := packets[vectorIndex].data + vector := &b.vectors[vectorIndex-sent] + *vector = unix.Iovec{} + if len(data) > 0 { + vector.Base = &data[0] + } + vector.SetLen(len(data)) + } + message.header.Iov = &b.vectors[index-sent] + message.header.SetIovlen(end - index) + if !connected { + name := &b.names[messageCount] + *name = unix.RawSockaddrAny{} + address := packet.destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return sent, unix.EAFNOSUPPORT + } + inet4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(name)) + inet4.Family = unix.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], packet.destination.Port()) + message.header.Namelen = unix.SizeofSockaddrInet4 + } else { + inet6 := (*unix.RawSockaddrInet6)(unsafe.Pointer(name)) + inet6.Family = unix.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], packet.destination.Port()) + message.header.Namelen = unix.SizeofSockaddrInet6 + } + message.header.Name = (*byte)(unsafe.Pointer(name)) + } + if end-index > 1 { + hasSegments = true + control := b.controls[messageCount][:unix.CmsgSpace(2)] + clear(control) + controlHeader := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + controlHeader.Level = unix.IPPROTO_UDP + controlHeader.Type = unix.UDP_SEGMENT + controlHeader.SetLen(unix.CmsgLen(2)) + binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):], uint16(segmentSize)) + message.header.Control = &control[0] + message.header.SetControllen(len(control)) + } + b.counts[messageCount] = end - index + messageCount++ + index = end + } + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_SENDMMSG, uintptr(socket.fd), uintptr(unsafe.Pointer(&b.headers[0])), uintptr(messageCount), goSocketSendFlags, 0, 0) + if errno == unix.EINTR { + continue + } + if errno != 0 { + if hasSegments { + switch errno { + case unix.EINVAL, unix.EIO, unix.ENOPROTOOPT, unix.EOPNOTSUPP: + socket.packetGSODisabled = true + coalesce = false + continue + case unix.EMSGSIZE: + coalesce = false + continue + } + } + return sent, errno + } + if count == 0 { + return sent, unix.EIO + } + for index := range int(count) { + sent += b.counts[index] + } + } + return sent, 0 +} diff --git a/stack_go_socket_nonunix.go b/stack_go_socket_nonunix.go new file mode 100644 index 00000000..cb057213 --- /dev/null +++ b/stack_go_socket_nonunix.go @@ -0,0 +1,38 @@ +//go:build !linux && !darwin + +package tun + +import "syscall" + +type goPacketBatchIO struct{} + +func (b *goPacketBatchIO) reset() {} + +func (s *goSocket) enablePacketOffload() {} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + length, source, errno := socket.receiveFrom(packets[index].data) + if errno != 0 { + return index, errno + } + packets[index].data = packets[index].data[:length] + packets[index].destination = source + } + return len(packets), 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + for index, packet := range packets { + var errno syscall.Errno + if connected { + _, errno = socket.write(packet.data) + } else { + errno = socket.sendTo(packet.data, packet.destination, family) + } + if errno != 0 { + return index, errno + } + } + return len(packets), 0 +} diff --git a/stack_go_socket_unix.go b/stack_go_socket_unix.go new file mode 100644 index 00000000..6ec31402 --- /dev/null +++ b/stack_go_socket_unix.go @@ -0,0 +1,122 @@ +//go:build darwin || linux + +package tun + +import ( + "net/netip" + "os" + "syscall" + "unsafe" + + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +const ( + goSocketSendFlags = unix.MSG_DONTWAIT | unix.MSG_NOSIGNAL + goSpliceDuplicatesSocket = true +) + +type goIOVector = unix.Iovec + +type goSocket struct { + fd int + token uint32 + registered bool + packetGRO bool + packetGSODisabled bool +} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + rawConn, err := conn.SyscallConn() + if err != nil { + return goSocket{}, err + } + socket := goSocket{fd: -1} + var duplicateError error + err = rawConn.Control(func(fd uintptr) { + socket.fd, duplicateError = unix.FcntlInt(fd, unix.F_DUPFD_CLOEXEC, 0) + }) + err = E.Errors(err, duplicateError) + if err != nil { + socket.close() + return goSocket{}, err + } + return socket, nil +} + +func (s *goSocket) close() { + if s.fd < 0 { + return + } + descriptor := s.fd + s.fd = -1 + unix.Close(descriptor) +} + +func newUnpolledFile(fd int, name string) *os.File { + unix.SetNonblock(fd, false) + file := os.NewFile(uintptr(fd), name) + unix.SetNonblock(fd, true) + return file +} + +func (s *goSocket) shutdownWrite() { + unix.Shutdown(s.fd, unix.SHUT_WR) +} + +func (s *goSocket) family() (uint8, error) { + name, err := unix.Getsockname(s.fd) + if err != nil { + return 0, err + } + switch name.(type) { + case *unix.SockaddrInet4: + return unix.AF_INET, nil + case *unix.SockaddrInet6: + return unix.AF_INET6, nil + default: + return 0, E.New("unsupported socket family") + } +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + name, err := unix.Getpeername(s.fd) + if err != nil { + return netip.AddrPort{}, false + } + return M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(name)).Unwrap().AddrPort(), true +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + iovecs = iovecs[:0] + for _, segment := range segments { + if len(segment) == 0 { + continue + } + vector := unix.Iovec{Base: &segment[0]} + vector.SetLen(len(segment)) + iovecs = append(iovecs, vector) + } + return iovecs +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == unix.EAGAIN || errno == unix.EWOULDBLOCK || errno == unix.EINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == unix.EMSGSIZE || errno == unix.EAFNOSUPPORT || errno == unix.ENOBUFS +} + +func goFatalReadError(err error) bool { + return E.IsMulti(err, unix.EBADF, unix.ENODEV, unix.ENXIO) +} + +func (s *goSocket) readVector(iovecs []unix.Iovec) (int, syscall.Errno) { + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_READV, uintptr(s.fd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) + return int(n), errno +} diff --git a/stack_go_socket_windows.go b/stack_go_socket_windows.go new file mode 100644 index 00000000..ae90e86d --- /dev/null +++ b/stack_go_socket_windows.go @@ -0,0 +1,201 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "runtime" + "syscall" + "unsafe" + + "github.com/sagernet/sing-tun/internal/afd" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/windows" +) + +const ( + goSpliceDuplicatesSocket = false + goSocketFIONBIO = 0x8004667e + goSocketReceiveBuffer = 4 << 20 +) + +type goIOVector = windows.WSABuf + +type goSocket struct { + handle windows.Handle + entry *goAFDEntry +} + +type goAFDEntry struct { + ioStatusBlock windows.IO_STATUS_BLOCK + pollInfo afd.PollInfo + baseHandle windows.Handle + token uint32 + interest uint8 + armed bool + cancelled bool + pinner runtime.Pinner +} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + rawConn, err := conn.SyscallConn() + if err != nil { + return goSocket{}, err + } + var handle windows.Handle + err = rawConn.Control(func(fd uintptr) { + handle = windows.Handle(fd) + }) + if err != nil { + return goSocket{}, err + } + var ( + mode uint32 = 1 + bytesReturned uint32 + ) + err = windows.WSAIoctl(handle, goSocketFIONBIO, (*byte)(unsafe.Pointer(&mode)), uint32(unsafe.Sizeof(mode)), nil, 0, &bytesReturned, nil, 0) + if err != nil { + return goSocket{}, E.Cause(err, "set socket non-blocking") + } + // Without an explicit receive buffer the socket advertises 64 KiB, and readiness-driven reads + // then cost a thread wake-up per window; the runtime poller hid this behind posted overlapped reads. + err = windows.SetsockoptInt(handle, windows.SOL_SOCKET, windows.SO_RCVBUF, goSocketReceiveBuffer) + if err != nil { + return goSocket{}, E.Cause(err, "set socket receive buffer") + } + return goSocket{handle: handle}, nil +} + +func (s *goSocket) close() { +} + +func (s *goSocket) shutdownWrite() { + windows.Shutdown(s.handle, windows.SHUT_WR) +} + +func (s *goSocket) family() (uint8, error) { + name, err := windows.Getsockname(s.handle) + if err != nil { + return 0, err + } + switch name.(type) { + case *windows.SockaddrInet4: + return windows.AF_INET, nil + case *windows.SockaddrInet6: + return windows.AF_INET6, nil + default: + return 0, E.New("unsupported socket family") + } +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + name, err := windows.Getpeername(s.handle) + if err != nil { + return netip.AddrPort{}, false + } + address := M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(name)).Unwrap() + return address.AddrPort(), address.IsValid() +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + iovecs = iovecs[:0] + for _, segment := range segments { + if len(segment) == 0 { + continue + } + iovecs = append(iovecs, windows.WSABuf{Len: uint32(len(segment)), Buf: &segment[0]}) + } + return iovecs +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == windows.WSAEWOULDBLOCK || errno == windows.WSAEINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == windows.WSAEMSGSIZE || errno == windows.WSAEAFNOSUPPORT || errno == windows.WSAENOBUFS +} + +func (s *goSocket) readVector(iovecs []goIOVector) (int, syscall.Errno) { + var ( + received uint32 + flags uint32 + ) + err := windows.WSARecv(s.handle, &iovecs[0], uint32(len(iovecs)), &received, &flags, nil, nil) + if err != nil { + return 0, err.(syscall.Errno) + } + return int(received), 0 +} + +func (s *goSocket) writeVector(iovecs []goIOVector) (int, syscall.Errno) { + var sent uint32 + err := windows.WSASend(s.handle, &iovecs[0], uint32(len(iovecs)), &sent, 0, nil, nil) + if err != nil { + return 0, err.(syscall.Errno) + } + return int(sent), 0 +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + vector := windows.WSABuf{Len: uint32(len(data))} + if len(data) > 0 { + vector.Buf = &data[0] + } + var sent uint32 + err := windows.WSASend(s.handle, &vector, 1, &sent, 0, nil, nil) + if err != nil { + return 0, err.(syscall.Errno) + } + return int(sent), 0 +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + storage windows.RawSockaddrAny + length int32 + ) + address := destination.Addr().Unmap() + if family == windows.AF_INET { + if !address.Is4() { + return windows.WSAEAFNOSUPPORT + } + inet4 := (*windows.RawSockaddrInet4)(unsafe.Pointer(&storage)) + inet4.Family = windows.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], destination.Port()) + length = int32(unsafe.Sizeof(*inet4)) + } else { + inet6 := (*windows.RawSockaddrInet6)(unsafe.Pointer(&storage)) + inet6.Family = windows.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], destination.Port()) + length = int32(unsafe.Sizeof(*inet6)) + } + vector := windows.WSABuf{Len: uint32(len(data))} + if len(data) > 0 { + vector.Buf = &data[0] + } + var sent uint32 + err := windows.WSASendTo(s.handle, &vector, 1, &sent, 0, &storage, length, nil, nil) + if err != nil { + return err.(syscall.Errno) + } + return 0 +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + var ( + storage windows.RawSockaddrAny + length = int32(unsafe.Sizeof(storage)) + received uint32 + flags uint32 + ) + vector := windows.WSABuf{Len: uint32(len(buffer)), Buf: &buffer[0]} + err := windows.WSARecvFrom(s.handle, &vector, 1, &received, &flags, &storage, &length, nil, nil) + if err != nil { + return 0, netip.AddrPort{}, err.(syscall.Errno) + } + return int(received), M.SocksaddrFromNetIP(M.AddrPortFromRawSockaddr(&storage.Addr)).Unwrap().AddrPort(), 0 +} diff --git a/stack_go_splice.go b/stack_go_splice.go new file mode 100644 index 00000000..24cce2d1 --- /dev/null +++ b/stack_go_splice.go @@ -0,0 +1,965 @@ +package tun + +import ( + "io" + "net" + "net/netip" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +const ( + goSocketEventBatch = 64 + goSpliceTokenShift = 8 + goSpliceReadChunk = 2 * goSlabSize +) + +type SpliceSocket interface { + syscall.Conn + io.Closer + Attach(closer io.Closer) (io.Closer, bool) + Detach() +} + +type SpliceOptions struct { + ReadCounters []N.CountFunc + WriteCounters []N.CountFunc + OnClose N.CloseHandlerFunc +} + +type PacketNAT struct { + Origin M.Socksaddr + Destination M.Socksaddr + FakeIP bool + Unidirectional bool +} + +type SplicePacketOptions struct { + SpliceOptions + Timeout time.Duration + NAT PacketNAT + Cached []*N.PacketBuffer + Offload N.PacketOffload + FrontHeadroom int + RearHeadroom int +} + +type goSpliceSlot struct { + stream *GoConn + packet *GoPacketConn + generation uint32 +} + +func (e *goEngine) allocateSpliceToken(stream *GoConn, packet *GoPacketConn) uint32 { + var index uint32 + if free := len(e.spliceFree); free > 0 { + index = e.spliceFree[free-1] + e.spliceFree = e.spliceFree[:free-1] + } else { + index = uint32(len(e.spliceSlots)) + e.spliceSlots = append(e.spliceSlots, goSpliceSlot{}) + } + slot := &e.spliceSlots[index] + slot.generation = (slot.generation + 1) & (1<> goSpliceTokenShift + slot := &e.spliceSlots[index] + slot.stream = nil + slot.packet = nil + e.spliceFree = append(e.spliceFree, index) +} + +func (e *goEngine) spliceSlot(token uint32) *goSpliceSlot { + index := token >> goSpliceTokenShift + if index >= uint32(len(e.spliceSlots)) { + return nil + } + slot := &e.spliceSlots[index] + if slot.generation != token&(1<= goConnStateAborted { + conn.splicePending.Store(nil) + e.spliceRelease(splice, conn.closeError()) + return + } + splice.token = e.allocateSpliceToken(conn, nil) + var err error + if goSpliceDuplicatesSocket { + err = splice.original.Close() + } + if err == nil { + err = e.platformIO.registerSocket(&splice.socket, splice.token, goInterestRead) + } + if err != nil { + conn.splicePending.Store(nil) + e.releaseSpliceToken(splice.token) + e.spliceRelease(splice, err) + e.spliceAbort(conn, err) + return + } + splice.interest = goInterestRead + conn.splice = splice + conn.spliced.Store(true) + conn.splicePending.Store(nil) + e.spliceFlushUpload(conn) +} + +func (e *goEngine) spliceRelease(splice *goSpliceStream, err error) { + splice.socket.close() + splice.owner.Detach() + splice.owner.Close() + if splice.options.OnClose != nil { + splice.options.OnClose(err) + } +} + +func (e *goEngine) spliceDetach(conn *GoConn, err error) { + pending := conn.splicePending.Swap(nil) + if pending != nil { + e.spliceRelease(pending, err) + } + splice := conn.splice + if splice == nil { + return + } + if splice.dirty { + e.spliceUnlinkDirty(conn) + } + conn.splice = nil + conn.spliced.Store(false) + e.platformIO.unregisterSocket(&splice.socket) + e.releaseSpliceToken(splice.token) + e.spliceRelease(splice, err) +} + +func (e *goEngine) spliceUnlinkDirty(conn *GoConn) { + splice := conn.splice + splice.dirty = false + if e.spliceDirtyList == conn { + e.spliceDirtyList = splice.dirtyNext + splice.dirtyNext = nil + return + } + for previous := e.spliceDirtyList; previous != nil; previous = previous.splice.dirtyNext { + if previous.splice.dirtyNext == conn { + previous.splice.dirtyNext = splice.dirtyNext + splice.dirtyNext = nil + return + } + } +} + +func (e *goEngine) spliceAbort(conn *GoConn, err error) { + if conn.connState.Load() >= goConnStateDead { + conn.storeError(err) + conn.receiveDrainable.Store(false) + e.spliceDetach(conn, err) + return + } + e.sendReset(conn) + e.detachConn(conn, err, goDeathAbortLinger) +} + +func (e *goEngine) spliceSetInterest(conn *GoConn, interest uint8) { + splice := conn.splice + if splice.interest == interest { + return + } + splice.interest = interest + err := e.platformIO.updateSocket(&splice.socket, interest) + if err != nil { + e.spliceAbort(conn, err) + } +} + +func (e *goEngine) handleStreamEvent(conn *GoConn, event goSocketEvent) { + if event.writable && conn.splice != nil { + conn.splice.blocked = false + e.spliceFlushUpload(conn) + } + if event.readable && conn.splice != nil { + e.spliceReadPeer(conn) + } +} + +func (e *goEngine) spliceDeliver(conn *GoConn, data []byte) { + conn.receiveChain.reserve(conn.receiveNext, len(data)) + conn.receiveChain.writeAt(conn.receiveNext, data) + conn.receiveNext += uint64(len(data)) + conn.receiveNextAck.Store(conn.receiveNext) + conn.receiveAvailable.Store(conn.receiveNext) + conn.publishReceiveWindow() + e.spliceMarkDirty(conn) +} + +func (e *goEngine) spliceMarkDirty(conn *GoConn) { + splice := conn.splice + if splice.dirty { + return + } + splice.dirty = true + splice.dirtyNext = e.spliceDirtyList + e.spliceDirtyList = conn +} + +func (e *goEngine) flushSpliceDirty() { + for conn := e.spliceDirtyList; conn != nil; { + splice := conn.splice + next := splice.dirtyNext + splice.dirtyNext = nil + splice.dirty = false + e.spliceFlushUpload(conn) + conn = next + } + e.spliceDirtyList = nil +} + +func (e *goEngine) spliceFlushUpload(conn *GoConn) { + splice := conn.splice + for !splice.blocked { + consumed := conn.consumedTail.Load() + available := conn.receiveAvailable.Load() + if consumed >= available { + break + } + e.spliceSegments = conn.receiveChain.appendRuns(e.spliceSegments[:0], consumed, int(available-consumed)) + e.spliceIovecs = goIovecsFromSegments(e.spliceIovecs, e.spliceSegments) + n, errno := splice.socket.writeVector(e.spliceIovecs) + switch { + case errno == 0: + consumed = conn.consumedTail.Add(uint64(n)) + goCount(splice.options.ReadCounters, n) + if consumed < available { + splice.blocked = true + } + if conn.windowUpdateDue(consumed) { + e.handleWindowUpdate(conn) + } + case goSocketWouldBlock(errno): + splice.blocked = true + default: + e.spliceAbort(conn, E.Cause(errno, "go: write peer")) + return + } + } + consumed := conn.consumedTail.Load() + conn.receiveChain.releaseBelow(consumed) + if consumed == conn.receiveAvailable.Load() && (conn.oooRanges == nil || conn.oooRanges.count == 0) { + conn.receiveChain.releaseDrained(consumed) + } + interest := splice.interest &^ goInterestWrite + if splice.blocked { + interest |= goInterestWrite + } + e.spliceSetInterest(conn, interest) + if conn.splice != nil { + e.spliceCheckUpload(conn) + } +} + +func (e *goEngine) spliceCheckUpload(conn *GoConn) { + splice := conn.splice + if splice.uploadClosed || !conn.finReceived || conn.receiveAvailable.Load() > conn.consumedTail.Load() { + return + } + splice.socket.shutdownWrite() + splice.uploadClosed = true + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceAfterInput(conn *GoConn) { + splice := conn.splice + if splice == nil { + return + } + if conn.receiveAvailable.Load() > conn.consumedTail.Load() || (conn.finReceived && !splice.uploadClosed) { + e.spliceMarkDirty(conn) + return + } + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceReadPeer(conn *GoConn) { + // Empty speculative reads must not keep the partial tail slab while the + // socket is waiting. Unacknowledged and out-of-order data remain retained. + defer conn.releaseDrainedSlabs() + splice := conn.splice + for !splice.downloadClosed { + budget := conn.writeBudget(0) + if budget <= 0 { + splice.readStalled = true + e.spliceSetInterest(conn, splice.interest&^goInterestRead) + return + } + budget = min(budget, goSpliceReadChunk) + buffered := conn.bufferedTail.Load() + conn.transmitStore.reserve(buffered, budget) + e.spliceSegments = conn.transmitStore.chain.appendRuns(e.spliceSegments[:0], buffered, budget) + e.spliceIovecs = goIovecsFromSegments(e.spliceIovecs, e.spliceSegments) + n, errno := splice.socket.readVector(e.spliceIovecs) + if errno != 0 || n < 0 { + n = 0 + } + if n < budget { + conn.transmitStore.shrinkReserve(buffered + uint64(n)) + } + switch { + case errno == 0 && n == 0: + splice.downloadClosed = true + e.spliceSetInterest(conn, splice.interest&^goInterestRead) + if conn.splice == nil { + return + } + conn.finPending = true + case errno == 0: + conn.bufferedTail.Store(buffered + uint64(n)) + goCount(splice.options.WriteCounters, n) + if !e.spliceTransmit(conn) { + return + } + case goSocketWouldBlock(errno): + e.spliceMaybeFinish(conn) + return + default: + e.spliceAbort(conn, E.Cause(errno, "go: read peer")) + return + } + } + e.maybeSendFin(conn) + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceTransmit(conn *GoConn) bool { + if conn.blockedValid || !conn.transmitOwner.CompareAndSwap(0, 1) { + return true + } + _, blocked := conn.transmitLoop(false, 0) + conn.transmitOwner.Store(0) + if conn.connState.Load() >= goConnStateAborted { + return false + } + if blocked { + e.handleTransmitBlocked(conn) + } + return true +} + +func (e *goEngine) spliceResume(conn *GoConn) { + if !e.spliceTransmit(conn) { + return + } + splice := conn.splice + if splice == nil || !splice.readStalled || splice.downloadClosed || conn.writeBudget(0) <= 0 { + return + } + splice.readStalled = false + e.spliceSetInterest(conn, splice.interest|goInterestRead) +} + +func (e *goEngine) spliceRetryBlocked(conn *GoConn) { + segment := conn.blockedSegment + end := segment.offset + uint64(segment.length) + err := conn.transmitFrame(&segment) + switch err { + case nil: + case errGoTransmitBlocked: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) + e.handleTransmitBlocked(conn) + return + case errGoFrameDropped: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmittedTail.Store(end) + e.handleDroppedFrames(conn) + return + default: + e.spliceAbort(conn, E.Cause(err, "go: write tun")) + return + } + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmittedTail.Store(end) + e.spliceResume(conn) + if conn.splice != nil && conn.splice.downloadClosed { + e.maybeSendFin(conn) + e.spliceMaybeFinish(conn) + } +} + +func (e *goEngine) spliceMaybeFinish(conn *GoConn) { + splice := conn.splice + if splice == nil || !splice.uploadClosed || !splice.downloadClosed || !conn.finSent { + return + } + e.spliceDetach(conn, nil) +} + +type goPacketMessage struct { + data []byte + destination netip.AddrPort + segmentSize int + truncated bool + payloadLength int +} + +type goPacketUpload struct { + message goPacketMessage + next int +} + +type goSplicePacket struct { + writer *GoPacketConn + owner SpliceSocket + original io.Closer + socket goSocket + token uint32 + family uint8 + connected bool + rewritePort bool + finished bool + peer netip.AddrPort + origin M.Socksaddr + destination M.Socksaddr + direct map[netip.Addr]struct{} + options SplicePacketOptions + uploadHead int + uploadTail int + nextDirty *goSplicePacket +} + +func (c *UDPNatConn) Splice(owner SpliceSocket, options SplicePacketOptions) bool { + writer, isGoPacketConn := c.writer.(*GoPacketConn) + if !isGoPacketConn || writer.conn.Load() != c { + return false + } + return writer.spliceTo(owner, options) +} + +func (w *GoPacketConn) spliceTo(owner SpliceSocket, options SplicePacketOptions) bool { + if options.NAT.Origin.IsValid() && (!options.NAT.Origin.IsIP() || !options.NAT.Destination.IsIP() && options.Offload == nil) { + return false + } + splice := &goSplicePacket{writer: w, owner: owner, options: options, uploadHead: -1, uploadTail: -1} + original, attached := owner.Attach(splice) + if !attached { + return false + } + splice.original = original + socket, err := goSpliceSocket(owner) + if err != nil { + owner.Detach() + w.engine.stack.logger.Debug(E.Cause(err, "go: splice packet connection")) + return false + } + family, err := socket.family() + if err != nil { + socket.close() + owner.Detach() + return false + } + splice.socket = socket + splice.family = family + splice.peer, splice.connected = socket.peerAddress() + if options.Offload != nil && !splice.connected { + socket.close() + owner.Detach() + return false + } + if options.NAT.Origin.IsValid() { + splice.origin = options.NAT.Origin + splice.destination = options.NAT.Destination + splice.rewritePort = splice.origin.Port != splice.destination.Port + } + if !w.splicePending.CompareAndSwap(nil, splice) { + socket.close() + owner.Detach() + return false + } + w.engine.postMessage(&w.spliceMessage) + return true +} + +func (s *goSplicePacket) Close() error { + s.writer.closeSplice(net.ErrClosed) + return nil +} + +func (e *goEngine) handlePacketSpliceEngage(w *GoPacketConn) { + splice := w.splicePending.Load() + if splice == nil { + return + } + if w.splice != nil { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, net.ErrClosed) + return + } + natConn := w.conn.Load() + if natConn == nil || natConn.isClosed() { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, io.ErrClosedPipe) + return + } + closeError := w.spliceCloseError.Load() + if closeError != nil { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, closeError.err) + natConn.Close() + return + } + if splice.options.Timeout > 0 { + current := natConn.Timeout() + if !(current > 0 && splice.options.Timeout >= current) && !natConn.SetTimeout(splice.options.Timeout) { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, io.ErrClosedPipe) + return + } + } + splice.token = e.allocateSpliceToken(nil, w) + var err error + if goSpliceDuplicatesSocket { + err = splice.original.Close() + } + if err == nil { + err = e.platformIO.registerSocket(&splice.socket, splice.token, goInterestRead) + } + if err != nil { + w.splicePending.Store(nil) + e.releaseSpliceToken(splice.token) + e.packetSpliceRelease(splice, err) + natConn.Close() + return + } + w.splice = splice + splice.socket.enablePacketOffload() + w.spliceActive.Store(true) + w.splicePending.Store(nil) + readOptions := N.ReadWaitOptions{ + FrontHeadroom: splice.options.FrontHeadroom, + RearHeadroom: splice.options.RearHeadroom, + } + e.updatePacketHeadroom(readOptions) + cached := splice.options.Cached + splice.options.Cached = nil + for _, packet := range cached { + if w.splice != nil { + packet.Buffer = readOptions.Copy(packet.Buffer) + e.packetSpliceUpload(w, packet.Buffer, packet.Destination) + e.flushPacketUploads() + } + packet.Buffer.Release() + N.PutPacketBuffer(packet) + } + for w.splice != nil { + select { + case packet := <-natConn.packetChan: + packet.Buffer = readOptions.Copy(packet.Buffer) + e.packetSpliceUpload(w, packet.Buffer, packet.Destination) + e.flushPacketUploads() + packet.Buffer.Release() + N.PutPacketBuffer(packet) + default: + e.packetSpliceRead(w) + return + } + } +} + +func (e *goEngine) packetSpliceRelease(splice *goSplicePacket, err error) { + splice.finished = true + N.ReleaseMultiPacketBuffer(splice.options.Cached) + splice.options.Cached = nil + splice.socket.close() + splice.owner.Detach() + splice.owner.Close() + if splice.options.OnClose != nil { + splice.options.OnClose(err) + } +} + +func (e *goEngine) handlePacketSpliceClose(w *GoPacketConn) { + var err error + closeError := w.spliceCloseError.Load() + if closeError != nil { + err = closeError.err + } + pending := w.splicePending.Swap(nil) + if pending != nil { + e.packetSpliceRelease(pending, err) + } + if w.splice == nil { + return + } + e.packetSpliceClose(w, err) +} + +func (e *goEngine) packetSpliceClose(w *GoPacketConn, err error) { + splice := w.splice + w.splice = nil + w.spliceActive.Store(false) + e.platformIO.unregisterSocket(&splice.socket) + e.releaseSpliceToken(splice.token) + natConn := w.conn.Load() + if natConn != nil { + natConn.Close() + } + e.packetSpliceRelease(splice, err) +} + +func (e *goEngine) releasePending(message *goMessage) { + switch message.kind { + case goMessageConnSplice: + pending := message.conn.splicePending.Swap(nil) + if pending != nil { + e.spliceRelease(pending, net.ErrClosed) + } + case goMessagePacketSplice: + pending := message.packet.splicePending.Swap(nil) + if pending != nil { + e.packetSpliceRelease(pending, net.ErrClosed) + } + case goMessageInject: + e.releaseInjected() + } +} + +func (e *goEngine) closeAllPacketSplices() { + for index := range e.spliceSlots { + packet := e.spliceSlots[index].packet + if packet != nil && packet.splice != nil { + e.packetSpliceClose(packet, net.ErrClosed) + } + } +} + +func (s *goSplicePacket) mapUpload(destination M.Socksaddr) M.Socksaddr { + if s.origin.IsValid() && destination.Addr == s.origin.Addr { + if !s.rewritePort { + return M.Socksaddr{Addr: s.destination.Addr, Fqdn: s.destination.Fqdn, Port: destination.Port} + } + if destination.Port == s.origin.Port { + return s.destination + } + } + if s.options.NAT.FakeIP && destination.IsIP() { + if s.direct == nil { + s.direct = make(map[netip.Addr]struct{}) + } + s.direct[destination.Addr] = struct{}{} + } + return destination +} + +func (s *goSplicePacket) mapDownload(source M.Socksaddr) M.Socksaddr { + if s.options.NAT.Unidirectional { + return source + } + if s.origin.IsValid() && source.Addr == s.destination.Addr && (s.destination.IsIP() || source.Fqdn == s.destination.Fqdn) { + if !s.rewritePort { + return M.Socksaddr{Addr: s.origin.Addr, Port: source.Port} + } + if source.Port == s.destination.Port { + return s.origin + } + } + if s.options.NAT.FakeIP { + _, direct := s.direct[source.Addr] + if !direct { + return M.Socksaddr{Addr: s.origin.Addr, Port: source.Port} + } + } + return source +} + +func (e *goEngine) packetSpliceUpload(w *GoPacketConn, buffer *buf.Buffer, destination M.Socksaddr) { + splice := w.splice + target := splice.mapUpload(destination) + payloadLength := buffer.Len() + if splice.options.Offload != nil { + err := splice.options.Offload.EncodePacket(buffer, target) + if err != nil { + e.packetSpliceClose(w, E.Cause(err, "go: encode packet")) + return + } + } + e.queuePacketUpload(splice, buffer.Bytes(), target.AddrPort(), payloadLength) +} + +func (e *goEngine) updatePacketHeadroom(options N.ReadWaitOptions) { + previous := *e.packetReadOptions.Load() + options.FrontHeadroom = max(previous.FrontHeadroom, options.FrontHeadroom-header.IPv4MinimumSize-header.UDPMinimumSize) + options.RearHeadroom = max(previous.RearHeadroom, options.RearHeadroom) + if options == previous { + return + } + e.flushPacketUploads() + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + continue + } + buffer := buf.NewSize(options.FrontHeadroom + goReassemblyCapacity + options.RearHeadroom) + copy(buffer.FreeBytes()[options.FrontHeadroom:], entry.buffer.Range(entry.headroom-goReassemblyHeadroom, entry.headroom+65535)) + entry.buffer.Release() + entry.buffer = buffer + entry.headroom = options.FrontHeadroom + goReassemblyHeadroom + } + clear(e.frames) + e.udpUserData.packet = nil + e.packetReadOptions.Store(&options) +} + +func (e *goEngine) packetSpliceRead(w *GoPacketConn) { + splice := w.splice + natConn := w.conn.Load() + batchSize := e.packetReceiveBatch + for index := range batchSize { + buffer := e.packetReceiveBuffers[index] + if buffer == nil { + buffer = buf.NewSize(65535) + e.packetReceiveBuffers[index] = buffer + } + buffer.Reset() + e.packetMessages[index] = goPacketMessage{data: buffer.FreeBytes()} + } + count, errno := e.packetIO.receive(&splice.socket, e.packetMessages[:batchSize], splice.connected) + if count == batchSize { + e.packetReceiveBatch = min(batchSize*2, goPacketBatchSize) + } else if count*2 < batchSize { + e.packetReceiveBatch = max(batchSize/2, goReceiveBatchMin) + } + defer func() { + e.flushPacketFrames() + clear(e.packetMessages[:]) + if errno != 0 && !goSocketWouldBlock(errno) && w.splice != nil { + e.packetSpliceClose(w, E.Cause(errno, "go: receive peer batch")) + } + }() + for index := range count { + message := e.packetMessages[index] + if message.truncated { + continue + } + buffer := e.packetReceiveBuffers[index] + offset := 0 + for { + length := len(message.data) - offset + if message.segmentSize > 0 { + length = min(length, message.segmentSize) + } + buffer.Resize(offset, length) + var destination M.Socksaddr + if splice.options.Offload != nil { + packetSource, err := splice.options.Offload.DecodePacket(buffer) + if err != nil { + e.packetSpliceClose(w, E.Cause(err, "go: decode packet")) + return + } + destination = splice.mapDownload(packetSource.Unwrap()) + } else { + source := message.destination + if !source.IsValid() { + source = splice.peer + } + destination = splice.mapDownload(M.SocksaddrFromNetIP(source)) + } + if destination.IsIP() && natConn.allowPeer(destination) { + goCount(splice.options.WriteCounters, buffer.Len()) + e.packetSpliceDownload(w, buffer.Bytes(), destination) + } + offset += length + if offset == len(message.data) { + break + } + } + } +} + +func (e *goEngine) flushPacketUploads() { + for e.packetDirtyList != nil { + splice := e.packetDirtyList + e.packetDirtyList = splice.nextDirty + splice.nextDirty = nil + count := 0 + for index := splice.uploadHead; index >= 0; index = e.packetUploads[index].next { + upload := &e.packetUploads[index] + e.packetMessages[count] = upload.message + count++ + } + splice.uploadHead = -1 + splice.uploadTail = -1 + if !splice.finished { + sent, errno := e.packetIO.send(&splice.socket, e.packetMessages[:count], splice.connected, splice.family) + length := 0 + for index := range sent { + length += e.packetMessages[index].payloadLength + } + goCount(splice.options.ReadCounters, length) + if errno != 0 && !goSocketWouldBlock(errno) && !goSocketDropped(errno) { + e.packetSpliceClose(splice.writer, E.Cause(errno, "go: send peer batch")) + } + } + clear(e.packetMessages[:count]) + } + clear(e.packetUploads[:e.packetUploadCount]) + e.packetUploadCount = 0 +} + +func (e *goEngine) queuePacketUpload(splice *goSplicePacket, data []byte, destination netip.AddrPort, length int) { + if e.packetUploadCount == len(e.packetUploads) { + e.flushPacketUploads() + if splice.finished { + return + } + } + index := e.packetUploadCount + e.packetUploadCount++ + e.packetUploads[index] = goPacketUpload{ + message: goPacketMessage{data: data, destination: destination, payloadLength: length}, + next: -1, + } + if splice.uploadHead < 0 { + splice.uploadHead = index + splice.nextDirty = e.packetDirtyList + e.packetDirtyList = splice + } else { + e.packetUploads[splice.uploadTail].next = index + } + splice.uploadTail = index +} + +func (e *goEngine) flushPacketFrames() { + if e.packetFrameCount == 0 { + return + } + err := e.platformIO.writePacketBatch(e.packetFrames[:e.packetFrameCount]) + if err != nil && err != errGoFrameDropped { + e.stack.logger.Trace(E.Cause(err, "go: write packet batch")) + } + clear(e.packetFrames[:e.packetFrameCount]) + e.packetFrameCount = 0 +} + +func (e *goEngine) packetSpliceDownload(w *GoPacketConn, data []byte, destination M.Socksaddr) { + if len(data)+w.templateLength > w.mtu { + e.flushPacketFrames() + buffer := buf.NewSize(w.FrontHeadroom() + len(data)) + buffer.Resize(w.FrontHeadroom(), 0) + copy(buffer.Extend(len(data)), data) + err := w.transmit(buffer, destination) + buffer.Release() + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write fragmented packet")) + } + return + } + frame := &e.packetFrames[e.packetFrameCount] + meta, err := w.preparePacketHeader(frame.header[:], data, destination, w.checksumOffload) + if err != nil { + if err != errGoFrameDropped { + e.stack.logger.Trace(err) + } + return + } + frame.length = w.templateLength + frame.payload = data + frame.meta = meta + e.packetFrameCount++ + if e.packetFrameCount == len(e.packetFrames) { + e.flushPacketFrames() + } +} diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go new file mode 100644 index 00000000..f68adff8 --- /dev/null +++ b/stack_go_tcp_input.go @@ -0,0 +1,2002 @@ +package tun + +import ( + "encoding/binary" + "hash/maphash" + "net" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" +) + +// goRTOFloorMicros is above the delayed acknowledgement timer of every kernel that can sit on +// the other side of the device: Linux holds a lone segment's ACK for 40 ms, Darwin for 100 ms and +// Windows for 200 ms, and Windows itself floors its own RTO at 300 ms. +const ( + goRTOFloorMicros = 300000 + goRTOInitialMicros = 1000000 + goRTOMaxMicros = 60000000 + goRTOGranularity = int32(goWheelTick / int64(time.Microsecond)) + goSynAckRetransmit = 500 * time.Millisecond + goSynAckAttempts = 5 + goPersistInitial = 500 * time.Millisecond + goPersistMax = 60 * time.Second + goFinLinger = 5 * time.Second + goAbortLinger = time.Second + goRetransmitLimit = 8 + goProbeFloorMicros = 50000 + goProbeDelayedAck = 200000 + goProbeNoSampleMicros = 1000000 + goProbeAttempts = 1 + goKeepaliveInterval = 75 * time.Second + goKeepaliveCount = 9 + goFinWait2Timeout = 60 * time.Second + goAckRampBytes = 256 << 10 + goDelayedAckInterval = 25 * time.Millisecond + goSweepInterval = 500 * time.Millisecond + goReclaimInterval = 60 * time.Second + goDyingLeakTimeout = 30 * time.Second + goMaxCongestionSize = 4 << 20 + goResetBurstLimit = 32 + goRecoveryBurst = 64 + goDupThresh = 3 + goInitialWindow = 10 + goAckCoalesceBytes = 256 << 10 +) + +const ( + goFRTOInactive uint8 = iota + goFRTOAwaitFirstAck + goFRTOAwaitSecondAck +) + +func (p *forwardPacket) isPureTCPSyn() bool { + return p.protocol == uint8(header.TCPProtocolNumber) && + p.tcpFlags&header.TCPFlagSyn != 0 && p.tcpFlags&header.TCPFlagAck == 0 +} + +func (e *goEngine) initialSequence(key flowKey, now int64) uint32 { + return uint32(maphash.Comparable(e.sequenceSeed, key)) + uint32(now/goTimeUnit) +} + +func (e *goEngine) demuxTCP(packet []byte, parsed *forwardPacket) { + if len(parsed.transport) < header.TCPMinimumSize { + return + } + if parsed.isPureTCPSyn() { + e.handleTCPSyn(parsed) + return + } + if parsed.tcpFlags&header.TCPFlagRst != 0 { + return + } + e.answerNoFlow(parsed) +} + +func (e *goEngine) answerNoFlow(parsed *forwardPacket) { + if e.resetBurst >= goResetBurstLimit { + return + } + e.resetBurst++ + e.controlIdent++ + reply, ok := goBuildNoFlowReset(e.controlScratch[:], parsed, uint16(e.controlIdent)) + if !ok { + return + } + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write reset")) + } +} + +func goBuildNoFlowReset(scratch []byte, parsed *forwardPacket, ident uint16) ([]byte, bool) { + tcpHdr := header.TCP(parsed.transport) + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return nil, false + } + ipHeaderLength := goNetworkHeaderLength(parsed.ipVersion) + packet := scratch[:ipHeaderLength+header.TCPMinimumSize] + replyTCP := header.TCP(packet[ipHeaderLength:]) + encodeResetTCP(replyTCP, tcpHdr) + pseudoSum := goEncodeNetworkHeader(packet, parsed.ipVersion, parsed.destination.Addr(), parsed.source.Addr(), header.TCPMinimumSize, ident) + replyTCP.SetChecksum(^replyTCP.CalculateChecksum(pseudoSum)) + return packet, true +} + +func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { + key := parsed.flowKey() + tcpHdr := header.TCP(parsed.transport) + clientISN := tcpHdr.SequenceNumber() + existing := e.flows[key] + if existing != nil { + if existing.clientISN == clientISN { + e.handleDuplicateSyn(existing) + return + } + if existing.connState.Load() < goConnStateDead && existing.state != goTCPSynReceived { + e.markAck(existing, true) + return + } + if existing.connState.Load() < goConnStateDead { + e.detachConn(existing, E.New("go: connection replaced by new SYN"), goDeathImmediate) + } + e.removeFlow(existing) + } + if len(e.flows) >= e.flowCapacity && !e.evictFlow() { + return + } + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return + } + synOptions := header.ParseSynOptions(tcpHdr[header.TCPMinimumSize:dataOffset], false) + now := e.now() + conn := new(GoConn) + source := M.SocksaddrFromNetIP(parsed.source) + destination := M.SocksaddrFromNetIP(parsed.destination) + conn.initialize(e, key, source, destination) + conn.ipVersion = parsed.ipVersion + conn.clientISN = clientISN + conn.sendISN = e.initialSequence(key, now) + conn.state = goTCPSynReceived + conn.sackPermitted = synOptions.SACKPermitted + if synOptions.WS >= 0 { + conn.localWindowShift = goLocalWindowShift + conn.peerWindowShift = uint8(synOptions.WS) + } + localMSS := e.localMSS(parsed.ipVersion) + conn.peerMSS = synOptions.MSS + conn.effectiveMSS = min(synOptions.MSS, localMSS) + if synOptions.TS { + conn.timestampsEnabled = true + conn.tsRecent.Store(synOptions.TSVal) + conn.effectiveMSS -= goTimestampOptionLength + } + conn.receiveCapacityMax = min(uint64(goReceiveCapacityMax), uint64(0xffff)<= goConnStateAborted { + e.removeFlow(conn) + return true + } + visited++ + if visited >= 64 { + break + } + } + for _, conn := range e.flows { + if conn.state == goTCPSynReceived { + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + e.removeFlow(conn) + return true + } + visited++ + if visited >= 128 { + break + } + } + return false +} + +func (e *goEngine) handleDuplicateSyn(conn *GoConn) { + switch { + case conn.connState.Load() == goConnStateEngaged: + err := conn.writeSynAck() + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) + } + case conn.connState.Load() >= goConnStateAborted && conn.deathClass == goDeathAbortLinger: + e.sendReset(conn) + } +} + +func (e *goEngine) inputTCP(conn *GoConn, parsed *forwardPacket) { + tcpHdr := header.TCP(parsed.transport) + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return + } + if conn.connState.Load() >= goConnStateAborted { + e.inputTombstone(conn, parsed) + return + } + conn.lastActivity = e.now() + payload := tcpHdr[dataOffset:] + flags := tcpHdr.Flags() + segOffset := conn.receiveOffset(tcpHdr.SequenceNumber()) + segmentLength := int64(len(payload)) + if flags&header.TCPFlagSyn != 0 { + segmentLength++ + } + if flags&header.TCPFlagFin != 0 { + segmentLength++ + } + if flags&header.TCPFlagRst != 0 { + e.handleReset(conn, segOffset) + return + } + if len(payload) > 0 && flags&header.TCPFlagSyn == 0 { + conn.recordDuplicate(segOffset, segOffset+int64(len(payload))) + } + var ( + timestampValue uint32 + timestampEcho uint32 + timestampRecent uint32 + hasTimestamp bool + ) + if conn.timestampsEnabled { + timestampValue, timestampEcho, hasTimestamp = goParseTimestampOption(tcpHdr[header.TCPMinimumSize:dataOffset]) + timestampRecent = conn.tsRecent.Load() + } + if hasTimestamp && int32(timestampValue-timestampRecent) < 0 { + e.markAck(conn, true) + return + } + if !conn.segmentAcceptable(segOffset, segmentLength) { + e.markAck(conn, true) + return + } + if hasTimestamp && timestampValue != timestampRecent && segOffset <= int64(conn.lastAckSent) { + conn.tsRecent.Store(timestampValue) + } + if flags&header.TCPFlagSyn != 0 { + segOffset++ + if len(payload) == 0 && flags&header.TCPFlagFin == 0 { + e.markAck(conn, true) + return + } + } + if flags&header.TCPFlagAck == 0 { + return + } + if !e.processAck(conn, tcpHdr, segOffset, len(payload), flags, timestampEcho, hasTimestamp) { + return + } + if conn.connState.Load() >= goConnStateAborted { + return + } + if len(payload) > 0 || flags&header.TCPFlagFin != 0 { + e.deliverSegment(conn, segOffset, payload, flags&header.TCPFlagFin != 0) + } + e.maybeSendFin(conn) + if conn.splice != nil { + e.spliceAfterInput(conn) + } +} + +func (e *goEngine) inputTombstone(conn *GoConn, parsed *forwardPacket) { + if parsed.tcpFlags&header.TCPFlagRst != 0 { + e.removeFlow(conn) + return + } + switch conn.deathClass { + case goDeathFinLinger: + if parsed.tcpFlags&header.TCPFlagFin != 0 { + e.sendAck(conn) + } + case goDeathAbortLinger: + e.sendReset(conn) + } +} + +func (e *goEngine) handleReset(conn *GoConn, segOffset int64) { + if segOffset == int64(conn.receiveNext) { + e.detachConn(conn, errGoReset, goDeathImmediate) + return + } + if conn.state != goTCPSynReceived && conn.segmentAcceptable(segOffset, 0) { + e.markAck(conn, true) + } +} + +func (c *GoConn) receiveOffset(sequence uint32) int64 { + current := uint32(uint64(c.clientISN) + c.receiveNext) + return int64(c.receiveNext) + int64(int32(sequence-current)) +} + +func (c *GoConn) recordDuplicate(start int64, end int64) { + if start < 0 || end <= start { + return + } + switch { + case uint64(end) <= c.receiveNext: + case uint64(start) < c.receiveNext: + end = int64(c.receiveNext) + case c.oooRanges != nil && c.oooRanges.covers(uint64(start), uint64(end)): + default: + return + } + c.dsackStart = uint64(start) + c.dsackEnd = uint64(end) +} + +func (c *GoConn) sendOffset(acknowledgement uint32) int64 { + unacked := c.sendUnacked.Load() + current := uint32(uint64(c.sendISN) + unacked) + return int64(unacked) + int64(int32(acknowledgement-current)) +} + +func (c *GoConn) segmentAcceptable(segOffset int64, segmentLength int64) bool { + receiveNext := int64(c.receiveNext) + edge := int64(c.publishedEdge) + if segmentLength == 0 { + if edge == receiveNext { + return segOffset == receiveNext + } + return segOffset >= receiveNext && segOffset <= edge + } + if edge == receiveNext { + return false + } + return segOffset < edge && segOffset+segmentLength > receiveNext +} + +func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, payloadLength int, flags header.TCPFlags, timestampEcho uint32, hasTimestamp bool) bool { + conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) + ackOffset := conn.sendOffset(tcpHdr.AckNumber()) + limit := int64(conn.sentTail.Load()) + if conn.finSent { + limit++ + } + if ackOffset > limit { + e.markAck(conn, true) + return false + } + unacked := int64(conn.sendUnacked.Load()) + segmentAck := ackOffset + if ackOffset < unacked { + ackOffset = unacked + } + windowRaw := tcpHdr.WindowSize() + handshakeAck := conn.state == goTCPSynReceived + if handshakeAck { + if conn.connState.Load() != goConnStateEngaged { + return false + } + if segmentAck != int64(conn.sentTail.Load()) { + e.sendReset(conn) + e.detachConn(conn, E.New("go: handshake ack out of range"), goDeathImmediate) + return false + } + e.establishConn(conn) + } + rawCount := 0 + if conn.sackPermitted { + rawCount = goParseSackBlocks(tcpHdr, int(tcpHdr.DataOffset()), conn, &e.sackRaw) + } + windowChanged := false + if segOffset > conn.windowLeft1 || (segOffset == conn.windowLeft1 && segmentAck >= conn.windowLeft2) { + windowChanged = windowRaw != conn.lastPeerWindow + conn.lastPeerWindow = windowRaw + conn.peerWindow = uint64(windowRaw) << conn.peerWindowShift + conn.windowLeft1 = segOffset + conn.windowLeft2 = segmentAck + } + newSack := false + if rawCount > 0 { + raw := e.sackRaw[:rawCount] + if goIsDuplicateSack(raw, ackOffset) { + e.applyDuplicateSack(conn, raw[0]) + raw = raw[1:] + } + sackCount := goSelectSackBlocks(raw, &e.sackScratch) + if sackCount > 0 { + newSack = e.applySackBlocks(conn, e.sackScratch[:sackCount]) + } + } + if ackOffset > unacked { + e.advanceUnacked(conn, uint64(ackOffset), timestampEcho, hasTimestamp) + } else if segmentAck == unacked && !handshakeAck && conn.hasOutstandingData() && payloadLength == 0 && flags&(header.TCPFlagSyn|header.TCPFlagFin) == 0 && !windowChanged && !newSack { + e.handleDuplicateAck(conn) + } + if conn.frtoState != goFRTOInactive { + e.stepFRTO(conn, uint64(ackOffset), uint64(unacked), newSack) + } + if windowChanged && ackOffset == unacked { + if conn.inRecovery { + e.retransmitHoles(conn) + } else if conn.frtoState == goFRTOInactive { + e.continueRetransmit(conn, uint64(ackOffset)) + } + } + e.updatePersist(conn) + conn.releaseIdleDescriptors() + conn.publishPermit(uint64(ackOffset)) + return true +} + +func (e *goEngine) establishConn(conn *GoConn) { + conn.state = goTCPEstablished + conn.handshakeDeadline = 0 + conn.everEstablished.Store(true) + conn.connState.CompareAndSwap(goConnStateEngaged, goConnStateEstablished) + close(conn.establishedSignal) + e.wokeHandlerThisBurst = true +} + +func (e *goEngine) advanceUnacked(conn *GoConn, ackOffset uint64, timestampEcho uint32, hasTimestamp bool) { + sent := conn.sentTail.Load() + previous := conn.sendUnacked.Load() + newUnacked := min(ackOffset, sent) + acked := newUnacked - previous + e.sampleRoundTrip(conn, newUnacked, timestampEcho, hasTimestamp) + conn.scoreboard.advance(newUnacked) + conn.releaseTransmitted(newUnacked) + conn.sendUnacked.Store(newUnacked) + if conn.finSent && ackOffset > conn.finOffset { + conn.finAcked = true + } + if conn.undoMarker != 0 && newUnacked >= conn.undoLimit && conn.frtoState == goFRTOInactive { + conn.undoMarker = 0 + } + if acked > 0 { + conn.duplicateAckCount = 0 + if conn.inRecovery { + if newUnacked >= conn.recoveryPoint { + conn.inRecovery = false + conn.congestionWindow = conn.slowStartThreshold + conn.pipe = 0 + } else { + conn.congestionWindow = max(conn.slowStartThreshold, uint32(conn.effectiveMSS)) + if conn.highestSacked <= newUnacked { + conn.highestRetransmit = newUnacked + e.retransmitFrom(conn, newUnacked) + } + e.retransmitHoles(conn) + } + } else { + conn.growCongestionWindow(acked) + if conn.frtoState == goFRTOInactive { + e.continueRetransmit(conn, newUnacked) + } + } + if conn.retransmitAttempts > 0 && conn.finSent && !conn.finAcked && !conn.hasOutstandingData() { + e.retransmitFrom(conn, newUnacked) + } + conn.retransmitAttempts = 0 + conn.probeAttempts = 0 + } + e.advanceFinState(conn) + e.rearmRetransmit(conn) +} + +func (c *GoConn) growCongestionWindow(acked uint64) { + mss := uint32(c.effectiveMSS) + counted := uint32(min(acked, uint64(2*mss))) + if c.congestionWindow < c.slowStartThreshold { + c.congestionWindow += counted + c.ackedBytes = 0 + } else { + c.ackedBytes += counted + if c.ackedBytes >= c.congestionWindow { + c.ackedBytes -= c.congestionWindow + c.congestionWindow += mss + } + } + c.congestionWindow = min(c.congestionWindow, goMaxCongestionSize) +} + +func (e *goEngine) sampleRoundTrip(conn *GoConn, acked uint64, timestampEcho uint32, hasTimestamp bool) { + var sample int32 + sampled := false + walked := false + slept := false + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.endOffset > acked { + break + } + walked = true + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + slept = true + } else if descriptor.flags&goDescriptorRetransmitted == 0 { + sample = descriptor.sentAt + sampled = true + } + } + now := e.now() + var measured int32 + switch { + case sampled: + units := int64(conn.stamp(now) - sample) + if units < 0 { + return + } + measured = int32(min(units*goTimeUnit/1000, goRTOMaxMicros)) + case hasTimestamp && walked && !slept && timestampEcho != 0: + elapsed := goTimestampAt(now) - timestampEcho + if int32(elapsed) < 0 { + return + } + measured = int32(min(int64(elapsed)< unacked { + return + } + sent := conn.sentTail.Load() + flight := sent - unacked + if flight == 0 { + return + } + conn.armCongestionUndo() + conn.slowStartThreshold = max(uint32(flight/2), 2*uint32(conn.effectiveMSS)) + conn.congestionWindow = conn.slowStartThreshold + conn.inRecovery = true + conn.recoveryPoint = sent + conn.highestRetransmit = unacked + conn.pipe = flight + conn.retransmitPoint = 0 + conn.duplicateAckCount = 0 + e.retransmitFrom(conn, unacked) + e.retransmitHoles(conn) +} + +func (c *GoConn) armCongestionUndo() { + if c.undoMarker != 0 { + return + } + c.undoMarker = c.sendUnacked.Load() + c.undoLimit = c.sentTail.Load() + c.undoCongestionWindow = c.congestionWindow + c.undoSlowStartThreshold = c.slowStartThreshold + c.undoRetransmits = 0 +} + +func (e *goEngine) applyDuplicateSack(conn *GoConn, block goRawSackBlock) { + if conn.undoMarker == 0 || block.start < int64(conn.undoMarker) { + return + } + conn.undoRetransmits-- + if conn.undoRetransmits > 0 { + return + } + conn.undoCongestionReduction() +} + +func (c *GoConn) undoCongestionReduction() { + c.undoMarker = 0 + c.slowStartThreshold = c.undoSlowStartThreshold + c.congestionWindow = max(c.congestionWindow, c.undoCongestionWindow) + c.inRecovery = false + c.duplicateAckCount = 0 + c.retransmitPoint = 0 + c.pipe = 0 +} + +func goParseTimestampOption(options []byte) (uint32, uint32, bool) { + for index := 0; index < len(options); { + switch options[index] { + case header.TCPOptionEOL: + return 0, 0, false + case header.TCPOptionNOP: + index++ + case header.TCPOptionTS: + if index+header.TCPOptionTSLength > len(options) || options[index+1] != header.TCPOptionTSLength { + return 0, 0, false + } + return binary.BigEndian.Uint32(options[index+2:]), binary.BigEndian.Uint32(options[index+6:]), true + default: + if index+2 > len(options) { + return 0, 0, false + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) { + return 0, 0, false + } + index += optionLength + } + } + return 0, 0, false +} + +type goRawSackBlock struct { + start int64 + end int64 +} + +func goParseSackBlocks(tcpHdr header.TCP, dataOffset int, conn *GoConn, blocks *[goMaxSackBlocks]goRawSackBlock) int { + options := tcpHdr[header.TCPMinimumSize:dataOffset] + count := 0 + for index := 0; index < len(options); { + switch options[index] { + case header.TCPOptionEOL: + return count + case header.TCPOptionNOP: + index++ + case header.TCPOptionSACK: + if index+2 > len(options) { + return count + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) || (optionLength-2)%8 != 0 { + return count + } + for offset := index + 2; offset+8 <= index+optionLength && count < goMaxSackBlocks; offset += 8 { + start := conn.sendOffset(binary.BigEndian.Uint32(options[offset:])) + end := conn.sendOffset(binary.BigEndian.Uint32(options[offset+4:])) + if end <= start { + continue + } + blocks[count] = goRawSackBlock{start: start, end: end} + count++ + } + index += optionLength + default: + if index+2 > len(options) { + return count + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) { + return count + } + index += optionLength + } + } + return count +} + +func goIsDuplicateSack(blocks []goRawSackBlock, ackOffset int64) bool { + first := blocks[0] + if first.end <= ackOffset { + return true + } + for _, block := range blocks[1:] { + if block.start <= first.start && first.end <= block.end { + return true + } + } + return false +} + +func goSelectSackBlocks(raw []goRawSackBlock, blocks *[goMaxSackBlocks]goSackBlock) int { + count := 0 + for _, block := range raw { + if block.start < 0 { + continue + } + blocks[count] = goSackBlock{start: uint64(block.start), end: uint64(block.end)} + count++ + } + return count +} + +func (e *goEngine) applySackBlocks(conn *GoConn, blocks []goSackBlock) bool { + unacked := conn.sendUnacked.Load() + updated := false + sackedBytes := uint64(0) + sackedRuns := 0 + previousSacked := false + start := unacked + scoreboard := &conn.scoreboard + for index := 0; index < len(scoreboard.entries); index++ { + descriptor := &scoreboard.entries[index] + end := descriptor.endOffset + if descriptor.flags&goDescriptorSacked == 0 { + for _, block := range blocks { + if block.end <= start || block.start >= end { + continue + } + if block.start > start { + scoreboard.split(index, block.start) + index++ + descriptor = &scoreboard.entries[index] + start = block.start + } + if block.end < end { + scoreboard.split(index, block.end) + descriptor = &scoreboard.entries[index] + end = block.end + } + descriptor.flags |= goDescriptorSacked + updated = true + break + } + } + if descriptor.flags&goDescriptorSacked == 0 { + previousSacked = false + } else { + sackedBytes += end - start + if !previousSacked { + sackedRuns++ + } + previousSacked = true + } + start = end + } + for _, block := range blocks { + if block.end > conn.highestSacked { + conn.highestSacked = block.end + } + } + if conn.inRecovery { + if updated { + e.retransmitHoles(conn) + } + return updated + } + if sackedBytes > uint64(goDupThresh-1)*uint64(conn.effectiveMSS) || sackedRuns >= goDupThresh { + e.enterRecovery(conn) + } + return updated +} + +func (c *GoConn) lostEdge(unacked uint64) uint64 { + threshold := uint64(goDupThresh-1) * uint64(c.effectiveMSS) + sackedBytes := uint64(0) + sackedRuns := 0 + previousSacked := false + entries := c.scoreboard.entries + for index := len(entries); index > 0; { + index-- + descriptor := &entries[index] + if descriptor.flags&goDescriptorSacked == 0 { + previousSacked = false + if sackedBytes > threshold || sackedRuns >= goDupThresh { + return descriptor.endOffset + } + continue + } + sackedBytes += descriptor.endOffset - c.scoreboard.startOf(index, unacked) + if !previousSacked { + sackedRuns++ + } + previousSacked = true + } + return 0 +} + +func (c *GoConn) estimatePipe(unacked uint64, lostEdge uint64) uint64 { + pipe := uint64(0) + start := unacked + for index := range c.scoreboard.entries { + descriptor := &c.scoreboard.entries[index] + end := descriptor.endOffset + if descriptor.flags&goDescriptorSacked == 0 { + if start >= lostEdge { + pipe += end - start + } + if start < c.highestRetransmit { + pipe += min(end, c.highestRetransmit) - start + } + } + start = end + } + sent := c.sentTail.Load() + if sent > start { + pipe += sent - start + } + return pipe +} + +func (e *goEngine) retransmitHoles(conn *GoConn) { + unacked := conn.sendUnacked.Load() + lostEdge := conn.lostEdge(unacked) + conn.pipe = conn.estimatePipe(unacked, lostEdge) + if lostEdge <= unacked { + return + } + window := uint64(conn.congestionWindow) + mss := uint64(conn.effectiveMSS) + sent := 0 + start := unacked + for index := 0; index < len(conn.scoreboard.entries); index++ { + descriptor := &conn.scoreboard.entries[index] + end := descriptor.endOffset + if start >= lostEdge { + return + } + if descriptor.flags&goDescriptorSacked != 0 || end <= conn.highestRetransmit { + start = end + continue + } + for offset := max(start, conn.highestRetransmit); offset < end; sent++ { + if sent == goRecoveryBurst { + conn.pipe = max(conn.pipe, window) + return + } + if conn.pipe+mss > window { + return + } + length := int(min(end-offset, mss)) + length = e.transmitRetransmit(conn, offset, length) + if length == 0 { + return + } + offset += uint64(length) + conn.pipe += uint64(length) + } + start = end + } +} + +func (c *GoConn) publishPermit(ackOffset uint64) { + unacked := c.sendUnacked.Load() + limit := unacked + uint64(c.congestionWindow) + switch { + case c.frtoState == goFRTOAwaitSecondAck: + limit = c.frtoSendLimit + case c.inRecovery: + limit = c.sentTail.Load() + if uint64(c.congestionWindow) > c.pipe { + limit += uint64(c.congestionWindow) - c.pipe + } + } + permit := max(min(limit, ackOffset+c.peerWindow), unacked) + c.sendPermit.Store(permit) + c.wakeTransmitter() + c.wakeWriter() +} + +func (c *GoConn) releaseTransmitted(unacked uint64) { + edge := min(unacked, c.transmittedTail.Load()) + if edge <= c.sendReleased.Load() { + return + } + c.transmitStore.releaseBelow(edge) + if edge == c.bufferedTail.Load() { + c.transmitStore.releaseDrained(edge) + } + c.sendReleased.Store(edge) +} + +func (c *GoConn) releaseIdleDescriptors() { + if len(c.scoreboard.entries) != 0 || !c.transmitOwner.CompareAndSwap(0, 1) { + return + } + if c.sendUnacked.Load() == c.sentTail.Load() && !c.blockedValid { + c.descriptors.release() + } + c.transmitOwner.Store(0) + // A writer can publish data while we own the transmitter. Resume it if its + // attempt to transmit raced with this cleanup. + if c.nextTransmitLength() != 0 { + c.wakeTransmitter() + } +} + +func (c *GoConn) releaseDrainedSlabs() { + consumed := c.consumedTail.Load() + if consumed == c.receiveAvailable.Load() && (c.oooRanges == nil || c.oooRanges.count == 0) { + c.receiveChain.releaseDrained(consumed) + } + buffered := c.bufferedTail.Load() + if min(c.sendUnacked.Load(), c.transmittedTail.Load()) == buffered { + c.transmitStore.releaseDrained(buffered) + } +} + +func (c *GoConn) wakeTransmitter() { + if c.splice != nil { + c.engine.spliceResume(c) + return + } + if c.nextTransmitLength() == 0 { + return + } + if goEngineInlineTransmit && c.transmitOnEngine() { + return + } + c.wakeTransmitterGoroutine() + c.engine.wokeHandlerThisBurst = true +} + +func (c *GoConn) wakeWriter() { + if !c.writerParked.Load() { + return + } + if !c.writeReady(int(max(c.writerNeeds.Load(), 1))) { + return + } + c.signalWriter() +} + +func (c *GoConn) signalWriter() { + if c.writeSignal.notify() { + c.engine.wokeHandlerThisBurst = true + } +} + +func (c *GoConn) signalReader() { + if c.readSignal.notify() { + c.engine.wokeHandlerThisBurst = true + } +} + +func (e *goEngine) deliverSegment(conn *GoConn, segOffset int64, payload []byte, fin bool) { + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + receiveNext := int64(conn.receiveNext) + edge := int64(conn.publishedEdge) + start := max(segOffset, receiveNext) + end := min(segOffset+int64(len(payload)), edge) + finOffset := segOffset + int64(len(payload)) + if end <= start { + if fin && finOffset == receiveNext { + e.acceptFin(conn) + return + } + e.markAck(conn, true) + return + } + data := payload[start-segOffset : end-segOffset] + if start == receiveNext { + e.deliverInOrder(conn, data) + if fin && finOffset == int64(conn.receiveNext) { + e.acceptFin(conn) + return + } + e.mergeOutOfOrder(conn) + e.measureReceiveRoundTrip(conn) + conn.ackPending += uint64(len(data)) + conn.ackCovered += uint64(len(data)) + e.markAck(conn, conn.ackPending >= conn.ackThreshold()) + return + } + if conn.discardReceive { + e.markAck(conn, true) + return + } + if conn.oooRanges == nil { + conn.oooRanges = new(goRangeSet) + } + if conn.oooRanges.bytes()+uint64(len(data)) > conn.receiveCapacity { + e.markAck(conn, true) + return + } + conn.receiveChain.reserve(uint64(start), len(data)) + conn.receiveChain.writeAt(uint64(start), data) + if !conn.oooRanges.insert(uint64(start), uint64(end), fin && finOffset == end) { + e.markAck(conn, true) + return + } + e.markAck(conn, true) +} + +func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { + if conn.discardReceive { + conn.receiveNext += uint64(len(data)) + conn.receiveNextAck.Store(conn.receiveNext) + conn.publishReceiveWindow() + return + } + if conn.splice != nil { + e.spliceDeliver(conn, data) + return + } + conn.receiveChain.reserve(conn.receiveNext, len(data)) + direct := 0 + taken := false + if conn.receiveAvailable.Load() == conn.consumedTail.Load() { + target := conn.postedTarget.Swap(nil) + if target != nil { + direct = copy(target.target, data) + target.filled = direct + taken = true + } + } + if direct < len(data) { + conn.receiveChain.writeAt(conn.receiveNext+uint64(direct), data[direct:]) + } + conn.receiveNext += uint64(len(data)) + conn.receiveNextAck.Store(conn.receiveNext) + conn.receiveAvailable.Store(conn.receiveNext) + conn.publishReceiveWindow() + if taken { + conn.targetDone.Store(true) + } + if conn.readerParked.Load() || taken { + conn.signalReader() + } +} + +func (e *goEngine) mergeOutOfOrder(conn *GoConn) { + if conn.oooRanges == nil { + return + } + for { + front, ok := conn.oooRanges.first() + if !ok || front.start > conn.receiveNext { + return + } + if front.end > conn.receiveNext { + conn.receiveNext = front.end + conn.receiveNextAck.Store(conn.receiveNext) + if !conn.discardReceive { + conn.receiveAvailable.Store(conn.receiveNext) + if conn.readerParked.Load() { + conn.signalReader() + } + } + conn.publishReceiveWindow() + } + fin := front.fin + conn.oooRanges.removeBelow(conn.receiveNext) + e.markAck(conn, true) + if fin { + e.acceptFin(conn) + return + } + } +} + +func (e *goEngine) acceptFin(conn *GoConn) { + if conn.finReceived { + e.markAck(conn, true) + return + } + conn.finReceived = true + conn.receiveNext++ + conn.receiveNextAck.Store(conn.receiveNext) + conn.receiveShutdown.Store(true) + conn.publishReceiveWindow() + if conn.readerParked.Load() { + conn.signalReader() + } + switch conn.state { + case goTCPEstablished: + conn.state = goTCPCloseWait + case goTCPFinWait1: + if conn.finAcked { + conn.state = goTCPFinWait2 + conn.finWait2Since = e.now() + e.markAck(conn, true) + e.finishClose(conn) + return + } + conn.state = goTCPClosing + case goTCPFinWait2: + e.markAck(conn, true) + e.finishClose(conn) + return + } + e.markAck(conn, true) +} + +func (e *goEngine) advanceFinState(conn *GoConn) { + if !conn.finSent || !conn.finAcked { + return + } + switch conn.state { + case goTCPFinWait1: + if conn.finReceived { + e.finishClose(conn) + return + } + conn.state = goTCPFinWait2 + conn.finWait2Since = e.now() + case goTCPClosing, goTCPLastAck: + e.finishClose(conn) + } +} + +func (e *goEngine) finishClose(conn *GoConn) { + conn.state = goTCPClosed + e.detachConn(conn, nil, goDeathFinLinger) +} + +func (c *GoConn) publishReceiveWindow() { + base := c.consumedTail.Load() + if c.discardReceive { + base = c.receiveNext + } + edge := base + c.receiveCapacity + if edge <= c.publishedEdge { + return + } + if base < c.receiveNext && edge-c.sentEdge.Load() < c.windowUpdateThreshold.Load() { + return + } + c.publishedEdge = edge + c.receiveEdge.Store(edge) +} + +func (e *goEngine) measureReceiveRoundTrip(conn *GoConn) { + now := e.now() + if conn.receiveRoundTripMark == 0 { + conn.receiveRoundTripMark = conn.receiveNext + conn.receiveCapacity + conn.receiveRoundTripStamp = now + return + } + if conn.receiveNext < conn.receiveRoundTripMark { + return + } + sample := max(int32(min((now-conn.receiveRoundTripStamp)/int64(time.Microsecond), goRTOMaxMicros)), 1) + if conn.receiveRoundTrip == 0 || sample < conn.receiveRoundTrip { + conn.receiveRoundTrip = sample + } + conn.receiveRoundTripMark = conn.receiveNext + conn.receiveCapacity + conn.receiveRoundTripStamp = now +} + +func (e *goEngine) moderateReceiveCapacity(conn *GoConn) { + if conn.receiveRoundTrip == 0 { + return + } + now := e.now() + if now-conn.receiveSpaceStamp < int64(conn.receiveRoundTrip)*int64(time.Microsecond) { + return + } + consumed := conn.consumedTail.Load() + copied := consumed - conn.receiveSpaceConsumed + previous := conn.receiveSpaceCopied + conn.receiveSpaceConsumed = consumed + conn.receiveSpaceCopied = copied + conn.receiveSpaceStamp = now + capacity := min(max(2*max(copied, previous), conn.receiveCapacity), e.receiveCapacityLimit(conn)) + if capacity == conn.receiveCapacity { + return + } + conn.receiveCapacity = capacity + conn.receiveCapacityPublished.Store(capacity) + conn.windowUpdateThreshold.Store(min(uint64(conn.effectiveMSS), capacity/2)) +} + +func (e *goEngine) receiveCapacityLimit(conn *GoConn) uint64 { + share := e.slabPool.shareBytes() + if share == 0 { + return conn.receiveCapacityMax + } + transmitHeld := conn.transmitStore.chain.heldBytes() + if share <= transmitHeld { + return min(goReceiveCapacityBase, conn.receiveCapacityMax) + } + return min(max(share-transmitHeld, goReceiveCapacityBase), conn.receiveCapacityMax) +} + +func (e *goEngine) markAck(conn *GoConn, forced bool) { + if forced { + conn.ackForced = true + } + if conn.ackDirty { + return + } + conn.ackDirty = true + conn.ackNext = e.ackList + e.ackList = conn +} + +func (e *goEngine) expireDelayedAckTick(now int64) { + e.drainAckList(now, false) +} + +func (e *goEngine) drainAckList(now int64, allowDefer bool) { + var kept *GoConn + for conn := e.ackList; conn != nil; { + next := conn.ackNext + conn.ackNext = nil + if allowDefer && !conn.ackForced && conn.ackPending < conn.ackThreshold() && conn.ackCovered >= goAckRampBytes { + conn.ackNext = kept + kept = conn + conn = next + continue + } + conn.ackDirty = false + if !e.sendAck(conn) { + conn.ackDirty = true + conn.ackNext = kept + kept = conn + } + conn = next + } + e.ackList = kept + if kept != nil && e.delayedAckTickNode.slot == nil { + e.wheel.schedule(&e.delayedAckTickNode, now+int64(goDelayedAckInterval)) + } +} + +func (e *goEngine) sendAck(conn *GoConn) bool { + e.moderateReceiveCapacity(conn) + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + conn.publishReceiveWindow() + + segment := goSegment{offset: conn.sendNext(), flags: header.TCPFlagAck} + if conn.sackPermitted && (conn.dsackEnd != 0 || conn.oooRanges != nil && conn.oooRanges.count > 0) { + count := 0 + if conn.dsackEnd != 0 { + e.sackScratch[0] = goSackBlock{start: conn.dsackStart, end: conn.dsackEnd} + count = 1 + } + if conn.oooRanges != nil { + count = conn.oooRanges.blocks(&e.sackScratch, count) + } + segment.sackBlocks = e.sackScratch[:count] + } + if !e.writeConnControl(conn, &segment) { + return false + } + conn.dsackStart = 0 + conn.dsackEnd = 0 + conn.ackPending = 0 + conn.ackForced = false + conn.lastAckSent = conn.receiveNext + return true +} + +func (e *goEngine) sendReset(conn *GoConn) bool { + segment := goSegment{offset: conn.sendNext(), flags: header.TCPFlagRst | header.TCPFlagAck} + return e.writeConnControl(conn, &segment) +} + +func (e *goEngine) writeConnControl(conn *GoConn, segment *goSegment) bool { + frame, _ := conn.buildFrame(e.controlScratch[:], e.controlSegments[:0], segment, &conn.transmitStore, false) + e.controlSegments = frame + err := e.platformIO.writeFrame(frame, ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write control frame")) + return false + } + return true +} + +func (e *goEngine) maybeSendFin(conn *GoConn) { + if !conn.finPending || conn.finSent || conn.state == goTCPSynReceived { + return + } + if conn.connState.Load() >= goConnStateAborted { + return + } + if conn.writerActive.Load() != 0 { + return + } + buffered := conn.bufferedTail.Load() + if conn.sentTail.Load() != buffered || conn.transmittedTail.Load() != buffered { + return + } + conn.finOffset = buffered + conn.finSent = true + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + e.writeConnControl(conn, &segment) + switch conn.state { + case goTCPCloseWait: + conn.state = goTCPLastAck + default: + conn.state = goTCPFinWait1 + } + e.rearmRetransmit(conn) +} + +func (e *goEngine) handleCloseRequest(conn *GoConn) { + if conn.connState.Load() >= goConnStateDead { + if conn.userClosed.Load() || conn.closeMode.Load() == goCloseModeAbort { + conn.receiveDrainable.Store(false) + e.spliceDetach(conn, conn.loadError()) + } + return + } + if conn.connState.Load() == goConnStateAborted { + e.detachConn(conn, nil, goDeathAbortLinger) + return + } + if conn.closeMode.Load() == goCloseModeAbort || conn.splice != nil || conn.splicePending.Load() != nil { + e.sendReset(conn) + e.detachConn(conn, net.ErrClosed, goDeathAbortLinger) + return + } + conn.finPending = true + e.maybeSendFin(conn) +} + +func (e *goEngine) handleReadShut(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + return + } + conn.discardReceive = true + conn.oooRanges = nil + if conn.readAccess.TryLock() { + conn.consumedTail.Store(conn.receiveAvailable.Load()) + conn.readAccess.Unlock() + conn.releaseDrainedSlabs() + } + conn.publishReceiveWindow() + if conn.readerParked.Load() { + conn.signalReader() + } +} + +func (e *goEngine) handleWindowUpdate(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + return + } + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + conn.publishReceiveWindow() + if conn.publishedEdge > conn.sentEdge.Load() || conn.ackPending > 0 { + e.markAck(conn, true) + } +} + +func (e *goEngine) handleRetransmitArm(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + return + } + e.rearmRetransmit(conn) +} + +func (e *goEngine) handleDroppedFrames(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + return + } + conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) + unacked := conn.sendUnacked.Load() + start := unacked + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + end := descriptor.endOffset + if descriptor.flags&goDescriptorDropped != 0 && descriptor.flags&goDescriptorSacked == 0 { + if e.transmitRetransmit(conn, start, int(end-start)) == 0 { + break + } + } + start = end + } + e.rearmRetransmit(conn) +} + +func (e *goEngine) handleTransmitBlocked(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + conn.transmitSignal.notify() + return + } + if !conn.onBlockedList { + conn.onBlockedList = true + conn.blockedNext = e.blockedList + e.blockedList = conn + } + armed, err := e.platformIO.armTransmitWritable() + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: arm transmit writable")) + } + if !armed { + e.releaseBlockedWriters() + } +} + +func (e *goEngine) releaseBlockedWriters() { + blocked := e.blockedList + e.blockedList = nil + for conn := blocked; conn != nil; { + next := conn.blockedNext + conn.blockedNext = nil + conn.onBlockedList = false + if conn.splice != nil { + e.spliceRetryBlocked(conn) + } else { + conn.transmitSignal.notify() + e.wokeHandlerThisBurst = true + } + conn = next + } +} + +func (c *GoConn) hasOutstandingData() bool { + return c.sendUnacked.Load() < c.sentTail.Load() +} + +func (c *GoConn) hasOutstanding() bool { + if c.state == goTCPSynReceived { + return false + } + return c.sendUnacked.Load() < c.sentTail.Load() || (c.finSent && !c.finAcked) +} + +func (e *goEngine) rearmRetransmit(conn *GoConn) { + if !conn.hasOutstanding() { + conn.retransmitDeadline = 0 + conn.retransmitArmed.Store(false) + if !conn.hasOutstanding() { + e.rearmTimer(conn) + return + } + conn.retransmitArmed.Store(true) + } + if conn.persistNeeded() { + e.updatePersist(conn) + return + } + conn.retransmitDeadline = e.now() + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.armProbe(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) armProbe(conn *GoConn) { + if conn.probeAttempts >= goProbeAttempts || !conn.hasOutstanding() || conn.frtoState != goFRTOInactive || conn.inRecovery { + conn.probeDeadline = 0 + return + } + timeout := int32(goProbeNoSampleMicros) + if conn.smoothedRoundTrip > 0 { + timeout = max(2*conn.smoothedRoundTrip, goProbeFloorMicros) + } + if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS) { + timeout += goProbeDelayedAck + } + if timeout >= conn.retransmitTimeout { + conn.probeDeadline = 0 + return + } + conn.probeDeadline = e.now() + int64(timeout)*int64(time.Microsecond) +} + +func (c *GoConn) persistNeeded() bool { + return c.peerWindow == 0 && (c.bufferedTail.Load() > c.sendUnacked.Load() || (c.finSent && !c.finAcked)) +} + +func (e *goEngine) updatePersist(conn *GoConn) { + if conn.persistNeeded() { + conn.retransmitDeadline = 0 + conn.probeDeadline = 0 + if conn.persistDeadline == 0 { + conn.persistAttempts = 0 + conn.persistDeadline = e.now() + int64(goPersistInitial) + } + e.rearmTimer(conn) + return + } + if conn.persistDeadline != 0 { + conn.persistDeadline = 0 + conn.persistAttempts = 0 + if conn.hasOutstanding() { + e.retransmitFrom(conn, conn.sendUnacked.Load()) + e.rearmRetransmit(conn) + } else { + e.rearmTimer(conn) + } + } +} + +func (e *goEngine) rearmTimer(conn *GoConn) { + next := int64(0) + for _, deadline := range [5]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline} { + if deadline == 0 { + continue + } + if next == 0 || deadline < next { + next = deadline + } + } + if next == 0 { + e.wheel.cancel(&conn.timerNode) + return + } + e.wheel.schedule(&conn.timerNode, next) +} + +func (c *GoConn) expireTimer(now int64) { + c.engine.fireConnTimer(c, now) +} + +func (e *goEngine) fireConnTimer(conn *GoConn, now int64) { + if conn.lingerDeadline != 0 && conn.lingerDeadline <= now { + conn.lingerDeadline = 0 + e.removeFlow(conn) + return + } + if conn.handshakeDeadline != 0 && conn.handshakeDeadline <= now { + e.expireHandshake(conn, now) + return + } + if conn.persistDeadline != 0 && conn.persistDeadline <= now { + e.expirePersist(conn, now) + return + } + if conn.probeDeadline != 0 && conn.probeDeadline <= now { + e.expireProbe(conn, now) + return + } + if conn.retransmitDeadline != 0 && conn.retransmitDeadline <= now { + e.expireRetransmit(conn, now) + return + } + e.rearmTimer(conn) +} + +func (e *goEngine) expireHandshake(conn *GoConn, now int64) { + conn.handshakeAttempts++ + if conn.handshakeAttempts >= goSynAckAttempts { + conn.handshakeDeadline = 0 + e.detachConn(conn, E.New("go: handshake timeout"), goDeathAbortLinger) + return + } + err := conn.writeSynAck() + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) + } + conn.handshakeDeadline = now + int64(goSynAckRetransmit)< unacked: + offset := unacked + if sent-unacked > uint64(conn.effectiveMSS) { + offset = sent - uint64(conn.effectiveMSS) + } + e.transmitRetransmit(conn, offset, int(sent-offset)) + case conn.finSent && !conn.finAcked: + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + e.writeConnControl(conn, &segment) + } + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.armProbe(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { + if !conn.hasOutstanding() { + conn.retransmitDeadline = 0 + conn.retransmitArmed.Store(false) + e.rearmTimer(conn) + return + } + if conn.persistNeeded() { + e.updatePersist(conn) + return + } + conn.probeDeadline = 0 + conn.probeAttempts = 0 + conn.retransmitAttempts++ + if conn.retransmitAttempts >= goRetransmitLimit && conn.persistDeadline == 0 { + conn.retransmitDeadline = 0 + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: retransmit limit"), goDeathImmediate) + return + } + conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) + unacked := conn.sendUnacked.Load() + sent := conn.sentTail.Load() + recovering := conn.inRecovery || (conn.retransmitPoint > unacked && conn.frtoState != goFRTOAwaitFirstAck) + flight := sent - unacked + if flight > 0 { + conn.armCongestionUndo() + conn.slowStartThreshold = max(uint32(flight/2), 2*uint32(conn.effectiveMSS)) + } + conn.congestionWindow = uint32(conn.effectiveMSS) + conn.inRecovery = false + conn.duplicateAckCount = 0 + conn.highestRetransmit = unacked + conn.pipe = 0 + conn.retransmitPoint = sent + conn.retransmitHighWater = unacked + conn.highestSacked = unacked + for index := range conn.scoreboard.entries { + conn.scoreboard.entries[index].flags &^= goDescriptorSacked + } + if conn.sackPermitted && !recovering && flight > 0 { + conn.frtoState = goFRTOAwaitFirstAck + } else { + conn.frtoState = goFRTOInactive + } + e.continueRetransmit(conn, unacked) + if !conn.hasOutstandingData() { + e.retransmitFrom(conn, unacked) + } + conn.retransmitTimeout = min(conn.retransmitTimeout*2, goRTOMaxMicros) + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + conn.publishPermit(unacked) +} + +func (e *goEngine) stepFRTO(conn *GoConn, ackOffset uint64, previousUnacked uint64, newSack bool) { + advanced := ackOffset > previousUnacked + switch conn.frtoState { + case goFRTOAwaitFirstAck: + if !advanced { + return + } + conn.frtoRecoveryPoint = conn.sentTail.Load() + conn.frtoSendLimit = conn.frtoRecoveryPoint + 2*uint64(conn.effectiveMSS) + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + if ackOffset < conn.retransmitHighWater || ackOffset >= conn.frtoRecoveryPoint || !conn.hasDataWaiting() || windowEnd < conn.frtoSendLimit { + e.abandonFRTO(conn, 2) + return + } + conn.frtoState = goFRTOAwaitSecondAck + case goFRTOAwaitSecondAck: + if !advanced && !newSack { + return + } + if ackOffset > conn.frtoRecoveryPoint || conn.highestSacked > conn.frtoRecoveryPoint { + e.abandonFRTO(conn, 3) + return + } + conn.frtoState = goFRTOInactive + conn.retransmitHighWater = 0 + conn.undoCongestionReduction() + } +} + +func (e *goEngine) abandonFRTO(conn *GoConn, segments uint32) { + conn.frtoState = goFRTOInactive + conn.congestionWindow = min(conn.congestionWindow, segments*uint32(conn.effectiveMSS)) + e.continueRetransmit(conn, conn.sendUnacked.Load()) +} + +func (e *goEngine) continueRetransmit(conn *GoConn, unacked uint64) { + if conn.retransmitPoint <= unacked { + conn.retransmitPoint = 0 + return + } + offset := max(unacked, conn.retransmitHighWater) + limit := min(conn.retransmitPoint, unacked+uint64(conn.congestionWindow)) + entries := conn.scoreboard.entries + index := 0 + for sent := 0; offset < limit && sent < goRecoveryBurst; { + for index < len(entries) && entries[index].endOffset <= offset { + index++ + } + end := limit + if index < len(entries) { + descriptor := &entries[index] + if descriptor.flags&goDescriptorSacked != 0 { + offset = min(descriptor.endOffset, limit) + continue + } + end = min(end, descriptor.endOffset) + } + length := int(min(end-offset, uint64(conn.effectiveMSS))) + length = e.transmitRetransmit(conn, offset, length) + if length == 0 { + break + } + offset += uint64(length) + sent++ + } + conn.retransmitHighWater = offset +} + +func (e *goEngine) retransmitFrom(conn *GoConn, offset uint64) { + sent := conn.sentTail.Load() + offset = max(offset, 1) + if offset < sent { + e.transmitRetransmit(conn, offset, int(min(sent-offset, uint64(conn.effectiveMSS)))) + return + } + if conn.finSent && !conn.finAcked { + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + e.writeConnControl(conn, &segment) + } +} + +func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int) int { + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + limit := min(conn.sentTail.Load(), windowEnd) + if conn.peerWindow == 0 || offset >= limit { + return 0 + } + length = int(min(uint64(length), uint64(conn.effectiveMSS), limit-offset)) + segment := goSegment{offset: offset, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} + if !e.writeConnControl(conn, &segment) { + return 0 + } + end := offset + uint64(length) + if end > conn.highestRetransmit { + conn.highestRetransmit = end + } + if conn.undoMarker != 0 && offset >= conn.undoMarker { + conn.undoRetransmits++ + } + e.markRetransmitted(conn, offset, end) + return length +} + +func (e *goEngine) markRetransmitted(conn *GoConn, start uint64, end uint64) { + previous := conn.sendUnacked.Load() + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if previous < end && descriptor.endOffset > start { + descriptor.flags |= goDescriptorRetransmitted + descriptor.flags &^= goDescriptorDropped + } + previous = descriptor.endOffset + } +} + +func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { + if conn.connState.Load() >= goConnStateDead { + return + } + if err != nil { + conn.storeError(err) + } + if class == goDeathFinLinger && conn.receiveAvailable.Load() > conn.consumedTail.Load() { + conn.receiveDrainable.Store(true) + } + conn.connState.Store(goConnStateDead) + conn.deathClass = class + conn.state = goTCPClosed + if class == goDeathFinLinger && conn.splice != nil && !conn.splice.uploadClosed { + e.spliceMarkDirty(conn) + } else { + e.spliceDetach(conn, conn.loadError()) + } + if !conn.everEstablished.Load() { + close(conn.establishedSignal) + } + close(conn.closeSignal) + conn.retransmitDeadline = 0 + conn.probeDeadline = 0 + conn.persistDeadline = 0 + conn.handshakeDeadline = 0 + conn.finPending = false + switch class { + case goDeathFinLinger: + conn.lingerDeadline = e.now() + int64(goFinLinger) + case goDeathAbortLinger: + conn.lingerDeadline = e.now() + int64(goAbortLinger) + default: + conn.lingerDeadline = 0 + } + e.rearmTimer(conn) + if class == goDeathImmediate { + e.removeFlowKey(conn) + } + if !conn.onDyingList { + conn.onDyingList = true + conn.dyingSince = e.now() + conn.dyingNext = e.dyingList + e.dyingList = conn + } + if conn.onBlockedList { + conn.transmitSignal.notify() + } + conn.wakeUser() + e.wokeHandlerThisBurst = true +} + +func (e *goEngine) removeFlow(conn *GoConn) { + if conn.connState.Load() < goConnStateDead { + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + return + } + conn.lingerDeadline = 0 + e.wheel.cancel(&conn.timerNode) + e.removeFlowKey(conn) +} + +func (e *goEngine) removeFlowKey(conn *GoConn) { + if !conn.keyed { + return + } + conn.keyed = false + delete(e.flows, conn.key) + e.stack.directory.remove(conn.key, conn) + reason := FlowCloseReset + if conn.finReceived && conn.finSent { + reason = FlowCloseFinished + } + e.dispatchStage.teardownFlow(conn.key, reason) +} + +func (e *goEngine) reapDying() { + var kept *GoConn + now := e.now() + for conn := e.dyingList; conn != nil; { + next := conn.dyingNext + conn.dyingNext = nil + if conn.reclaimable() { + conn.releaseResources() + conn.onDyingList = false + } else { + if now-conn.dyingSince > int64(goDyingLeakTimeout) { + conn.dyingSince = now + e.stack.logger.Warn("go: connection stuck in teardown: ", conn.destination) + } + conn.dyingNext = kept + kept = conn + } + conn = next + } + e.dyingList = kept +} + +func (c *GoConn) reclaimable() bool { + if c.spliced.Load() || c.writerActive.Load() != 0 || c.transmitterActive.Load() != 0 || c.postedTarget.Load() != nil { + return false + } + if c.receiveDrainable.Load() && !c.userClosed.Load() && c.receiveAvailable.Load() > c.consumedTail.Load() { + return false + } + c.receiveReleased.Store(true) + if c.readerActive.Load() != 0 { + c.receiveReleased.Store(false) + return false + } + return true +} + +func (c *GoConn) releaseResources() { + c.receiveChain.releaseAll() + c.transmitStore.releaseAll() + c.oooRanges = nil + c.descriptors.release() + c.scoreboard.reset() + if c.receiveTarget.buffer != nil { + c.receiveTarget.buffer.Release() + c.receiveTarget.buffer = nil + } +} + +func (e *goEngine) closeAllFlows() []*GoConn { + var unreset []*GoConn + for _, conn := range e.flows { + if conn.connState.Load() < goConnStateDead && !e.sendReset(conn) { + unreset = append(unreset, conn) + } + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + e.removeFlowKey(conn) + } + return unreset +} + +func (e *goEngine) expireSweepTick(now int64) { + for _, conn := range e.flows { + if conn.connState.Load() >= goConnStateDead { + continue + } + sent := conn.sentTail.Load() + if sent != conn.sweepSentTail { + conn.sweepSentTail = sent + conn.lastActivity = now + } + e.keepalive(conn, now) + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + conn.releaseTransmitted(conn.sendUnacked.Load()) + conn.releaseDrainedSlabs() + conn.releaseIdleDescriptors() + conn.wakeWriter() + e.maybeSendFin(conn) + e.updatePersist(conn) + if conn.hasOutstanding() && conn.retransmitDeadline == 0 { + e.rearmRetransmit(conn) + } + } + e.wheel.schedule(&e.sweepTickNode, now+int64(goSweepInterval)) +} + +func (e *goEngine) keepalive(conn *GoConn, now int64) { + if conn.state == goTCPFinWait2 && conn.userClosed.Load() && now-conn.finWait2Since > int64(goFinWait2Timeout) { + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: orphaned FIN_WAIT_2"), goDeathImmediate) + return + } + idle := now - conn.lastActivity + if idle < int64(tcpEstablishedTimeout) { + conn.keepaliveProbes = 0 + return + } + if conn.state == goTCPSynReceived || conn.hasOutstanding() { + return + } + sinceIdle := idle - int64(tcpEstablishedTimeout) + due := int64(conn.keepaliveProbes) * int64(goKeepaliveInterval) + if sinceIdle < due { + return + } + if conn.keepaliveProbes >= goKeepaliveCount { + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: keepalive timeout"), goDeathImmediate) + return + } + conn.keepaliveProbes++ + segment := goSegment{offset: conn.sendNext() - 1, flags: header.TCPFlagAck} + e.writeConnControl(conn, &segment) +} + +func (e *goEngine) expireReclaimTick(now int64) { + e.reclaim() + e.wheel.schedule(&e.reclaimTickNode, now+int64(goReclaimInterval)) +} + +func (e *goEngine) reclaim() { + e.slabPool.trim() + e.descriptorPool.trim() + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + entry.buffer.Release() + entry.buffer = nil + } + } +} diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go new file mode 100644 index 00000000..530a2c98 --- /dev/null +++ b/stack_go_tcp_output.go @@ -0,0 +1,742 @@ +package tun + +import ( + "context" + "net" + "net/netip" + "os" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing-tun/gtcpip/seqnum" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" +) + +const ( + goHeaderScratchSize = 128 + goGSOMaxPayload = 61440 + goEngineInlineBurst = 256 << 10 + goTransmitReadAhead = 64 << 10 +) + +const goTimestampOptionLength = 2 + header.TCPOptionTSLength + +// include/uapi/linux/virtio_net.h +const ( + goGSOTypeIPv4 uint8 = 1 + goGSOTypeIPv6 uint8 = 4 +) + +type goSegment struct { + offset uint64 + length int + flags header.TCPFlags + sackBlocks []goSackBlock + gsoSize uint16 +} + +func (c *GoConn) Write(p []byte) (int, error) { + err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) + if err != nil { + return 0, err + } + c.writeAccess.Lock() + defer c.writeAccess.Unlock() + if !c.enterWriter() { + return 0, net.ErrClosed + } + defer c.exitWriter() + return c.writeLocked(p) +} + +func (c *GoConn) writeLocked(p []byte) (int, error) { + total := 0 + for { + if c.connState.Load() >= goConnStateAborted { + return total, c.closeError() + } + if c.writeShut.Load() { + return total, net.ErrClosed + } + select { + case <-c.writeDeadline.Wait(): + return total, os.ErrDeadlineExceeded + default: + } + if len(p) == 0 { + return total, nil + } + budget := c.writeBudget(len(p)) + if budget <= 0 { + err := c.parkWriter(1) + if err != nil { + return total, err + } + continue + } + buffered := c.bufferedTail.Load() + c.transmitStore.reserve(buffered, budget) + c.transmitStore.writeAt(buffered, p[:budget]) + c.publishBuffered(buffered + uint64(budget)) + total += budget + p = p[budget:] + if len(p) == 0 { + return total, nil + } + } +} + +func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { + err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) + if err != nil { + buffer.Release() + return err + } + c.writeAccess.Lock() + defer c.writeAccess.Unlock() + if !c.enterWriter() { + buffer.Release() + return net.ErrClosed + } + defer c.exitWriter() + length := buffer.Len() + if length > goTransmitCapacityMax { + defer buffer.Release() + _, err = c.writeLocked(buffer.Bytes()) + return err + } + for { + if c.connState.Load() >= goConnStateAborted { + buffer.Release() + return c.closeError() + } + if c.writeShut.Load() { + buffer.Release() + return net.ErrClosed + } + select { + case <-c.writeDeadline.Wait(): + buffer.Release() + return os.ErrDeadlineExceeded + default: + } + if length == 0 { + buffer.Release() + return nil + } + budget := c.writeBudget(length) + if budget < length { + if budget > 0 && c.storeEmpty() { + defer buffer.Release() + _, err = c.writeLocked(buffer.Bytes()) + return err + } + err = c.parkWriter(length) + if err != nil { + buffer.Release() + return err + } + continue + } + buffered := c.bufferedTail.Load() + if !c.transmitStore.adopt(buffered, buffer) { + defer buffer.Release() + _, err = c.writeLocked(buffer.Bytes()) + return err + } + c.publishBuffered(buffered + uint64(length)) + return nil + } +} + +func (c *GoConn) FrontHeadroom() int { + return header.IPv6MinimumSize + header.TCPMinimumSize + goTimestampOptionLength +} + +func (c *GoConn) WriterMTU() int { + mss := int(c.effectiveMSS) + if c.engine.platformIO.transmitSegmentOffload() { + return max(goGSOMaxPayload/mss*mss, mss) + } + return max(0xffff/mss*mss, mss) +} + +func (c *GoConn) publishBuffered(tail uint64) { + c.bufferedTail.Store(tail) + if c.transmitterActive.Load() != 0 { + c.transmitSignal.notify() + return + } + c.transmitInline() +} + +func (c *GoConn) transmitInline() { + for { + if !c.transmitOwner.CompareAndSwap(0, 1) { + c.transmitSignal.notify() + return + } + _, blocked := c.transmitLoop(false, 0) + c.transmitOwner.Store(0) + if blocked { + c.wakeTransmitterGoroutine() + return + } + if c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 { + return + } + } +} + +func (c *GoConn) transmitOnEngine() bool { + budget := c.engine.inlineTransmitBudget + if budget <= 0 { + return false + } + if !c.transmitOwner.CompareAndSwap(0, 1) { + c.transmitSignal.notify() + return true + } + written, blocked := c.transmitLoop(false, budget) + c.transmitOwner.Store(0) + c.engine.inlineTransmitBudget = budget - written + if blocked { + return false + } + return c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 +} + +func (c *GoConn) wakeTransmitterGoroutine() { + if c.transmitterActive.CompareAndSwap(0, 1) { + go c.runTransmitter() + return + } + c.transmitSignal.notify() +} + +func (c *GoConn) hasDataWaiting() bool { + return c.bufferedTail.Load() > c.sentTail.Load() || c.writerParked.Load() +} + +func (c *GoConn) runTransmitter() { + defer func() { + c.transmitterActive.Store(0) + c.engine.reapAfterExit() + }() + for { + if c.connState.Load() >= goConnStateAborted { + return + } + if c.transmitOwner.CompareAndSwap(0, 1) { + c.transmitLoop(true, 0) + c.transmitOwner.Store(0) + } + select { + case <-c.transmitSignal: + case <-c.closeSignal: + return + } + } +} + +func (c *GoConn) nextTransmitLength() int { + sent := c.sentTail.Load() + pending := c.bufferedTail.Load() - sent + if pending == 0 { + return 0 + } + permit := c.sendPermit.Load() + if permit <= sent { + return 0 + } + return c.frameLength(int(min(pending, permit-sent))) +} + +func (c *GoConn) writeBudget(pending int) int { + released := c.sendReleased.Load() + buffered := c.bufferedTail.Load() + stored := buffered - released + if stored >= goTransmitCapacityMax { + return 0 + } + if stored >= goTransmitReadAhead { + share := c.engine.slabPool.shareBytes() + if share > 0 && c.slabHolder.heldBytes() >= share { + return 0 + } + } + unacked := c.sendUnacked.Load() + permit := max(c.sendPermit.Load(), unacked) + limit := min(permit+max(permit-unacked, goTransmitReadAhead), released+goTransmitCapacityMax) + budget := int64(limit) - int64(buffered) + if pending > 0 { + budget = min(budget, int64(pending)) + } + if budget <= 0 { + return 0 + } + return int(budget) +} + +func (c *GoConn) storeEmpty() bool { + return c.bufferedTail.Load() == c.sendReleased.Load() +} + +func (c *GoConn) writeReady(required int) bool { + budget := c.writeBudget(required) + return budget >= required || (budget > 0 && c.storeEmpty()) +} + +func (c *GoConn) parkWriter(required int) error { + c.writerNeeds.Store(int32(required)) + c.writerParked.Store(true) + c.writeSignal.drain() + if c.writeReady(required) { + c.writerParked.Store(false) + return nil + } + if c.connState.Load() >= goConnStateAborted { + c.writerParked.Store(false) + return c.closeError() + } + if c.writeShut.Load() { + c.writerParked.Store(false) + return net.ErrClosed + } + deadlineSignal := c.writeDeadline.Wait() + select { + case <-c.writeSignal: + case <-deadlineSignal: + c.writerParked.Store(false) + return os.ErrDeadlineExceeded + case <-c.closeSignal: + c.writerParked.Store(false) + return c.closeError() + } + c.writerParked.Store(false) + if c.connState.Load() >= goConnStateAborted { + return c.closeError() + } + return nil +} + +func (c *GoConn) parkTransmitBlocked() error { + c.engine.postMessage(&c.blockedMessage) + select { + case <-c.transmitSignal: + case <-c.closeSignal: + return c.closeError() + } + if c.connState.Load() >= goConnStateAborted { + return c.closeError() + } + return nil +} + +func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, bool) { + if transmitter { + defer c.engine.platformIO.flush() + } + sentAny := false + written := 0 + if c.blockedValid { + if !transmitter { + return written, true + } + if !c.retryBlocked() { + return written, false + } + sentAny = true + } + for c.connState.Load() < goConnStateAborted { + if budget > 0 && written >= budget { + break + } + length := c.nextTransmitLength() + if length == 0 { + break + } + written += length + sent := c.sentTail.Load() + segment := goSegment{offset: sent, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} + if length > int(c.effectiveMSS) { + segment.gsoSize = c.effectiveMSS + } + var flags uint8 + if c.engine.engineState.Load() == goEngineParked { + flags |= goDescriptorNoSample + } + c.pushDescriptors(sent, length, c.stamp(c.engine.now()), flags) + c.sentTail.Store(sent + uint64(length)) + sentAny = true + err := c.transmitFrame(&segment) + switch err { + case nil: + c.transmittedTail.Store(sent + uint64(length)) + case errGoTransmitBlocked: + c.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) + c.blockedSegment = segment + c.blockedValid = true + if !transmitter { + c.finishTransmit(sentAny) + return written, true + } + if !c.retryBlocked() { + return written, false + } + case errGoFrameDropped: + c.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) + c.engine.postMessage(&c.droppedMessage) + c.transmittedTail.Store(sent + uint64(length)) + default: + if c.connState.Load() < goConnStateAborted { + c.requestAbort(E.Cause(err, "go: write tun")) + } + return written, false + } + } + c.finishTransmit(sentAny) + return written, false +} + +func (c *GoConn) retryBlocked() bool { + for { + parkErr := c.parkTransmitBlocked() + if parkErr != nil { + return false + } + end := c.blockedSegment.offset + uint64(c.blockedSegment.length) + err := c.transmitFrame(&c.blockedSegment) + switch err { + case nil: + case errGoTransmitBlocked: + c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorNoSample) + continue + case errGoFrameDropped: + c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorDropped) + c.engine.postMessage(&c.droppedMessage) + default: + if c.connState.Load() < goConnStateAborted { + c.requestAbort(E.Cause(err, "go: write tun")) + } + return false + } + c.blockedValid = false + c.blockedFrame = nil + c.transmittedTail.Store(end) + return true + } +} + +func (c *GoConn) finishTransmit(sentAny bool) { + if !sentAny { + return + } + if c.spliced.Load() { + if c.retransmitDeadline == 0 { + c.engine.rearmRetransmit(c) + } + return + } + c.armRetransmitIfIdle() + if c.finRequested.Load() && c.sentTail.Load() == c.bufferedTail.Load() { + c.engine.postMessage(&c.closeMessage) + } +} + +func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint8) { + c.descriptors.push(offset+uint64(length), int32(length), goDescriptorAmend|flags) +} + +func (c *GoConn) pushDescriptors(offset uint64, length int, sentAt int32, flags uint8) { + step := int(c.effectiveMSS) + for length > 0 { + span := min(length, step) + offset += uint64(span) + length -= span + c.descriptors.push(offset, sentAt, flags) + } +} + +func (c *GoConn) frameLength(pending int) int { + limit := int(c.effectiveMSS) + if c.engine.platformIO.transmitSegmentOffload() { + limit = max(goGSOMaxPayload/limit*limit, limit) + } + return min(pending, limit) +} + +func (c *GoConn) transmitFrame(segment *goSegment) error { + if c.blockedValid && c.blockedFrame != nil && segment.offset == c.blockedSegment.offset && segment.length == c.blockedSegment.length { + return c.engine.platformIO.writeData(c.blockedFrame, c.blockedMeta) + } + offload := c.engine.platformIO.transmitChecksumOffload() + buffer, payload, contiguous := c.transmitStore.leadingRun(segment.offset, segment.length) + var ( + frame [][]byte + meta ForwardFrameMeta + ) + if contiguous && buffer.Start() >= c.FrontHeadroom() { + var packet []byte + packet, meta = c.buildContiguousFrame(buffer, payload, segment, offload) + c.transmitSegments = append(c.transmitSegments[:0], packet) + frame = c.transmitSegments + } else { + frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload) + c.transmitSegments = frame + } + err := c.engine.platformIO.writeData(frame, meta) + if err == errGoTransmitBlocked { + c.blockedFrame = frame + c.blockedMeta = meta + } + return err +} + +func (c *GoConn) armRetransmitIfIdle() { + if c.retransmitArmed.CompareAndSwap(false, true) { + c.engine.postMessage(&c.retransmitMessage) + } +} + +type goFrameLayout struct { + ipHeaderLength int + tcpHeaderLength int + timestampLength int + sackBlocks []goSackBlock +} + +func (l *goFrameLayout) headerLength() int { + return l.ipHeaderLength + l.tcpHeaderLength +} + +func (c *GoConn) frameLayout(segment *goSegment) goFrameLayout { + var layout goFrameLayout + if c.timestampsEnabled { + layout.timestampLength = goTimestampOptionLength + } + layout.sackBlocks = segment.sackBlocks + if len(layout.sackBlocks) > goMaxSackBlocksSent { + layout.sackBlocks = layout.sackBlocks[:goMaxSackBlocksSent] + } + sackLength := 0 + if len(layout.sackBlocks) > 0 { + sackLength = 4 + 8*len(layout.sackBlocks) + } + layout.tcpHeaderLength = header.TCPMinimumSize + layout.timestampLength + sackLength + layout.ipHeaderLength = goNetworkHeaderLength(c.ipVersion) + return layout +} + +func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *goSegment) uint16 { + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + receiveAck := c.receiveNextAck.Load() + edge := c.receiveEdge.Load() + var window uint16 + if edge > receiveAck { + window = uint16(min((edge-receiveAck)>>c.localWindowShift, 0xffff)) + } + for { + sent := c.sentEdge.Load() + if edge <= sent || c.sentEdge.CompareAndSwap(sent, edge) { + break + } + } + tcpHdr.Encode(&header.TCPFields{ + SrcPort: c.destination.Port, + DstPort: c.source.Port, + SeqNum: uint32(uint64(c.sendISN) + segment.offset), + AckNum: uint32(uint64(c.clientISN) + receiveAck), + DataOffset: uint8(layout.tcpHeaderLength), + Flags: segment.flags, + WindowSize: window, + }) + options := packet[layout.ipHeaderLength+header.TCPMinimumSize:] + if layout.timestampLength > 0 { + goEncodeTimestampOption(options, goTimestampAt(c.engine.now()), c.tsRecent.Load()) + } + if len(layout.sackBlocks) > 0 { + goEncodeSackOption(options[layout.timestampLength:], layout.sackBlocks, c.clientISN) + } + return goEncodeNetworkHeader(packet, c.ipVersion, c.destination.Addr, c.source.Addr, layout.tcpHeaderLength+segment.length, uint16(c.ident.Add(1))) +} + +func goNetworkHeaderLength(ipVersion uint8) int { + if ipVersion == 4 { + return header.IPv4MinimumSize + } + return header.IPv6MinimumSize +} + +func goEncodeNetworkHeader(packet []byte, ipVersion uint8, source netip.Addr, destination netip.Addr, tcpLength int, ident uint16) uint16 { + var network header.Network + if ipVersion == 4 { + ipHdr := header.IPv4(packet) + ipHdr.Encode(&header.IPv4Fields{ + TotalLength: uint16(header.IPv4MinimumSize + tcpLength), + ID: ident, + TTL: synthesizedTTL, + Protocol: uint8(header.TCPProtocolNumber), + SrcAddr: source, + DstAddr: destination, + }) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + network = ipHdr + } else { + ipHdr := header.IPv6(packet) + ipHdr.Encode(&header.IPv6Fields{ + PayloadLength: uint16(tcpLength), + TransportProtocol: header.TCPProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: source, + DstAddr: destination, + }) + network = ipHdr + } + return header.PseudoHeaderChecksum(header.TCPProtocolNumber, network.SourceAddressSlice(), network.DestinationAddressSlice(), uint16(tcpLength)) +} + +func (c *GoConn) offloadMeta(tcpHdr header.TCP, pseudoSum uint16, layout *goFrameLayout, segment *goSegment) ForwardFrameMeta { + tcpHdr.SetChecksum(pseudoSum) + meta := ForwardFrameMeta{ + needsChecksum: true, + checksumStart: uint16(layout.ipHeaderLength), + checksumOffset: header.TCPChecksumOffset, + } + if segment.gsoSize > 0 { + meta.gsoSize = segment.gsoSize + meta.gsoType = goGSOType(c.ipVersion) + } + return meta +} + +func (c *GoConn) buildFrame(scratch []byte, segments [][]byte, segment *goSegment, store *goTransmitStore, offload bool) ([][]byte, ForwardFrameMeta) { + layout := c.frameLayout(segment) + packet := scratch[:layout.headerLength()] + pseudoSum := c.encodeHeaders(packet, &layout, segment) + segments = append(segments, packet) + if segment.length > 0 { + segments = store.appendRuns(segments, segment.offset, segment.length) + } + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + if offload { + return segments, c.offloadMeta(tcpHdr, pseudoSum, &layout, segment) + } + tcpHdr.SetChecksum(0) + sum := checksum.Checksum(tcpHdr, pseudoSum) + sum = goChecksumRuns(sum, segments[1:], layout.tcpHeaderLength%2 == 1) + tcpHdr.SetChecksum(^sum) + return segments, ForwardFrameMeta{} +} + +func (c *GoConn) buildContiguousFrame(buffer *buf.Buffer, payload []byte, segment *goSegment, offload bool) ([]byte, ForwardFrameMeta) { + layout := c.frameLayout(segment) + headerLength := layout.headerLength() + packet := buffer.ExtendHeader(headerLength) + packet = packet[:headerLength+len(payload)] + pseudoSum := c.encodeHeaders(packet[:headerLength], &layout, segment) + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + if offload { + return packet, c.offloadMeta(tcpHdr, pseudoSum, &layout, segment) + } + tcpHdr.SetChecksum(0) + tcpHdr.SetChecksum(^checksum.Checksum(packet[layout.ipHeaderLength:], pseudoSum)) + return packet, ForwardFrameMeta{} +} + +func goChecksumRuns(initial uint16, runs [][]byte, odd bool) uint16 { + sum := initial + for _, run := range runs { + if odd { + partial := checksum.Checksum(run, 0) + sum = checksum.Combine(sum, partial>>8|partial<<8) + } else { + sum = checksum.Checksum(run, sum) + } + if len(run)%2 == 1 { + odd = !odd + } + } + return sum +} + +func goTimestampAt(now int64) uint32 { + return uint32(now >> goTimestampShift) +} + +func goEncodeTimestampOption(options []byte, timestampValue uint32, timestampEcho uint32) { + options[0] = header.TCPOptionNOP + options[1] = header.TCPOptionNOP + header.EncodeTSOption(timestampValue, timestampEcho, options[2:goTimestampOptionLength]) +} + +func goEncodeSackOption(options []byte, blocks []goSackBlock, base uint32) { + var sackBlocks [goMaxSackBlocksSent]header.SACKBlock + for index, block := range blocks { + sackBlocks[index] = header.SACKBlock{ + Start: seqnum.Value(uint64(base) + block.start), + End: seqnum.Value(uint64(base) + block.end), + } + } + options[0] = header.TCPOptionNOP + options[1] = header.TCPOptionNOP + optionsLength := 2 + header.EncodeSACKBlocks(sackBlocks[:len(blocks)], options[2:]) + header.AddTCPOptionPadding(options, optionsLength) +} + +func (c *GoConn) buildSynAck(synOptions header.TCPSynOptions, localMSS uint16) { + ipHeaderLength := goNetworkHeaderLength(c.ipVersion) + optionsStorage := c.synAckImage[ipHeaderLength+header.TCPMinimumSize:] + optionsLength := header.EncodeMSSOption(uint32(localMSS), optionsStorage) + if synOptions.WS >= 0 { + optionsLength += header.EncodeWSOption(int(c.localWindowShift), optionsStorage[optionsLength:]) + } + if synOptions.SACKPermitted { + optionsLength += header.EncodeSACKPermittedOption(optionsStorage[optionsLength:]) + } + if c.timestampsEnabled { + optionsLength += header.EncodeTSOption(goTimestampAt(c.engine.now()), c.tsRecent.Load(), optionsStorage[optionsLength:]) + } + optionsLength += header.AddTCPOptionPadding(optionsStorage, optionsLength) + tcpHeaderLength := header.TCPMinimumSize + optionsLength + totalLength := ipHeaderLength + tcpHeaderLength + c.synAckLength = uint8(totalLength) + packet := c.synAckImage[:totalLength] + tcpHdr := header.TCP(packet[ipHeaderLength:]) + tcpHdr.Encode(&header.TCPFields{ + SrcPort: c.destination.Port, + DstPort: c.source.Port, + SeqNum: c.sendISN, + AckNum: c.clientISN + 1, + DataOffset: uint8(tcpHeaderLength), + Flags: header.TCPFlagSyn | header.TCPFlagAck, + WindowSize: uint16(min(c.receiveCapacity, 0xffff)), + }) + pseudoSum := goEncodeNetworkHeader(packet, c.ipVersion, c.destination.Addr, c.source.Addr, tcpHeaderLength, uint16(c.ident.Add(1))) + tcpHdr.SetChecksum(0) + tcpHdr.SetChecksum(^checksum.Checksum(tcpHdr, pseudoSum)) +} + +func (c *GoConn) writeSynAck() error { + var frame [1][]byte + frame[0] = c.synAckImage[:c.synAckLength] + return goIgnoreDropped(c.engine.platformIO.writeFrame(frame[:], ForwardFrameMeta{})) +} + +func (c *GoConn) writeReset() error { + var scratch [header.IPv6MinimumSize + header.TCPMinimumSize + goTimestampOptionLength]byte + var frame [1][]byte + segment := goSegment{offset: 0, flags: header.TCPFlagRst | header.TCPFlagAck} + segments, _ := c.buildFrame(scratch[:], frame[:0], &segment, &c.transmitStore, false) + return goIgnoreDropped(c.engine.platformIO.writeFrame(segments, ForwardFrameMeta{})) +} + +var errGoTransmitBlocked = E.New("go: transmit blocked") + +func goGSOType(ipVersion uint8) uint8 { + if ipVersion == 4 { + return goGSOTypeIPv4 + } + return goGSOTypeIPv6 +} diff --git a/stack_go_udp.go b/stack_go_udp.go new file mode 100644 index 00000000..4896d969 --- /dev/null +++ b/stack_go_udp.go @@ -0,0 +1,507 @@ +package tun + +import ( + "context" + "io" + "net/netip" + "os" + "runtime" + "slices" + "sync" + "sync/atomic" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +const ( + goPacketBatchSize = 32 + goUDPGSOType = 5 + goUDPChecksumOffset = 6 +) + +type goUDPFrame struct { + header [header.IPv6MinimumSize + header.UDPMinimumSize]byte + length int + payload []byte + meta ForwardFrameMeta +} + +type goUDPUserData struct { + engine *goEngine + owner *goEngine + key udpNatSessionKey + packet []byte + meta ForwardFrameMeta + ipVersion uint8 + created *GoPacketConn +} + +func (e *goEngine) processUDPSegments(frame *goFrame) { + meta := frame.meta + packet := frame.buffer.Bytes() + parsed, parsedOK := parseForwardPacket(packet) + headerLength := int(meta.checksumStart) + header.UDPMinimumSize + if !parsedOK || parsed.protocol != uint8(header.UDPProtocolNumber) || parsed.fragment || meta.gsoSize == 0 || headerLength > len(packet) { + return + } + count := (len(packet) - headerLength + int(meta.gsoSize) - 1) / int(meta.gsoSize) + if count == 0 { + return + } + options := *e.packetReadOptions.Load() + buffers := newGoReadBuffers(headerLength+int(meta.gsoSize), count, options) + defer buf.ReleaseMulti(buffers) + segments := make([][]byte, count) + sizes := make([]int, count) + for index, buffer := range buffers { + segments[index] = buffer.FreeBytes() + } + count, err := GSOSplit(packet, GSOOptions{ + GSOType: GSOUDPL4, HdrLen: uint16(headerLength), GSOSize: meta.gsoSize, + CsumStart: meta.checksumStart, CsumOffset: meta.checksumOffset, NeedsCsum: meta.needsChecksum, + }, segments, sizes, 0) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: split UDP segments")) + return + } + for index := range count { + buffer := buffers[index] + buffer.Truncate(sizes[index]) + options.PostReturn(buffer) + e.processFrame(&goFrame{buffer: buffer}) + } + e.flushPacketUploads() +} + +func (e *goEngine) demuxUDP(packetBuffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + packet := packetBuffer.Bytes() + if len(parsed.transport) < header.UDPMinimumSize { + return + } + switch parsed.verdict.Action { + case ActionReject: + meta.completeChecksum(packet) + reply, ok := BuildUnreachable(packet, netip.Addr{}, 0) + if ok { + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write unreachable")) + } + } + return + case ActionDrop: + return + case ActionHijackDNS: + e.dispatcher.hijackDNSPacket(parsed) + return + } + payload := header.UDP(parsed.transport).Payload() + source := M.SocksaddrFromNetIP(parsed.source) + destination := M.SocksaddrFromNetIP(parsed.destination) + key := e.udpNat.sessionKey(source, destination) + userData := &e.udpUserData + *userData = goUDPUserData{ + engine: e, + key: key, + packet: packet, + meta: meta, + ipVersion: parsed.ipVersion, + } + conn, ok := e.udpNat.getOrCreate(key, source, destination, userData) + if !ok { + if userData.owner != nil { + userData.owner.inject(packet, meta) + } + return + } + if userData.created != nil { + e.attachUDPSession(conn, userData.created, &parsed.verdict) + userData.created = nil + } + if writer, isGoPacketConn := conn.writer.(*GoPacketConn); isGoPacketConn { + if writer.engine != e { + writer.engine.inject(packet, meta) + return + } + if writer.splice != nil { + if len(e.packetFlows) < goReadBatch { + e.packetFlows[parsed.flowKey()] = writer + } + e.packetSpliceInput(writer, packetBuffer, parsed) + return + } + trackerPointer := writer.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountForward(len(packet)) + } + } + readWaitOptions := conn.loadReadWaitOptions() + buffer := readWaitOptions.NewBufferSize(len(payload)) + buffer.Write(payload) + readWaitOptions.PostReturn(buffer) + conn.enqueue(buffer, destination) + e.wokeHandlerThisBurst = true +} + +func (e *goEngine) packetSpliceInput(writer *GoPacketConn, packetBuffer *buf.Buffer, parsed *forwardPacket) { + trackerPointer := writer.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountForward(packetBuffer.Len()) + } + headerLength := header.IPv6MinimumSize + if parsed.ipVersion == 4 { + headerLength = int(header.IPv4(parsed.network).HeaderLength()) + } + payloadLength := len(header.UDP(parsed.transport).Payload()) + packetBuffer.Advance(headerLength + header.UDPMinimumSize) + packetBuffer.Truncate(payloadLength) + e.packetSpliceUpload(writer, packetBuffer, M.SocksaddrFromNetIP(parsed.destination)) +} + +func (e *goEngine) attachUDPSession(conn *UDPNatConn, writer *GoPacketConn, verdict *FlowVerdict) { + writer.conn.Store(conn) + if verdict.UDPTimeout > 0 { + conn.SetTimeout(verdict.UDPTimeout) + } + if verdict.NewTracker != nil { + tracker := verdict.NewTracker() + if tracker != nil { + writer.tracker.Store(&tracker) + tracker.AttachFlow(writer) + } + } +} + +func (s *Go) prepareUDPConnection(source M.Socksaddr, destination M.Socksaddr, userData any) (bool, context.Context, N.PacketWriter, N.CloseHandlerFunc) { + data := userData.(*goUDPUserData) + directory := &s.directory + if directory.udpFlows != nil { + directory.access.Lock() + defer directory.access.Unlock() + if owner := directory.udpFlows[data.key]; owner != nil { + conn := owner.conn.Load() + if conn == nil || !conn.isClosed() { + data.owner = owner.engine + return false, nil, nil, nil + } + } + } + writer := &GoPacketConn{ + engine: data.engine, + key: data.key, + platformIO: data.engine.platformIO, + mtu: s.mtu, + checksumOffload: data.engine.platformIO.transmitChecksumOffload(), + snapshot: slices.Clone(data.packet), + snapshotMeta: data.meta, + } + writer.spliceMessage = goMessage{kind: goMessagePacketSplice, packet: writer} + writer.closeMessage = goMessage{kind: goMessagePacketClose, packet: writer} + writer.buildTemplate(data.ipVersion, source) + data.created = writer + if directory.udpFlows != nil { + directory.udpFlows[data.key] = writer + } + return true, s.ctx, writer, writer.handleSessionClose +} + +type GoPacketConn struct { + engine *goEngine + key udpNatSessionKey + platformIO goPlatformIO + splice *goSplicePacket + splicePending atomic.Pointer[goSplicePacket] + spliceActive atomic.Bool + spliceCloseError atomic.Pointer[goConnError] + spliceMessage goMessage + closeMessage goMessage + mtu int + ipVersion uint8 + templateLength int + checksumOffload bool + template [header.IPv6MinimumSize + header.UDPMinimumSize]byte + conn atomic.Pointer[UDPNatConn] + tracker atomic.Pointer[FlowTracker] + trackerClosed atomic.Bool + snapshotAccess sync.Mutex + snapshot []byte + snapshotMeta ForwardFrameMeta +} + +func (w *GoPacketConn) buildTemplate(ipVersion uint8, client M.Socksaddr) { + w.ipVersion = ipVersion + clientAddr := client.Addr + if ipVersion == 4 { + w.templateLength = header.IPv4MinimumSize + header.UDPMinimumSize + ipHdr := header.IPv4(w.template[:w.templateLength]) + ipHdr.Encode(&header.IPv4Fields{ + TotalLength: uint16(w.templateLength), + TTL: synthesizedTTL, + Protocol: uint8(header.UDPProtocolNumber), + SrcAddr: clientAddr, + DstAddr: clientAddr, + }) + header.UDP(ipHdr[header.IPv4MinimumSize:]).Encode(&header.UDPFields{ + DstPort: client.Port, + }) + } else { + w.templateLength = header.IPv6MinimumSize + header.UDPMinimumSize + ipHdr := header.IPv6(w.template[:w.templateLength]) + ipHdr.Encode(&header.IPv6Fields{ + PayloadLength: header.UDPMinimumSize, + TransportProtocol: header.UDPProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: clientAddr, + DstAddr: clientAddr, + }) + header.UDP(ipHdr[header.IPv6MinimumSize:]).Encode(&header.UDPFields{ + DstPort: client.Port, + }) + } +} + +func (w *GoPacketConn) preparePacketHeader(packet []byte, payload []byte, destination M.Socksaddr, checksumOffload bool) (ForwardFrameMeta, error) { + maximumPayload := 65535 - header.UDPMinimumSize + if w.ipVersion == 4 { + maximumPayload -= header.IPv4MinimumSize + } + if len(payload) > maximumPayload { + return ForwardFrameMeta{}, errGoFrameDropped + } + if !destination.IsIP() || w.ipVersion == 4 && !destination.IsIPv4() { + return ForwardFrameMeta{}, E.New("go: invalid packet destination") + } + if w.ipVersion == 6 && destination.IsIPv4() { + destination = M.SocksaddrFrom(netip.AddrFrom16(destination.Addr.As16()), destination.Port) + } + var meta ForwardFrameMeta + copy(packet, w.template[:w.templateLength]) + udpLength := uint16(header.UDPMinimumSize + len(payload)) + var sourceAddress, destinationAddress []byte + if w.ipVersion == 4 { + ipHdr := header.IPv4(packet[:header.IPv4MinimumSize]) + ipHdr.SetTotalLength(uint16(w.templateLength + len(payload))) + ipHdr.SetSourceAddr(destination.Addr) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } else { + ipHdr := header.IPv6(packet[:header.IPv6MinimumSize]) + ipHdr.SetPayloadLength(udpLength) + ipHdr.SetSourceAddr(destination.Addr) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } + udpHdr := header.UDP(packet[w.templateLength-header.UDPMinimumSize : w.templateLength]) + udpHdr.SetSourcePort(destination.Port) + udpHdr.SetLength(udpLength) + pseudoSum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpLength) + if checksumOffload { + udpHdr.SetChecksum(pseudoSum) + meta.needsChecksum = true + meta.checksumStart = uint16(w.templateLength - header.UDPMinimumSize) + meta.checksumOffset = goUDPChecksumOffset + } else { + sum := ^checksum.Checksum(payload, udpHdr.CalculateChecksum(pseudoSum)) + if sum == 0 { + sum = 0xffff + } + udpHdr.SetChecksum(sum) + } + trackerPointer := w.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountReverse(w.templateLength + len(payload)) + } + return meta, nil +} + +func (w *GoPacketConn) FrontHeadroom() int { + return w.platformIO.transmitPrefix() + w.templateLength +} + +func (w *GoPacketConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + if !destination.IsIP() { + buffer.Release() + return E.Cause(os.ErrInvalid, "invalid destination") + } + defer buffer.Release() + return w.transmit(buffer, destination) +} + +func (w *GoPacketConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + return nil, false + } + return w, true +} + +func (w *GoPacketConn) WritePacketBatch(buffers []*buf.Buffer, destinations []M.Socksaddr) error { + defer buf.ReleaseMulti(buffers) + if len(buffers) == 0 || len(buffers) != len(destinations) { + return os.ErrInvalid + } + var frames [goPacketBatchSize]goUDPFrame + count := 0 + for index, buffer := range buffers { + if !destinations[index].IsIP() { + return os.ErrInvalid + } + if buffer.Len()+w.templateLength > w.mtu { + if count > 0 { + err := goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) + if err != nil { + return err + } + clear(frames[:count]) + count = 0 + } + err := w.transmit(buffer, destinations[index]) + if err != nil { + return err + } + continue + } + frame := &frames[count] + meta, err := w.preparePacketHeader(frame.header[:], buffer.Bytes(), destinations[index], w.checksumOffload) + if err == errGoFrameDropped { + continue + } else if err != nil { + return err + } + frame.length = w.templateLength + frame.payload = buffer.Bytes() + frame.meta = meta + count++ + if count == len(frames) { + err = goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) + if err != nil { + return err + } + clear(frames[:count]) + count = 0 + } + } + if count == 0 { + return nil + } + return goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) +} + +func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) error { + payload := buffer.Bytes() + fragmented := w.templateLength+len(payload) > w.mtu + meta, err := w.preparePacketHeader(buffer.ExtendHeader(w.templateLength), payload, destination, w.checksumOffload && !fragmented) + if err != nil { + if w.ipVersion == 4 && destination.IsIPv6() { + return E.New("send IPv6 packet to IPv4 connection") + } + return goIgnoreDropped(err) + } + packet := buffer.Bytes() + if !fragmented { + return goIgnoreDropped(w.platformIO.writeFrame([][]byte{packet}, meta)) + } + var ( + fragments [][]byte + fragmentsBuilt bool + ) + if w.ipVersion == 4 { + ipHdr := header.IPv4(packet) + ipHdr.SetID(uint16(w.engine.stack.udpIdentification.Add(1))) + ipHdr.SetChecksum(0) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + fragments, fragmentsBuilt = fragmentIPv4Packet(ipHdr, uint32(w.mtu)) + } else { + fragments, fragmentsBuilt = fragmentIPv6Packet(header.IPv6(packet), uint32(w.mtu), w.engine.stack.udpIdentification.Add(1)) + } + if !fragmentsBuilt { + return nil + } + var writeErr error + for _, fragment := range fragments { + writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeFrame([][]byte{fragment}, ForwardFrameMeta{}))) + } + return writeErr +} + +func (w *GoPacketConn) HandshakeSuccess() error { + w.snapshotAccess.Lock() + w.snapshot = nil + w.snapshotAccess.Unlock() + return nil +} + +func (w *GoPacketConn) HandshakeFailure(err error) error { + w.snapshotAccess.Lock() + snapshot := w.snapshot + snapshotMeta := w.snapshotMeta + w.snapshot = nil + w.snapshotAccess.Unlock() + if snapshot == nil { + return os.ErrInvalid + } + snapshotMeta.completeChecksum(snapshot) + reply, ok := BuildUnreachable(snapshot, netip.Addr{}, 0) + if !ok { + return nil + } + return goIgnoreDropped(w.platformIO.writeFrame([][]byte{reply}, ForwardFrameMeta{})) +} + +func (w *GoPacketConn) closeSplice(err error) { + w.spliceCloseError.CompareAndSwap(nil, &goConnError{err: err}) + w.engine.postMessage(&w.closeMessage) +} + +func (w *GoPacketConn) Close() error { + directory := &w.engine.stack.directory + if directory.udpFlows != nil { + directory.access.Lock() + if directory.udpFlows[w.key] == w { + delete(directory.udpFlows, w.key) + } + directory.access.Unlock() + } + if w.splicePending.Load() != nil || w.spliceActive.Load() { + w.closeSplice(io.ErrClosedPipe) + } + return nil +} + +func (w *GoPacketConn) CloseFlow() { + conn := w.conn.Load() + if conn != nil { + conn.Close() + } +} + +func (w *GoPacketConn) handleSessionClose(err error) { + trackerPointer := w.tracker.Load() + if trackerPointer == nil { + return + } + if !w.trackerClosed.CompareAndSwap(false, true) { + return + } + if err != nil && !E.IsClosedOrCanceled(err) { + (*trackerPointer).CloseFlow(FlowCloseReset) + } else { + (*trackerPointer).CloseFlow(FlowCloseTimeout) + } +} + +var ( + _ N.PacketWriter = (*GoPacketConn)(nil) + _ N.FrontHeadroom = (*GoPacketConn)(nil) + _ N.HandshakeSuccess = (*GoPacketConn)(nil) + _ N.HandshakeFailure = (*GoPacketConn)(nil) + _ io.Closer = (*GoPacketConn)(nil) + _ FlowHandle = (*GoPacketConn)(nil) + _ N.PacketBatchWriteCreator = (*GoPacketConn)(nil) + _ N.PacketBatchWriter = (*GoPacketConn)(nil) +) diff --git a/stack_go_wheel.go b/stack_go_wheel.go new file mode 100644 index 00000000..4ff7b157 --- /dev/null +++ b/stack_go_wheel.go @@ -0,0 +1,148 @@ +package tun + +import ( + "math" + "time" +) + +const ( + goWheelTick = int64(time.Millisecond) + goWheelLevel0Slots = 128 + goWheelLevel1Slots = 64 + goWheelLevel1Ticks = goWheelLevel0Slots + goWheelRotationTicks = goWheelLevel1Slots * goWheelLevel1Ticks +) + +type goWheelNode struct { + prev *goWheelNode + next *goWheelNode + slot **goWheelNode + deadline int64 + expire func(now int64) +} + +type goWheel struct { + level0 [goWheelLevel0Slots]*goWheelNode + level1 [goWheelLevel1Slots]*goWheelNode + overflow *goWheelNode + currentTick int64 + scheduled int +} + +func (w *goWheel) schedule(node *goWheelNode, deadline int64) { + if node.slot != nil { + w.unlink(node) + } else { + w.scheduled++ + } + node.deadline = deadline + w.file(node) +} + +func (w *goWheel) cancel(node *goWheelNode) { + if node.slot == nil { + return + } + w.unlink(node) + w.scheduled-- +} + +func (w *goWheel) file(node *goWheelNode) { + tick := max((node.deadline+goWheelTick-1)/goWheelTick, w.currentTick+1) + var slot **goWheelNode + if tick-w.currentTick < goWheelLevel0Slots { + slot = &w.level0[tick%goWheelLevel0Slots] + } else if tick/goWheelLevel1Ticks-w.currentTick/goWheelLevel1Ticks < goWheelLevel1Slots { + slot = &w.level1[(tick/goWheelLevel1Ticks)%goWheelLevel1Slots] + } else { + slot = &w.overflow + } + node.prev = nil + node.next = *slot + if node.next != nil { + node.next.prev = node + } + *slot = node + node.slot = slot +} + +func (w *goWheel) unlink(node *goWheelNode) { + if node.prev != nil { + node.prev.next = node.next + } else { + *node.slot = node.next + } + if node.next != nil { + node.next.prev = node.prev + } + node.prev = nil + node.next = nil + node.slot = nil +} + +func (w *goWheel) advance(now int64) { + targetTick := now / goWheelTick + for w.currentTick < targetTick { + if w.scheduled == 0 { + w.currentTick = targetTick + return + } + w.currentTick++ + if w.currentTick%goWheelLevel1Ticks == 0 { + if w.currentTick%goWheelRotationTicks == 0 { + w.refile(&w.overflow) + } + w.refile(&w.level1[(w.currentTick/goWheelLevel1Ticks)%goWheelLevel1Slots]) + } + w.fireSlot(&w.level0[w.currentTick%goWheelLevel0Slots], now) + } +} + +func (w *goWheel) refile(slot **goWheelNode) { + chain := *slot + *slot = nil + for chain != nil { + node := chain + chain = chain.next + node.prev = nil + node.next = nil + node.slot = nil + w.file(node) + } +} + +func (w *goWheel) fireSlot(slot **goWheelNode, now int64) { + for *slot != nil { + node := *slot + w.unlink(node) + w.scheduled-- + node.expire(now) + } +} + +func (w *goWheel) nextDeadline() (int64, bool) { + if w.scheduled == 0 { + return 0, false + } + deadline := int64(math.MaxInt64) + for offset := int64(1); offset < goWheelLevel0Slots; offset++ { + tick := w.currentTick + offset + if w.level0[tick%goWheelLevel0Slots] != nil { + deadline = tick * goWheelTick + break + } + } + level1Index := w.currentTick / goWheelLevel1Ticks + for offset := int64(1); offset < goWheelLevel1Slots; offset++ { + index := level1Index + offset + if w.level1[index%goWheelLevel1Slots] != nil { + deadline = min(deadline, index*goWheelLevel1Ticks*goWheelTick) + break + } + } + if w.overflow != nil { + rotation := (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks * goWheelTick + deadline = min(deadline, rotation) + } + return deadline, true +} diff --git a/stack_gvisor_filter.go b/stack_gvisor_filter.go index 6f889ed0..96a245f5 100644 --- a/stack_gvisor_filter.go +++ b/stack_gvisor_filter.go @@ -32,7 +32,7 @@ func (w *LinkEndpointFilter) Attach(dispatcher stack.NetworkDispatcher) { NetworkDispatcher: dispatcher, broadcastAddress: w.BroadcastAddress, writer: w.Writer, - dispatcher: w.Dispatcher, + dispatchStage: w.Dispatcher.NewStage(nil), inet4Address: w.Inet4Address, inet6Address: w.Inet6Address, inet4LoopbackAddress: w.Inet4LoopbackAddress, @@ -46,7 +46,7 @@ type networkDispatcherFilter struct { stack.NetworkDispatcher broadcastAddress netip.Addr writer GVisorTun - dispatcher *ForwardDispatcher + dispatchStage *ForwardStage inet4Address netip.Addr inet6Address netip.Addr inet4LoopbackAddress []netip.Addr @@ -73,10 +73,10 @@ func (w *networkDispatcherFilter) DeliverNetworkPacket(protocol tcpip.NetworkPro w.writer.WritePacket(pkt) return } - if w.dispatcher != nil && pkt.GSOOptions.Type == stack.GSONone && !pkt.GSOOptions.NeedsCsum { + if w.dispatchStage != nil && pkt.GSOOptions.Type == stack.GSONone && !pkt.GSOOptions.NeedsCsum { if view, loaded := pkt.Data().PullUp(pkt.Data().Size()); loaded { consumed := w.dispatch(protocol, destination, view) - w.dispatcher.Flush() + w.dispatchStage.Flush() if consumed { return } @@ -113,5 +113,5 @@ func (w *networkDispatcherFilter) dispatch(protocol tcpip.NetworkProtocolNumber, } } } - return w.dispatcher.Dispatch(view) + return w.dispatchStage.Dispatch(view) } diff --git a/stack_mixed.go b/stack_mixed.go index cafc9e7c..50b53603 100644 --- a/stack_mixed.go +++ b/stack_mixed.go @@ -127,7 +127,7 @@ func (m *Mixed) tunLoop() { m.logger.Trace(E.Cause(err, "write packet")) } } - m.dispatcher.Flush() + m.dispatchStage.Flush() } } @@ -147,7 +147,7 @@ func (m *Mixed) wintunLoop(winTun WinTun) { m.logger.Trace(E.Cause(err, "write packet")) } } - m.dispatcher.Flush() + m.dispatchStage.Flush() release() } } @@ -195,7 +195,7 @@ func (m *Mixed) batchLoopLinux(linuxTUN LinuxTUN, batchSize int) { } writeBuffers = writeBuffers[:0] } - m.dispatcher.Flush() + m.dispatchStage.Flush() } } @@ -241,7 +241,7 @@ func (m *Mixed) batchLoopDarwin(darwinTUN DarwinTUN) { } buf.ReleaseMulti(writeBuffers) } - m.dispatcher.Flush() + m.dispatchStage.Flush() buf.ReleaseMulti(releaseBuffers) } } diff --git a/stack_system.go b/stack_system.go index 3ff1c4ff..ad87433f 100644 --- a/stack_system.go +++ b/stack_system.go @@ -52,6 +52,7 @@ type System struct { udpNat *UDPNat udpNATOptions UDPNatOptions dispatcher *ForwardDispatcher + dispatchStage *ForwardStage bindInterface bool interfaceFinder control.InterfaceFinder frontHeadroom int @@ -201,6 +202,7 @@ func (s *System) start() error { } if s.handler != nil { s.dispatcher = NewForwardDispatcher(s.handler, newSystemWriteback(s.tun, s.frontHeadroom), s.logger, s.udpTimeout, s.icmpTimeout) + s.dispatchStage = s.dispatcher.NewStage(nil) } return nil } @@ -248,7 +250,7 @@ func (s *System) tunLoop() { s.logger.Trace(E.Cause(err, "write packet")) } } - s.dispatcher.Flush() + s.dispatchStage.Flush() } } @@ -268,7 +270,7 @@ func (s *System) wintunLoop(winTun WinTun) { s.logger.Trace(E.Cause(err, "write packet")) } } - s.dispatcher.Flush() + s.dispatchStage.Flush() release() } } @@ -316,7 +318,7 @@ func (s *System) batchLoopLinux(linuxTUN LinuxTUN, batchSize int) { } writeBuffers = writeBuffers[:0] } - s.dispatcher.Flush() + s.dispatchStage.Flush() } } @@ -362,7 +364,7 @@ func (s *System) batchLoopDarwin(darwinTUN DarwinTUN) { } buf.ReleaseMulti(writeBuffers) } - s.dispatcher.Flush() + s.dispatchStage.Flush() buf.ReleaseMulti(releaseBuffers) } } @@ -420,7 +422,7 @@ func (s *System) dispatchIPv4(ipHdr header.IPv4, destination netip.Addr) bool { return false } } - return s.dispatcher.Dispatch(ipHdr) + return s.dispatchStage.Dispatch(ipHdr) } func (s *System) dispatchIPv6(ipHdr header.IPv6, destination netip.Addr) bool { @@ -439,7 +441,7 @@ func (s *System) dispatchIPv6(ipHdr header.IPv6, destination netip.Addr) bool { return false } } - return s.dispatcher.Dispatch(ipHdr) + return s.dispatchStage.Dispatch(ipHdr) } func (s *System) processIPv4(ipHdr header.IPv4) (writeBack bool, err error) { @@ -729,8 +731,19 @@ func (s *System) processIPv4ICMP(ipHdr header.IPv4, icmpHdr header.ICMPv4) (bool if len(icmpHdr) < header.ICMPv4MinimumSize { return false, E.New("ipv4: icmp: invalid packet") } + return rewriteEchoReplyIPv4(ipHdr, icmpHdr), nil +} + +func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool, error) { + if len(icmpHdr) < header.ICMPv6MinimumSize { + return false, E.New("ipv6: icmp: invalid packet") + } + return rewriteEchoReplyIPv6(ipHdr, icmpHdr), nil +} + +func rewriteEchoReplyIPv4(ipHdr header.IPv4, icmpHdr header.ICMPv4) bool { if icmpHdr.Type() != header.ICMPv4Echo || icmpHdr.Code() != 0 { - return false, nil + return false } icmpHdr.SetType(header.ICMPv4EchoReply) sourceAddress := ipHdr.SourceAddr() @@ -738,15 +751,12 @@ func (s *System) processIPv4ICMP(ipHdr header.IPv4, icmpHdr header.ICMPv4) (bool ipHdr.SetDestinationAddr(sourceAddress) icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - return true, nil + return true } -func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool, error) { - if len(icmpHdr) < header.ICMPv6MinimumSize { - return false, E.New("ipv6: icmp: invalid packet") - } +func rewriteEchoReplyIPv6(ipHdr header.IPv6, icmpHdr header.ICMPv6) bool { if icmpHdr.Type() != header.ICMPv6EchoRequest || icmpHdr.Code() != 0 { - return false, nil + return false } icmpHdr.SetType(header.ICMPv6EchoReply) sourceAddress := ipHdr.SourceAddr() @@ -757,7 +767,7 @@ func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool Src: ipHdr.SourceAddressSlice(), Dst: ipHdr.DestinationAddressSlice(), })) - return true, nil + return true } type systemUDPPacketWriter4 struct { @@ -772,9 +782,8 @@ func (w *systemUDPPacketWriter4) FrontHeadroom() int { return w.frontHeadroom + len(w.header) } -func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { +func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) { payloadLen := buffer.Len() - buffer = (N.ReadWaitOptions{FrontHeadroom: w.FrontHeadroom()}).Copy(buffer) copy(buffer.ExtendHeader(len(w.header)), w.header) ipHdr := header.IPv4(buffer.Bytes()) ipHdr.SetTotalLength(uint16(buffer.Len())) @@ -796,22 +805,20 @@ func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M udpHdr.SetChecksum(0) } ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - return buffer } -func (w *systemUDPPacketWriter4) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { - buffer = w.preparePacket(buffer, destination) +func (w *systemUDPPacketWriter4) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) { + w.preparePacket(buffer, destination) if PacketOffset > 0 { PacketFillHeader(buffer.ExtendHeader(PacketOffset), header.IPv4Version) } if remainingHeadroom := w.frontHeadroom - PacketOffset; remainingHeadroom > 0 { buffer.Advance(-remainingHeadroom) } - return buffer } func (w *systemUDPPacketWriter4) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - buffer = w.prepareWritePacket(buffer, destination) + w.prepareWritePacket(buffer, destination) defer buffer.Release() return common.Error(w.tun.Write(buffer.Bytes())) } @@ -837,15 +844,14 @@ func (w *systemUDPPacketWriter4) WritePacketBatch(buffers []*buf.Buffer, destina case LinuxTUN: packets := make([][]byte, len(buffers)) for index, buffer := range buffers { - buffer = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) buffer.Advance(-w.frontHeadroom) - buffers[index] = buffer packets[index] = buffer.Bytes() } return common.Error(tunInterface.BatchWrite(packets, w.frontHeadroom)) case DarwinTUN: for index, buffer := range buffers { - buffers[index] = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) } return tunInterface.BatchWrite(buffers) default: @@ -865,9 +871,8 @@ func (w *systemUDPPacketWriter6) FrontHeadroom() int { return w.frontHeadroom + len(w.header) } -func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { +func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) { payloadLen := buffer.Len() - buffer = (N.ReadWaitOptions{FrontHeadroom: w.FrontHeadroom()}).Copy(buffer) copy(buffer.ExtendHeader(len(w.header)), w.header) ipHdr := header.IPv6(buffer.Bytes()) udpLen := uint16(header.UDPMinimumSize + payloadLen) @@ -889,22 +894,20 @@ func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M } else { udpHdr.SetChecksum(0) } - return buffer } -func (w *systemUDPPacketWriter6) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { - buffer = w.preparePacket(buffer, destination) +func (w *systemUDPPacketWriter6) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) { + w.preparePacket(buffer, destination) if PacketOffset > 0 { PacketFillHeader(buffer.ExtendHeader(PacketOffset), header.IPv6Version) } if remainingHeadroom := w.frontHeadroom - PacketOffset; remainingHeadroom > 0 { buffer.Advance(-remainingHeadroom) } - return buffer } func (w *systemUDPPacketWriter6) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - buffer = w.prepareWritePacket(buffer, destination) + w.prepareWritePacket(buffer, destination) defer buffer.Release() return common.Error(w.tun.Write(buffer.Bytes())) } @@ -930,15 +933,14 @@ func (w *systemUDPPacketWriter6) WritePacketBatch(buffers []*buf.Buffer, destina case LinuxTUN: packets := make([][]byte, len(buffers)) for index, buffer := range buffers { - buffer = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) buffer.Advance(-w.frontHeadroom) - buffers[index] = buffer packets[index] = buffer.Bytes() } return common.Error(tunInterface.BatchWrite(packets, w.frontHeadroom)) case DarwinTUN: for index, buffer := range buffers { - buffers[index] = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) } return tunInterface.BatchWrite(buffers) default: diff --git a/tun.go b/tun.go index 53c032c4..12f70651 100644 --- a/tun.go +++ b/tun.go @@ -73,6 +73,7 @@ type Options struct { Inet6Address []netip.Prefix MTU uint32 GSO bool + MultiQueue bool AutoRoute bool InterfaceScope bool Inet4Gateway netip.Addr diff --git a/tun_darwin.go b/tun_darwin.go index 46f9922c..2c5ff863 100644 --- a/tun_darwin.go +++ b/tun_darwin.go @@ -386,6 +386,31 @@ func raiseReceiveBuffer(tunFd int) int { return current } +func (t *NativeTun) rawFileDescriptor() int { + return t.tunFd +} + +// os.NewFile registers a non-blocking descriptor with the runtime poller, which then wakes an +// idle thread for every packet the engine loop is already waiting for on its own kqueue. +func (t *NativeTun) detachRuntimePoller() error { + duplicated, err := unix.FcntlInt(uintptr(t.tunFd), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return err + } + previous := t.tunFile + t.tunFd = duplicated + t.tunFile = newUnpolledFile(duplicated, "utun") + return previous.Close() +} + +func (t *NativeTun) transmitAccess() *sync.Mutex { + return &t.writeAccess +} + +func (t *NativeTun) enableMaxPendingPackets() error { + return configure(t.tunFd, true, int(t.options.MTU)) +} + func (t *NativeTun) BatchRead() ([]*buf.Buffer, error) { for i := 0; i < t.batchSize; i++ { iovecs := t.iovecs[i].nextIovecs() diff --git a/tun_linux.go b/tun_linux.go index 82b98d8e..aaaedc8e 100644 --- a/tun_linux.go +++ b/tun_linux.go @@ -39,6 +39,7 @@ type NativeTun struct { readAccess sync.Mutex writeAccess sync.Mutex vnetHdr bool + multiQueue bool writeBuffer []byte readRawConn syscall.RawConn pendingBuffer []byte @@ -54,7 +55,7 @@ type NativeTun struct { func New(options Options) (Tun, error) { if options.FileDescriptor == 0 { return execInNetworkNamespace(options.NetNs, func() (Tun, error) { - tunFd, err := open(options.Name, options.GSO) + tunFd, multiQueue, err := open(options.Name, options.GSO, options.MultiQueue) if err != nil { return nil, E.Cause(err, "open tun") } @@ -63,9 +64,10 @@ func New(options Options) (Tun, error) { return nil, E.Errors(err, unix.Close(tunFd)) } nativeTun := &NativeTun{ - tunFd: tunFd, - tunFile: os.NewFile(uintptr(tunFd), "tun"), - options: options, + tunFd: tunFd, + tunFile: os.NewFile(uintptr(tunFd), "tun"), + options: options, + multiQueue: multiQueue, } err = nativeTun.configure(tunLink) if err != nil { @@ -102,7 +104,30 @@ func init() { } } -func open(name string, vnetHdr bool) (int, error) { +// A persistent device created without IFF_MULTI_QUEUE refuses the flag with EINVAL +// (tun_set_iff), so the plain flags are retried and such a device stays single-queue. +func open(name string, vnetHdr bool, multiQueue bool) (int, bool, error) { + flags := unix.IFF_TUN | unix.IFF_NO_PI + if vnetHdr { + flags |= unix.IFF_VNET_HDR + } + if multiQueue { + fd, err := openTun(name, flags|unix.IFF_MULTI_QUEUE) + if err == nil { + return fd, true, nil + } + if !errors.Is(err, unix.EINVAL) { + return -1, false, err + } + } + fd, err := openTun(name, flags) + if err != nil { + return -1, false, err + } + return fd, false, nil +} + +func openTun(name string, flags int) (int, error) { fd, err := unix.Open(controlPath, unix.O_RDWR, 0) if err != nil { return -1, E.Cause(err, "open ", controlPath) @@ -110,22 +135,38 @@ func open(name string, vnetHdr bool) (int, error) { ifr, err := unix.NewIfreq(name) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "create ifreq") - } - flags := unix.IFF_TUN | unix.IFF_NO_PI - if vnetHdr { - flags |= unix.IFF_VNET_HDR + return -1, E.Cause(err, "create ifreq") } ifr.SetUint16(uint16(flags)) err = unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "TUNSETIFF") + return -1, E.Cause(err, "TUNSETIFF") } err = unix.SetNonblock(fd, true) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "set nonblock") + return -1, E.Cause(err, "set nonblock") + } + return fd, nil +} + +func (t *NativeTun) openQueue() (int, error) { + if !t.multiQueue { + return -1, E.New("tun device is not multi-queue") + } + flags := unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE + if t.options.GSO { + flags |= unix.IFF_VNET_HDR + } + var fd int + err := runInNetworkNamespace(t.options.NetNs, func() error { + var openErr error + fd, openErr = openTun(t.options.Name, flags) + return openErr + }) + if err != nil { + return -1, err } return fd, nil } @@ -200,8 +241,6 @@ func (t *NativeTun) enableGSO() error { return E.Cause(err, "enable offload: IFF_VNET_HDR not enabled") } t.vnetHdr = true - t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) - t.pendingBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) t.tcpGROTable = newTCPGROTable() t.udpGROTable = newUDPGROTable() err = setTCPOffload(t.tunFd) @@ -404,6 +443,9 @@ func (t *NativeTun) Close() error { func (t *NativeTun) Read(p []byte) (n int, err error) { if t.vnetHdr { + if t.writeBuffer == nil { + t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) + } n, err = t.tunFile.Read(t.writeBuffer) if err != nil { if errors.Is(err, syscall.EBADFD) { @@ -526,6 +568,9 @@ func (t *NativeTun) BatchRead(buffers [][]byte, offset int, readN []int) (int, e used = count } for used < len(buffers) { + if t.writeBuffer == nil { + t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) + } var ( readLength int err error @@ -629,6 +674,28 @@ func (t *NativeTun) TXChecksumOffload() bool { return t.txChecksumOffload } +func (t *NativeTun) rawFileDescriptor() int { + return t.tunFd +} + +// os.NewFile registers a non-blocking descriptor with the runtime poller, which then wakes an +// idle thread for every packet the engine loop is already waiting for on its own epoll. +func (t *NativeTun) detachRuntimePoller() error { + duplicated, err := unix.FcntlInt(uintptr(t.tunFd), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return err + } + previous := t.tunFile + t.tunFd = duplicated + t.tunFile = newUnpolledFile(duplicated, "tun") + t.readRawConn, err = t.tunFile.SyscallConn() + return E.Errors(err, previous.Close()) +} + +func (t *NativeTun) vnetHeaderEnabled() (bool, error) { + return checkVNETHDREnabled(t.tunFd, t.options.Name) +} + func prefixToIPNet(prefix netip.Prefix) *net.IPNet { return &net.IPNet{ IP: prefix.Addr().AsSlice(), diff --git a/tun_windows.go b/tun_windows.go index 6dfce2f2..29ad40e9 100644 --- a/tun_windows.go +++ b/tun_windows.go @@ -550,13 +550,86 @@ func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { return 0, fmt.Errorf("write failed: %w", err) } +func (t *NativeTun) readWaitHandle() windows.Handle { + return t.readWait +} + +func (t *NativeTun) resizeSessionRing(capacity uint32) error { + t.session.End() + t.session = wintun.Session{} + t.readWait = 0 + session, err := t.adapter.StartSession(capacity) + if err != nil { + return E.Cause(err, "restart wintun session") + } + t.session = session + t.readWait = session.ReadWaitEvent() + return nil +} + +func (t *NativeTun) receiveInto(buffer []byte) (int, error) { + t.running.Add(1) + defer t.running.Done() + for { + if t.close.Load() == 1 { + return 0, os.ErrClosed + } + packet, err := t.session.ReceivePacket() + if err != nil { + switch err { + case windows.ERROR_NO_MORE_ITEMS: + return 0, nil + case windows.ERROR_HANDLE_EOF: + return 0, os.ErrClosed + case windows.ERROR_INVALID_DATA: + return 0, E.New("wintun: receive ring corrupt") + } + return 0, E.Cause(err, "wintun: receive packet") + } + if len(packet) > len(buffer) { + t.session.ReleaseReceivePacket(packet) + continue + } + n := copy(buffer, packet) + t.session.ReleaseReceivePacket(packet) + return n, nil + } +} + +func (t *NativeTun) transmitGather(segments [][]byte) error { + t.running.Add(1) + defer t.running.Done() + if t.close.Load() == 1 { + return os.ErrClosed + } + var packetSize int + for _, segment := range segments { + packetSize += len(segment) + } + packet, err := t.session.AllocateSendPacket(packetSize) + if err != nil { + if err == windows.ERROR_HANDLE_EOF { + return os.ErrClosed + } + return err + } + var index int + for _, segment := range segments { + index += copy(packet[index:], segment) + } + t.session.SendPacket(packet) + return nil +} + func (t *NativeTun) Close() error { var err error t.closeOnce.Do(func() { t.close.Store(1) windows.SetEvent(t.readWait) t.running.Wait() - t.session.End() + if t.session != (wintun.Session{}) { + t.session.End() + } t.adapter.Close() if t.fwpmSession != 0 { winsys.FwpmEngineClose0(t.fwpmSession) diff --git a/udp_nat.go b/udp_nat.go index 6d5dd945..32a8b64d 100644 --- a/udp_nat.go +++ b/udp_nat.go @@ -16,7 +16,6 @@ import ( "github.com/sagernet/sing/common/buf" "github.com/sagernet/sing/common/canceler" "github.com/sagernet/sing/common/control" - "github.com/sagernet/sing/common/memory" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" "github.com/sagernet/sing/common/pipe" @@ -92,15 +91,15 @@ type UDPNat struct { timeout time.Duration mapping NATMapping filtering NATFiltering - cache *freelru.Cache[udpNatSessionKey, *udpNatConn] - filterCache *freelru.Cache[udpNatFilterKey, *udpNatConn] + cache *freelru.Cache[udpNatSessionKey, *UDPNatConn] + filterCache *freelru.Cache[udpNatFilterKey, *UDPNatConn] nextFilterSessionID atomic.Uint64 interfaceFinder control.InterfaceFinder excludeInterface []string interfaceElement *list.Element[control.InterfaceUpdateCallback] egress atomic.Pointer[udpNatEgressTable] classAccess sync.Mutex - classConns map[uint32]map[*udpNatConn]struct{} + classConns map[uint32]map[*UDPNatConn]struct{} cleanup *udpNatCleanupQueue state atomic.Uint32 lifecycleAccess sync.Mutex @@ -117,18 +116,16 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { if maxSize == 0 { if runtime.GOOS == "ios" { maxSize = 4096 - } else if totalMemory := memory.Total(); totalMemory == 0 { - maxSize = 16384 } else { - maxSize = uint32(min(max(totalMemory/16384, 4096), 16384)) + maxSize = 16384 } } hasher := maphash.NewHasher[udpNatSessionKey]() - cache := common.Must1(freelru.New[udpNatSessionKey, *udpNatConn](maxSize, hasher.Hash32, options.Shared)) - var filterCache *freelru.Cache[udpNatFilterKey, *udpNatConn] + cache := common.Must1(freelru.New[udpNatSessionKey, *UDPNatConn](maxSize, hasher.Hash32, options.Shared)) + var filterCache *freelru.Cache[udpNatFilterKey, *UDPNatConn] if NATMapping(options.Filtering) > options.Mapping { filterHasher := maphash.NewHasher[udpNatFilterKey]() - filterCache = common.Must1(freelru.New[udpNatFilterKey, *udpNatConn](maxSize, filterHasher.Hash32, options.Shared)) + filterCache = common.Must1(freelru.New[udpNatFilterKey, *UDPNatConn](maxSize, filterHasher.Hash32, options.Shared)) } service := &UDPNat{ handler: options.Handler, @@ -140,12 +137,12 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { filterCache: filterCache, interfaceFinder: options.InterfaceFinder, excludeInterface: options.ExcludeInterface, - classConns: make(map[uint32]map[*udpNatConn]struct{}), + classConns: make(map[uint32]map[*UDPNatConn]struct{}), cleanupDone: make(chan struct{}), } service.cleanup = newUDPNatCleanupQueue(service) cache.SetLifetime(options.Timeout) - cache.SetHealthCheck(func(_ udpNatSessionKey, conn *udpNatConn) bool { + cache.SetHealthCheck(func(_ udpNatSessionKey, conn *UDPNatConn) bool { select { case <-conn.doneChan: return false @@ -153,11 +150,11 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { return true } }) - cache.SetOnEvict(func(_ udpNatSessionKey, conn *udpNatConn) { + cache.SetOnEvict(func(_ udpNatSessionKey, conn *UDPNatConn) { conn.closeFromCache() }) if filterCache != nil { - filterCache.SetOnEvict(func(key udpNatFilterKey, conn *udpNatConn) { + filterCache.SetOnEvict(func(key udpNatFilterKey, conn *UDPNatConn) { conn.removeFilterPeer(key.peer) }) } @@ -212,7 +209,7 @@ func (s *UDPNat) updateInterfaces(interfaces []control.Interface) { } } s.egress.Store(newUDPNatEgressTable(entries)) - var closeConns []*udpNatConn + var closeConns []*UDPNatConn s.classAccess.Lock() for interfaceIndex, conns := range s.classConns { if !slices.ContainsFunc(entries, func(entry udpNatEgressEntry) bool { @@ -295,18 +292,18 @@ func (t *udpNatEgressTable) lookup(address netip.Addr) uint32 { return 0 } -func (s *UDPNat) registerClass(conn *udpNatConn) { +func (s *UDPNat) registerClass(conn *UDPNatConn) { s.classAccess.Lock() conns := s.classConns[conn.interfaceIndex] if conns == nil { - conns = make(map[*udpNatConn]struct{}) + conns = make(map[*UDPNatConn]struct{}) s.classConns[conn.interfaceIndex] = conns } conns[conn] = struct{}{} s.classAccess.Unlock() } -func (s *UDPNat) unregisterClass(conn *udpNatConn) { +func (s *UDPNat) unregisterClass(conn *UDPNatConn) { s.classAccess.Lock() conns := s.classConns[conn.interfaceIndex] if conns != nil { @@ -319,7 +316,7 @@ func (s *UDPNat) unregisterClass(conn *udpNatConn) { } func (s *UDPNat) NewPacket(bufferSlices [][]byte, source M.Socksaddr, destination M.Socksaddr, userData any) { - conn, ok := s.getOrCreateConn(source, destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(source, destination), source, destination, userData) if !ok { return } @@ -336,10 +333,7 @@ func (s *UDPNat) NewPacket(bufferSlices [][]byte, source M.Socksaddr, destinatio conn.enqueue(buffer, destination) } -func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, userData any) (*udpNatConn, bool) { - if s.state.Load() != udpNatStateStarted { - return nil, false - } +func (s *UDPNat) sessionKey(source M.Socksaddr, destination M.Socksaddr) udpNatSessionKey { key := udpNatSessionKey{ sourceAddr: source.Addr.Unmap(), sourcePort: source.Port, @@ -353,16 +347,23 @@ func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, us key.destinationAddr = destination.Addr.Unmap() key.destinationPort = destination.Port } + return key +} + +func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destination M.Socksaddr, userData any) (*UDPNatConn, bool) { + if s.state.Load() != udpNatStateStarted { + return nil, false + } var ( newContext context.Context newOnClose N.CloseHandlerFunc ) - conn, loaded, ok := s.cache.GetAndRefreshOrAdd(key, func() (*udpNatConn, bool) { + conn, loaded, ok := s.cache.GetAndRefreshOrAdd(key, func() (*UDPNatConn, bool) { ok, ctx, writer, onClose := s.prepare(source, destination, userData) if !ok { return nil, false } - newConn := &udpNatConn{ + newConn := &UDPNatConn{ service: s, key: key, writer: writer, @@ -419,7 +420,7 @@ func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, us return conn, true } -func (c *udpNatConn) enqueue(buffer *buf.Buffer, destination M.Socksaddr) { +func (c *UDPNatConn) enqueue(buffer *buf.Buffer, destination M.Socksaddr) { c.packetAccess.RLock() select { case <-c.doneChan: @@ -448,7 +449,7 @@ func (s *UDPNat) NewPacketBatch(buffers []*buf.Buffer, sources []M.Socksaddr, de return } for index, buffer := range buffers { - conn, ok := s.getOrCreateConn(sources[index], destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(sources[index], destination), sources[index], destination, userData) if !ok { buffer.Release() continue @@ -477,13 +478,13 @@ func (s *UDPNat) PurgeExpired() { } var ( - _ N.PacketConn = (*udpNatConn)(nil) - _ canceler.PacketConn = (*udpNatConn)(nil) - _ N.PacketBatchReadWaitCreator = (*udpNatConn)(nil) - _ N.PacketBatchWriteCreator = (*udpNatConn)(nil) + _ N.PacketConn = (*UDPNatConn)(nil) + _ canceler.PacketConn = (*UDPNatConn)(nil) + _ N.PacketBatchReadWaitCreator = (*UDPNatConn)(nil) + _ N.PacketBatchWriteCreator = (*UDPNatConn)(nil) ) -type udpNatConn struct { +type UDPNatConn struct { service *UDPNat key udpNatSessionKey interfaceIndex uint32 @@ -509,7 +510,7 @@ type udpNatReadBatch struct { destinations []M.Socksaddr } -func (c *udpNatConn) loadReadWaitOptions() N.ReadWaitOptions { +func (c *UDPNatConn) loadReadWaitOptions() N.ReadWaitOptions { options := c.readWaitOptions.Load() if options == nil { return N.ReadWaitOptions{} @@ -517,7 +518,7 @@ func (c *udpNatConn) loadReadWaitOptions() N.ReadWaitOptions { return *options } -func (c *udpNatConn) addFilterPeer(destination M.Socksaddr) { +func (c *UDPNatConn) addFilterPeer(destination M.Socksaddr) { if c.filterSessionID == 0 || !destination.IsIP() { return } @@ -553,13 +554,13 @@ func (c *udpNatConn) addFilterPeer(destination M.Socksaddr) { } } -func (c *udpNatConn) removeFilterPeer(peer netip.AddrPort) { +func (c *UDPNatConn) removeFilterPeer(peer netip.AddrPort) { c.filterAccess.Lock() delete(c.filterPeers, peer) c.filterAccess.Unlock() } -func (c *udpNatConn) clearFilterPeers() { +func (c *UDPNatConn) clearFilterPeers() { if c.filterSessionID == 0 { return } @@ -575,7 +576,7 @@ func (c *udpNatConn) clearFilterPeers() { } } -func (c *udpNatConn) allowPeer(destination M.Socksaddr) bool { +func (c *UDPNatConn) allowPeer(destination M.Socksaddr) bool { if c.service.filtering == NATFilteringEndpointIndependent || !destination.IsIP() { return true } @@ -593,10 +594,14 @@ func (c *udpNatConn) allowPeer(destination M.Socksaddr) bool { return loaded && filterConn == c } -func (c *udpNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error) { +func (c *UDPNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error) { select { case p := <-c.packetChan: - _, err = buffer.ReadOnceFrom(p.Buffer) + if buffer.IsFull() { + err = io.ErrShortBuffer + } else { + _, err = buffer.Write(p.Buffer.Bytes()) + } destination := p.Destination p.Buffer.Release() N.PutPacketBuffer(p) @@ -608,7 +613,7 @@ func (c *udpNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error } } -func (c *udpNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { +func (c *UDPNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { if !c.allowPeer(destination) { buffer.Release() return nil @@ -616,7 +621,7 @@ func (c *udpNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) er return c.writer.WritePacket(buffer, destination) } -func (c *udpNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { +func (c *UDPNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { if c.service.filtering != NATFilteringEndpointIndependent { return nil, false } @@ -629,16 +634,16 @@ func (c *udpNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { return nil, false } -func (c *udpNatConn) InitializeReadWaiter(options N.ReadWaitOptions) (needCopy bool) { +func (c *UDPNatConn) InitializeReadWaiter(options N.ReadWaitOptions) (needCopy bool) { c.readWaitOptions.Store(&options) return false } -func (c *udpNatConn) WaitReadPacket() (buffer *buf.Buffer, destination M.Socksaddr, err error) { +func (c *UDPNatConn) WaitReadPacket() (buffer *buf.Buffer, destination M.Socksaddr, err error) { return c.waitReadPacket(c.loadReadWaitOptions()) } -func (c *udpNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buffer, destination M.Socksaddr, err error) { +func (c *UDPNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buffer, destination M.Socksaddr, err error) { select { case packet := <-c.packetChan: buffer = options.Copy(packet.Buffer) @@ -652,11 +657,16 @@ func (c *udpNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buff } } -func (c *udpNatConn) CreatePacketBatchReadWaiter() (N.PacketBatchReadWaiter, bool) { +func (c *UDPNatConn) CreatePacketBatchReadWaiter() (N.PacketBatchReadWaiter, bool) { return c, true } -func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M.Socksaddr, err error) { +func (c *UDPNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M.Socksaddr, err error) { + batch := c.readBatch + if batch != nil { + clear(batch.buffers) + clear(batch.destinations) + } options := c.loadReadWaitOptions() buffer, destination, err := c.waitReadPacket(options) if err != nil { @@ -666,13 +676,9 @@ func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M. if batchSize <= 0 { batchSize = 1 } - batch := c.readBatch if batch == nil { batch = new(udpNatReadBatch) c.readBatch = batch - } else { - clear(batch.buffers) - clear(batch.destinations) } buffers = batch.buffers[:0] destinations = batch.destinations[:0] @@ -695,7 +701,7 @@ func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M. return } -func (c *udpNatConn) Timeout() time.Duration { +func (c *UDPNatConn) Timeout() time.Duration { rawConn, lifetime, loaded := c.service.cache.PeekWithLifetime(c.key) if !loaded || rawConn != c { return 0 @@ -706,7 +712,7 @@ func (c *udpNatConn) Timeout() time.Duration { return time.Until(lifetime) } -func (c *udpNatConn) SetTimeout(timeout time.Duration) bool { +func (c *UDPNatConn) SetTimeout(timeout time.Duration) bool { updated := c.service.cache.UpdateLifetime(c.key, c, timeout) if !updated { return false @@ -719,7 +725,7 @@ func (c *udpNatConn) SetTimeout(timeout time.Duration) bool { return true } -func (c *udpNatConn) Close() error { +func (c *UDPNatConn) Close() error { c.close() if c.service.state.Load() == udpNatStateStarted { c.service.cleanup.addOrUpdate(c.cleanupEntry, time.Now()) @@ -727,7 +733,7 @@ func (c *udpNatConn) Close() error { return nil } -func (c *udpNatConn) close() { +func (c *UDPNatConn) close() { c.closeOnce.Do(func() { c.packetAccess.Lock() close(c.doneChan) @@ -746,15 +752,16 @@ func (c *udpNatConn) close() { if c.interfaceIndex != 0 { c.service.unregisterClass(c) } + common.Close(c.writer) }) } -func (c *udpNatConn) closeFromCache() { +func (c *UDPNatConn) closeFromCache() { c.close() c.service.cleanup.remove(c.cleanupEntry) } -func (c *udpNatConn) isClosed() bool { +func (c *UDPNatConn) isClosed() bool { select { case <-c.doneChan: return true @@ -763,27 +770,27 @@ func (c *udpNatConn) isClosed() bool { } } -func (c *udpNatConn) LocalAddr() net.Addr { +func (c *UDPNatConn) LocalAddr() net.Addr { return c.localAddr } -func (c *udpNatConn) RemoteAddr() net.Addr { +func (c *UDPNatConn) RemoteAddr() net.Addr { return M.Socksaddr{} } -func (c *udpNatConn) SetDeadline(t time.Time) error { +func (c *UDPNatConn) SetDeadline(t time.Time) error { return os.ErrInvalid } -func (c *udpNatConn) SetReadDeadline(t time.Time) error { +func (c *UDPNatConn) SetReadDeadline(t time.Time) error { c.readDeadline.Set(t) return nil } -func (c *udpNatConn) SetWriteDeadline(t time.Time) error { +func (c *UDPNatConn) SetWriteDeadline(t time.Time) error { return os.ErrInvalid } -func (c *udpNatConn) Upstream() any { +func (c *UDPNatConn) Upstream() any { return c.writer } diff --git a/udp_nat_cleanup.go b/udp_nat_cleanup.go index c63b8af1..a2d00908 100644 --- a/udp_nat_cleanup.go +++ b/udp_nat_cleanup.go @@ -34,7 +34,7 @@ func (s *UDPNat) Start() error { } type udpNatCleanupEntry struct { - conn *udpNatConn + conn *UDPNatConn deadline time.Time index int } From 292a95fcd889f0e2ba428db69b7db3b6f05a296f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 14 Sep 2026 17:15:34 +0800 Subject: [PATCH 08/25] Fix go stack --- gtcpip/header/checksum.go | 15 +- internal/afd/afd_windows.go | 15 +- internal/afd/zsyscall_windows.go | 11 +- internal/wintun/session_windows.go | 8 +- stack.go | 3 +- stack_go.go | 14 +- stack_go_conn.go | 249 ++-- stack_go_engine.go | 61 +- stack_go_integration_cc_test.go | 482 +++++++ stack_go_integration_cpu_unix_test.go | 20 + stack_go_integration_cpu_windows_test.go | 20 + stack_go_integration_darwin_test.go | 144 +- stack_go_integration_flow_linux_test.go | 203 ++- stack_go_integration_lifecycle_linux_test.go | 66 +- stack_go_integration_lifecycle_test.go | 149 +- stack_go_integration_linux_test.go | 8 + stack_go_integration_packet_flow_test.go | 36 +- ...o_integration_packet_offload_linux_test.go | 9 +- stack_go_integration_packet_test.go | 129 +- stack_go_integration_tcp_test.go | 530 ++++++++ stack_go_integration_test.go | 80 +- stack_go_integration_windows_test.go | 8 + stack_go_io.go | 18 +- stack_go_io_darwin.go | 19 +- stack_go_io_linux.go | 24 +- stack_go_io_windows.go | 24 +- stack_go_profile_default.go | 3 + stack_go_profile_ios.go | 3 + stack_go_ring.go | 135 +- stack_go_socket_windows.go | 55 +- stack_go_splice.go | 7 +- stack_go_tcp_ack.go | 1193 +++++++++++++++++ stack_go_tcp_bbr.go | 598 +++++++++ stack_go_tcp_cong.go | 707 ++++++++++ stack_go_tcp_cubic.go | 298 ++++ stack_go_tcp_input.go | 734 +--------- stack_go_tcp_output.go | 235 +++- stack_go_tcp_rate.go | 120 ++ stack_go_udp.go | 55 +- tun_windows.go | 70 +- 40 files changed, 5092 insertions(+), 1466 deletions(-) create mode 100644 stack_go_integration_cc_test.go create mode 100644 stack_go_integration_cpu_unix_test.go create mode 100644 stack_go_integration_cpu_windows_test.go create mode 100644 stack_go_integration_tcp_test.go create mode 100644 stack_go_tcp_ack.go create mode 100644 stack_go_tcp_bbr.go create mode 100644 stack_go_tcp_cong.go create mode 100644 stack_go_tcp_cubic.go create mode 100644 stack_go_tcp_rate.go diff --git a/gtcpip/header/checksum.go b/gtcpip/header/checksum.go index 303502cc..72274600 100644 --- a/gtcpip/header/checksum.go +++ b/gtcpip/header/checksum.go @@ -22,21 +22,22 @@ import ( "github.com/sagernet/sing-tun/gtcpip" "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/internal/tschecksum" ) // PseudoHeaderChecksum calculates the pseudo-header checksum for the given // destination protocol and network address. Pseudo-headers are needed by // transport layers when calculating their own checksum. func PseudoHeaderChecksum(protocol tcpip.TransportProtocolNumber, srcAddr []byte, dstAddr []byte, totalLen uint16) uint16 { + if (len(srcAddr) == 4 || len(srcAddr) == 16) && (len(dstAddr) == 4 || len(dstAddr) == 16) { + return tschecksum.PseudoHeaderChecksum(uint8(protocol), srcAddr, dstAddr, totalLen) + } xsum := checksum.Checksum(srcAddr, 0) xsum = checksum.Checksum(dstAddr, xsum) - - // Add the length portion of the checksum to the pseudo-checksum. - var tmp [2]byte - binary.BigEndian.PutUint16(tmp[:], totalLen) - xsum = checksum.Checksum(tmp[:], xsum) - - return checksum.Checksum([]byte{0, uint8(protocol)}, xsum) + var trailer [4]byte + binary.BigEndian.PutUint16(trailer[:2], totalLen) + trailer[3] = uint8(protocol) + return checksum.Checksum(trailer[:], xsum) } // checksumUpdate2ByteAlignedUint16 updates a uint16 value in a calculated diff --git a/internal/afd/afd_windows.go b/internal/afd/afd_windows.go index 02d0b7e4..3a012200 100644 --- a/internal/afd/afd_windows.go +++ b/internal/afd/afd_windows.go @@ -114,12 +114,9 @@ func (d *Device) Poll(baseSocket windows.Handle, events uint32, ioStatusBlock *w pollInfo.Handles[0].Status = 0 ioStatusBlock.Status = windows.NTStatus(STATUS_PENDING) size := uint32(unsafe.Sizeof(*pollInfo)) - err := NtDeviceIoControlFile(d.handle, 0, 0, uintptr(unsafe.Pointer(ioStatusBlock)), ioStatusBlock, IOCTL_AFD_POLL, unsafe.Pointer(pollInfo), size, unsafe.Pointer(pollInfo), size) - if err != nil { - if ntstatus, isStatus := err.(windows.NTStatus); isStatus && uint32(ntstatus) == STATUS_PENDING { - return nil - } - return err + status := NtDeviceIoControlFile(d.handle, 0, 0, uintptr(unsafe.Pointer(ioStatusBlock)), ioStatusBlock, IOCTL_AFD_POLL, unsafe.Pointer(pollInfo), size, unsafe.Pointer(pollInfo), size) + if status != 0 && uint32(status) != STATUS_PENDING { + return status } return nil } @@ -188,7 +185,11 @@ func (p *WaitCompletionPacket) Associate(iocp windows.Handle, target windows.Han } func (p *WaitCompletionPacket) Cancel() error { - return NtCancelWaitCompletionPacket(p.handle, 1) + err := NtCancelWaitCompletionPacket(p.handle, 1) + if status, isStatus := err.(windows.NTStatus); isStatus && uint32(status) == STATUS_CANCELLED { + return nil + } + return err } func (p *WaitCompletionPacket) Close() error { diff --git a/internal/afd/zsyscall_windows.go b/internal/afd/zsyscall_windows.go index f56989b8..d6432a54 100644 --- a/internal/afd/zsyscall_windows.go +++ b/internal/afd/zsyscall_windows.go @@ -20,7 +20,7 @@ var ( procNtCancelWaitCompletionPacket = modntdll.NewProc("NtCancelWaitCompletionPacket") ) -func GetQueuedCompletionStatusEx(cphandle windows.Handle, entries *OverlappedEntry, count uint32, numRemoved *uint32, timeout uint32, alertable bool) error { +func GetQueuedCompletionStatusEx(cphandle windows.Handle, entries *OverlappedEntry, count uint32, numRemoved *uint32, timeout uint32, alertable bool) syscall.Errno { var alertableValue uint32 if alertable { alertableValue = 1 @@ -32,7 +32,7 @@ func GetQueuedCompletionStatusEx(cphandle windows.Handle, entries *OverlappedEnt } return syscall.EINVAL } - return nil + return 0 } func NtCancelIoFileEx(handle windows.Handle, ioRequestToCancel *windows.IO_STATUS_BLOCK, ioStatusBlock *windows.IO_STATUS_BLOCK) error { @@ -51,12 +51,9 @@ func NtCreateFile(handle *windows.Handle, access uint32, objectAttributes *Objec return nil } -func NtDeviceIoControlFile(handle windows.Handle, event windows.Handle, apcRoutine uintptr, apcContext uintptr, ioStatusBlock *windows.IO_STATUS_BLOCK, ioControlCode uint32, inputBuffer unsafe.Pointer, inputBufferLength uint32, outputBuffer unsafe.Pointer, outputBufferLength uint32) error { +func NtDeviceIoControlFile(handle windows.Handle, event windows.Handle, apcRoutine uintptr, apcContext uintptr, ioStatusBlock *windows.IO_STATUS_BLOCK, ioControlCode uint32, inputBuffer unsafe.Pointer, inputBufferLength uint32, outputBuffer unsafe.Pointer, outputBufferLength uint32) windows.NTStatus { r0, _, _ := syscall.SyscallN(procNtDeviceIoControlFile.Addr(), uintptr(handle), uintptr(event), uintptr(apcRoutine), uintptr(apcContext), uintptr(unsafe.Pointer(ioStatusBlock)), uintptr(ioControlCode), uintptr(inputBuffer), uintptr(inputBufferLength), uintptr(outputBuffer), uintptr(outputBufferLength)) - if r0 != 0 { - return windows.NTStatus(r0) - } - return nil + return windows.NTStatus(r0) } func NtCreateWaitCompletionPacket(handle *windows.Handle, access uint32, objectAttributes *ObjectAttributes) error { diff --git a/internal/wintun/session_windows.go b/internal/wintun/session_windows.go index 306f3b12..99656491 100644 --- a/internal/wintun/session_windows.go +++ b/internal/wintun/session_windows.go @@ -59,11 +59,11 @@ func (session Session) ReadWaitEvent() (handle windows.Handle) { return } -func (session Session) ReceivePacket() (packet []byte, err error) { +func (session Session) ReceivePacket() (packet []byte, errno syscall.Errno) { var packetSize uint32 r0, _, e1 := syscall.SyscallN(procWintunReceivePacket.Addr(), session.handle, uintptr(unsafe.Pointer(&packetSize))) if r0 == 0 { - err = e1 + errno = e1 return } packet = unsafe.Slice((*byte)(unsafe.Pointer(r0)), packetSize) @@ -74,10 +74,10 @@ func (session Session) ReleaseReceivePacket(packet []byte) { syscall.SyscallN(procWintunReleaseReceivePacket.Addr(), session.handle, uintptr(unsafe.Pointer(&packet[0]))) } -func (session Session) AllocateSendPacket(packetSize int) (packet []byte, err error) { +func (session Session) AllocateSendPacket(packetSize int) (packet []byte, errno syscall.Errno) { r0, _, e1 := syscall.SyscallN(procWintunAllocateSendPacket.Addr(), session.handle, uintptr(packetSize)) if r0 == 0 { - err = e1 + errno = e1 return } packet = unsafe.Slice((*byte)(unsafe.Pointer(r0)), packetSize) diff --git a/stack.go b/stack.go index 8e17419e..3b1e9869 100644 --- a/stack.go +++ b/stack.go @@ -41,6 +41,7 @@ type StackOptions struct { IncludeAllNetworks bool InterfaceFinder control.InterfaceFinder MemoryPressure func() MemoryPressure + TCPCongestionControl string } func NewStack( @@ -49,7 +50,7 @@ func NewStack( ) (Stack, error) { switch stack { case "", "go": - return NewGo(options), nil + return NewGo(options) case "gvisor": return NewGVisor(options) case "mixed": diff --git a/stack_go.go b/stack_go.go index fa470e11..7dea18a5 100644 --- a/stack_go.go +++ b/stack_go.go @@ -30,13 +30,21 @@ type Go struct { udpIdentification atomic.Uint32 dispatcher *ForwardDispatcher directory goFlowDirectory + congestion *goCongestionOps + queueFactory func(stack *Go) ([]goPlatformIO, error) access sync.Mutex engines []*goEngine closed atomic.Bool } -func NewGo(options StackOptions) *Go { +func NewGo(options StackOptions) (*Go, error) { + congestion, err := goLookupCongestionControl(options.TCPCongestionControl) + if err != nil { + return nil, err + } return &Go{ + congestion: congestion, + queueFactory: newGoPlatformQueues, ctx: options.Context, tun: options.Tun, mtu: int(options.TunOptions.MTU), @@ -56,7 +64,7 @@ func NewGo(options StackOptions) *Go { ExcludeInterface: []string{options.TunOptions.Name}, }, memoryPressure: options.MemoryPressure, - } + }, nil } func (s *Go) Start() error { @@ -65,7 +73,7 @@ func (s *Go) Start() error { if s.closed.Load() { return E.New("stack is closed") } - queues, err := newGoPlatformQueues(s) + queues, err := s.queueFactory(s) if err != nil { return err } diff --git a/stack_go_conn.go b/stack_go_conn.go index 38199e83..107884ec 100644 --- a/stack_go_conn.go +++ b/stack_go_conn.go @@ -68,100 +68,138 @@ type goConnError struct { } type GoConn struct { - engine *goEngine - key flowKey - source M.Socksaddr - destination M.Socksaddr - epoch int64 - receiveNext uint64 - receiveCapacity uint64 - receiveCapacityMax uint64 - publishedEdge uint64 - receiveRoundTripMark uint64 - receiveRoundTripStamp int64 - receiveSpaceConsumed uint64 - receiveSpaceCopied uint64 - receiveSpaceStamp int64 - receiveChain goSlabChain - oooRanges *goRangeSet - dsackStart uint64 - dsackEnd uint64 - congestionWindow uint32 - slowStartThreshold uint32 - undoCongestionWindow uint32 - undoSlowStartThreshold uint32 - undoRetransmits int32 - smoothedRoundTrip int32 - roundTripVariance int32 - retransmitTimeout int32 - receiveRoundTrip int32 - recoveryPoint uint64 - highestSacked uint64 - highestRetransmit uint64 - pipe uint64 - undoMarker uint64 - undoLimit uint64 - peerWindow uint64 - ackPending uint64 - ackCovered uint64 - lastAckSent uint64 - lastActivity int64 - sweepSentTail uint64 - retransmitPoint uint64 - retransmitHighWater uint64 - windowLeft1 int64 - windowLeft2 int64 - ackedBytes uint32 - keepaliveProbes uint8 - finWait2Since int64 - scoreboard goScoreboard - frtoRecoveryPoint uint64 - frtoSendLimit uint64 - timerNode goWheelNode - retransmitDeadline int64 - probeDeadline int64 - persistDeadline int64 - lingerDeadline int64 - handshakeDeadline int64 - retransmitAttempts uint8 - probeAttempts uint8 - handshakeAttempts uint8 - persistAttempts uint8 - duplicateAckCount uint8 - frtoState uint8 - state uint8 - ipVersion uint8 - deathClass uint8 - peerWindowShift uint8 - localWindowShift uint8 - peerMSS uint16 - effectiveMSS uint16 - lastPeerWindow uint16 - synAckLength uint8 - keyed bool - finPending bool - sackPermitted bool - timestampsEnabled bool - discardReceive bool - finSent bool - finAcked bool - finReceived bool - inRecovery bool - ackForced bool - ackDirty bool - onBlockedList bool - onDyingList bool - clientISN uint32 - sendISN uint32 - finOffset uint64 - dyingSince int64 - ackNext *GoConn - blockedNext *GoConn - dyingNext *GoConn - synAckImage [96]byte - _ [64]byte + engine *goEngine + key flowKey + source M.Socksaddr + destination M.Socksaddr + epoch int64 + receiveNext uint64 + receiveCapacity uint64 + receiveCapacityMax uint64 + publishedEdge uint64 + receiveRoundTripMark uint64 + receiveRoundTripStamp int64 + receiveSpaceConsumed uint64 + receiveSpaceCopied uint64 + receiveSpaceStamp int64 + receiveChain goSlabChain + oooRanges *goRangeSet + dsackStart uint64 + dsackEnd uint64 + smoothedRoundTrip int32 + roundTripVariance int32 + retransmitTimeout int32 + receiveRoundTrip int32 + highestSacked uint64 + peerWindow uint64 + maxPeerWindow uint64 + ackPending uint64 + ackCovered uint64 + lastAckSent uint64 + lastActivity int64 + sweepSentTail uint64 + windowLeft1 int64 + windowLeft2 int64 + keepaliveProbes uint8 + finWait2Since int64 + scoreboard goScoreboard + timerNode goWheelNode + retransmitDeadline int64 + probeDeadline int64 + persistDeadline int64 + lingerDeadline int64 + handshakeDeadline int64 + idleDeadline int64 + pacingDeadline int64 + reorderDeadline int64 + retransmitAttempts uint8 + probeAttempts uint8 + handshakeAttempts uint8 + persistAttempts uint8 + state uint8 + + congestion *goCongestionOps + congestionPrivate any + flight goFlightSummary + roundTripMin goMinMax + congestionState uint8 + frto bool + sackReneging bool + windowLimited bool + rateAppLimited bool + probeRetransmitted bool + congestionWindow uint32 + slowStartThreshold uint32 + congestionClamp uint32 + congestionWindowCount uint32 + congestionWindowUsed uint32 + priorCongestionWindow uint32 + priorSlowStartThreshold uint32 + reductionDelivered uint32 + reductionRetransmits uint32 + reductionSegmentsOut uint32 + totalRetransmits uint64 + duplicateSegments uint64 + delivered uint32 + lost uint32 + renoSacked uint32 + reordering uint32 + reorderingSeen uint32 + maxPacketsOut uint32 + rateDelivered uint32 + rateIntervalMicros uint32 + retransmitStamp uint32 + lastTimestampEcho uint32 + undoRetransmits int32 + deliveredStamp int32 + deliveredTime int64 + lastSendStamp int64 + undoMarker uint64 + highSeq uint64 + probeHighSeq uint64 + rackEndOffset uint64 + rackRoundTripMicros int64 + rackStamp int32 + rackAdvanced bool + rackDSACKSeen bool + rackLastDelivered uint32 + reorderWindowSteps uint32 + reorderWindowPersist uint8 + windowUsageSeq uint64 + congestionWindowStamp int64 + ipVersion uint8 + deathClass uint8 + peerWindowShift uint8 + localWindowShift uint8 + peerMSS uint16 + effectiveMSS uint16 + lastPeerWindow uint16 + synAckLength uint8 + keyed bool + finPending bool + sackPermitted bool + timestampsEnabled bool + discardReceive bool + finSent bool + finAcked bool + finReceived bool + ackForced bool + ackDirty bool + onBlockedList bool + onDyingList bool + clientISN uint32 + sendISN uint32 + finOffset uint64 + dyingSince int64 + ackNext *GoConn + blockedNext *GoConn + dyingNext *GoConn + synAckImage [96]byte + _ [64]byte sendPermit atomic.Uint64 + sendPacketPermit atomic.Uint32 + packetCreditBase atomic.Uint32 sendUnacked atomic.Uint64 sendReleased atomic.Uint64 receiveNextAck atomic.Uint64 @@ -170,7 +208,17 @@ type GoConn struct { receiveAvailable atomic.Uint64 receiveCapacityPublished atomic.Uint64 windowUpdateThreshold atomic.Uint64 + deliveryState atomic.Uint64 + pacingRate atomic.Uint64 + peakFlight atomic.Uint64 + pacingStamp atomic.Int64 + pacingRequest atomic.Int64 + firstSentStamp atomic.Int32 + appLimited atomic.Uint32 + frameLimit atomic.Uint32 tsRecent atomic.Uint32 + permitWindowBound atomic.Bool + windowLimitedSince atomic.Bool ident atomic.Uint32 connState atomic.Uint32 receiveShutdown atomic.Bool @@ -185,9 +233,11 @@ type GoConn struct { writerParked atomic.Bool retransmitArmed atomic.Bool writerActive atomic.Int32 + flushActive atomic.Int32 writerNeeds atomic.Int32 transmitterActive atomic.Int32 transmitOwner atomic.Int32 + dataSegmentsOut atomic.Uint32 _ [64]byte consumedTail atomic.Uint64 @@ -232,6 +282,7 @@ type GoConn struct { windowMessage goMessage blockedMessage goMessage droppedMessage goMessage + pacingMessage goMessage spliceMessage goMessage splicePending atomic.Pointer[goSpliceStream] spliced atomic.Bool @@ -244,6 +295,7 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, d c.source = source c.destination = destination c.epoch = engine.now() + c.lastSendStamp = -1 c.readDeadline = pipe.MakeDeadline() c.writeDeadline = pipe.MakeDeadline() c.readSignal = make(goSignal, 1) @@ -262,6 +314,7 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, d c.windowMessage = goMessage{kind: goMessageConnWindow, conn: c} c.blockedMessage = goMessage{kind: goMessageConnBlocked, conn: c} c.droppedMessage = goMessage{kind: goMessageConnDropped, conn: c} + c.pacingMessage = goMessage{kind: goMessageConnPacing, conn: c} c.spliceMessage = goMessage{kind: goMessageConnSplice, conn: c} } @@ -315,7 +368,17 @@ func (c *GoConn) enterWriter() bool { } func (c *GoConn) exitWriter() { + c.flushActive.Add(1) c.writerActive.Add(-1) + if c.connState.Load() < goConnStateAborted { + sent := c.sentTail.Load() + if c.bufferedTail.Load() > sent { + c.transmitInline() + } else { + c.checkAppLimited(0, c.sendPermit.Load(), sent) + } + } + c.flushActive.Add(-1) c.engine.reapAfterExit() if c.finRequested.Load() { c.engine.postMessage(&c.closeMessage) @@ -613,11 +676,13 @@ func (c *GoConn) parkReader() (bool, error) { func (c *GoConn) claimTarget(target *goPostedReceive) bool { if c.postedTarget.CompareAndSwap(target, nil) { + target.target = nil return false } for !c.targetDone.Load() { <-c.readSignal } + target.target = nil return true } diff --git a/stack_go_engine.go b/stack_go_engine.go index d9aa1da7..6d8caf72 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -35,6 +35,7 @@ const ( goMessageConnWindow goMessageConnBlocked goMessageConnDropped + goMessageConnPacing goMessageConnSplice goMessagePacketSplice goMessagePacketClose @@ -134,6 +135,8 @@ type goEngine struct { wokeHandlerThisBurst bool tunPending bool transmitReady bool + readBuffersHeld bool + idleSince int64 parkedNanos int64 loadWindowStart int64 ackCoalescing bool @@ -171,6 +174,14 @@ type goEngine struct { controlSegments [][]byte sackScratch [goMaxSackBlocks]goSackBlock sackRaw [goMaxSackBlocks]goRawSackBlock + rateSample goRateSample + ackSample goAckSample + + gsoReadOptions N.ReadWaitOptions + gsoBuffers [goGSOMaxSegments]*buf.Buffer + gsoSegments [goGSOMaxSegments][]byte + gsoSizes [goGSOMaxSegments]int + gsoFrame goFrame } func (e *goEngine) singleFrame(packet []byte) [][]byte { @@ -209,6 +220,7 @@ func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine engine.sweepTickNode.expire = engine.expireSweepTick engine.reclaimTickNode.expire = engine.expireReclaimTick engine.refreshCoarseTime() + engine.wheel.currentTick = engine.now() / goWheelTick engine.wheel.schedule(&engine.sweepTickNode, engine.now()+int64(goSweepInterval)) engine.wheel.schedule(&engine.reclaimTickNode, engine.now()+int64(goReclaimInterval)) return engine @@ -219,7 +231,7 @@ func (e *goEngine) run() { defer e.exit() e.loadWindowStart = e.now() for { - parkStart := int64(time.Since(e.epoch)) + parkStart := e.readClock() tunReadable, eventCount, err := e.park() e.parkedNanos += e.now() - parkStart e.updateLoad(e.now()) @@ -332,15 +344,38 @@ func (e *goEngine) park() (bool, int, error) { tunReadable, eventCount, err := e.platformIO.wait(0, e.socketEvents) e.transmitReady = e.platformIO.takeTransmitWritable() if timeout != 0 && !tunReadable && eventCount == 0 && !e.transmitReady && err == nil && e.controlStack.head.Load() == nil { - e.releaseReadBuffers() - tunReadable, eventCount, err = e.platformIO.wait(timeout, e.socketEvents) + tunReadable, eventCount, err = e.platformIO.wait(e.idleTimeout(timeout), e.socketEvents) + } + if tunReadable || eventCount > 0 { + e.readBuffersHeld = true + e.idleSince = -1 } e.engineState.Store(goEngineRunning) e.refreshCoarseTime() return tunReadable, eventCount, err } +func (e *goEngine) idleTimeout(timeout time.Duration) time.Duration { + if !e.readBuffersHeld { + return timeout + } + now := e.now() + if e.idleSince < 0 { + e.idleSince = now + } + releaseAt := e.idleSince + int64(goReadBufferIdle) + if now >= releaseAt || e.slabPool.pressureLevel() != MemoryPressureNone { + e.releaseReadBuffers() + return timeout + } + if timeout == goWaitIndefinite || time.Duration(releaseAt-now) < timeout { + return time.Duration(releaseAt - now) + } + return timeout +} + func (e *goEngine) releaseReadBuffers() { + e.readBuffersHeld = false e.flushPacketUploads() e.flushPacketFrames() clear(e.frames) @@ -351,6 +386,10 @@ func (e *goEngine) releaseReadBuffers() { clear(e.controlSegments[:cap(e.controlSegments)]) buf.ReleaseMulti(e.packetReceiveBuffers[:]) clear(e.packetReceiveBuffers[:]) + buf.ReleaseMulti(e.gsoBuffers[:]) + clear(e.gsoBuffers[:]) + clear(e.gsoSegments[:]) + e.gsoFrame = goFrame{} clear(e.packetMessages[:]) clear(e.packetFrames[:]) clear(e.packetUploads[:]) @@ -380,6 +419,10 @@ func (e *goEngine) postMessage(message *goMessage) { func (e *goEngine) drainControlQueue() { message := e.controlStack.head.Swap(nil) + if message != nil { + e.readBuffersHeld = true + e.idleSince = -1 + } for message != nil { next := message.next message.next = nil @@ -408,6 +451,8 @@ func (e *goEngine) handleMessage(message *goMessage) { e.handleTransmitBlocked(message.conn) case goMessageConnDropped: e.handleDroppedFrames(message.conn) + case goMessageConnPacing: + e.handlePacingRequest(message.conn) case goMessageConnSplice: e.handleSpliceEngage(message.conn) case goMessagePacketSplice: @@ -639,8 +684,12 @@ func (e *goEngine) now() int64 { return e.coarseTime.Load() } +func (e *goEngine) readClock() int64 { + return int64(time.Since(e.epoch)) +} + func (e *goEngine) refreshCoarseTime() int64 { - now := int64(time.Since(e.epoch)) + now := e.readClock() e.coarseTime.Store(now) return now } @@ -864,6 +913,10 @@ func (e *goEngine) expireReassemblyTick(now int64) { } if entry.deadline <= now { entry.active = false + if !e.readBuffersHeld { + entry.buffer.Release() + entry.buffer = nil + } } else { next = min(next, entry.deadline) } diff --git a/stack_go_integration_cc_test.go b/stack_go_integration_cc_test.go new file mode 100644 index 00000000..25112143 --- /dev/null +++ b/stack_go_integration_cc_test.go @@ -0,0 +1,482 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "fmt" + "io" + "maps" + "net" + "runtime" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + N "github.com/sagernet/sing/common/network" +) + +func kernelTCPResult[T any](t *testing.T, result <-chan T, description string) T { + t.Helper() + select { + case value := <-result: + return value + case <-time.After(kernelTCPTimeout): + t.Fatalf("%s timed out", description) + var value T + return value + } +} + +func kernelTCPWrite(conn net.Conn, payload []byte, buffered bool) error { + if buffered { + buffer := buf.NewSize(len(payload)) + copy(buffer.Extend(len(payload)), payload) + return conn.(N.ExtendedWriter).WriteBuffer(buffer) + } + n, err := conn.Write(payload) + if err == nil && n != len(payload) { + return io.ErrShortWrite + } + return err +} + +func TestGoKernelTCPLifecycle(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + t.Run(congestion, func(algorithmTest *testing.T) { + fixture, traffic := newKernelTCPFixture(algorithmTest, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{}) + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, scenario := range []struct { + name string + size int + peerFirst bool + drain bool + }{ + {name: "empty"}, + {name: "empty_peer_first", peerFirst: true}, + {name: "pending_data", size: 16385}, + {name: "acknowledged_data", size: 16385, drain: true}, + {name: "peer_first", size: 16385, peerFirst: true}, + } { + algorithmTest.Run(fmt.Sprintf("ipv6=%v/%s/%s", ipv6, writer, scenario.name), func(test *testing.T) { + test.Parallel() + client, server, conn := traffic.pair(test, fixture, ipv6, writer) + if scenario.peerFirst { + err := kernelTransfer(client, server, kernelPayload(37, 11), false) + if err != nil { + test.Fatal(err) + } + } + payload := kernelPayload(scenario.size, 13) + if scenario.drain { + written := make(chan error, 1) + go func() { written <- kernelTCPWrite(server, payload, writer == "buffer") }() + received := make([]byte, len(payload)) + _, err := io.ReadFull(client, received) + if err != nil || !bytes.Equal(received, payload) { + test.Fatalf("stream before close: %v", err) + } + err = kernelTCPResult(test, written, "write before close") + if err != nil { + test.Fatal(err) + } + traffic.await(test, conn, "drain before close", func(flow kernelTCPFlow) bool { + return flow.acked >= uint64(len(payload)+1) && conn.sendUnacked.Load() == conn.sentTail.Load() + }) + payload = nil + } + err := kernelTransfer(server, client, payload, writer == "buffer") + if err != nil { + test.Fatal("server half-close:", err) + } + if !scenario.peerFirst { + err = kernelTransfer(client, server, kernelPayload(37, 17), false) + if err != nil { + test.Fatal("reply after half-close:", err) + } + } + traffic.await(test, conn, "graceful close", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.acked >= flow.fin && flow.peerFIN && conn.connState.Load() >= goConnStateAborted + }) + }) + } + } + } + }) + } +} + +func TestGoKernelTCPSendBudget(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, phase := range []string{"ack", "control"} { + for _, overlap := range []int{0, goInitialWindow / 2, goInitialWindow - 1} { + for _, buffered := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/overlap=%d/buffer=%v", congestion, phase, overlap, buffered), func(test *testing.T) { + test.Parallel() + var stage atomic.Int32 + var window atomic.Uint32 + entered := make(chan struct{}) + resume := make(chan struct{}) + idle := make(chan struct{}) + release := make(chan struct{}) + checkpoint := func(current string, conn *GoConn) { + if current == phase && stage.CompareAndSwap(0, 1) { + close(entered) + <-resume + } + if current == "controlled" && stage.CompareAndSwap(1, 2) { + window.Store(conn.congestionWindow) + } + if (current == "idle" || current == "receive") && stage.CompareAndSwap(2, 3) { + close(idle) + <-release + } + } + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{checkpoint: checkpoint}) + resumeOnce := sync.OnceFunc(func() { close(resume) }) + releaseOnce := sync.OnceFunc(func() { close(release) }) + test.Cleanup(resumeOnce) + test.Cleanup(releaseOnce) + client, server := fixture.pair(test, false) + mss := int(server.effectiveMSS) + err := kernelTCPWrite(server, kernelPayload(mss, 19), buffered) + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(client, make([]byte, mss)) + if err != nil { + test.Fatal(err) + } + kernelTCPResult(test, entered, "ACK checkpoint") + overlapping := kernelPayload(overlap*mss, 23) + err = kernelTCPWrite(server, overlapping, buffered) + resumeOnce() + if err != nil { + test.Fatal(err) + } + kernelTCPResult(test, idle, "ACK completion") + allowance := int(window.Load()) * mss + payload := kernelPayload(2*allowance, 29) + err = kernelTCPWrite(server, payload, buffered) + if err != nil { + test.Fatal(err) + } + flow := traffic.snapshot(server.key) + if flow.failure != "" || flow.sent-flow.acked > uint64(allowance) { + test.Fatalf("send budget: window=%d bytes, unacknowledged wire data=%d bytes, failure=%q; recent TCP events: %+v", allowance, flow.sent-flow.acked, flow.failure, flow.events) + } + releaseOnce() + expected := slices.Concat(overlapping, payload) + received := make([]byte, len(expected)) + _, err = io.ReadFull(client, received) + if err != nil || !bytes.Equal(received, expected) { + test.Fatalf("stream after ACK backpressure: %v", err) + } + traffic.await(test, server, "send budget stream ACK processing", func(flow kernelTCPFlow) bool { + return flow.processed >= uint64(1+mss+len(expected)) + }) + }) + } + } + } + } +} + +func TestGoKernelTCPWaiting(t *testing.T) { + previous := runtime.GOMAXPROCS(2) + defer runtime.GOMAXPROCS(previous) + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, buffered := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/buffer=%v", congestion, buffered), func(test *testing.T) { + var stage atomic.Int32 + ready := make(chan struct{}) + checkpoint := func(phase string, _ *GoConn) { + if phase == "controlled" { + stage.CompareAndSwap(0, 1) + } + if phase == "idle" && stage.CompareAndSwap(1, 2) { + close(ready) + } + } + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{receiveDelay: 100 * time.Millisecond, checkpoint: checkpoint}) + client, server := fixture.pair(test, false) + size := goInitialWindow * int(server.effectiveMSS) + warmup := kernelPayload(size, 31) + payload := kernelPayload(2*size, 37) + readResult := make(chan error, 1) + go func() { + received := make([]byte, len(warmup)+len(payload)) + _, readErr := io.ReadFull(client, received) + if readErr == nil && !bytes.Equal(received, slices.Concat(warmup, payload)) { + readErr = io.ErrUnexpectedEOF + } + readResult <- readErr + }() + err := kernelTCPWrite(server, warmup, buffered) + if err != nil { + test.Fatal(err) + } + traffic.await(test, server, "warmup ACK", func(flow kernelTCPFlow) bool { return flow.acked >= uint64(size+1) }) + kernelTCPResult(test, ready, "warmup congestion update") + cpuStart := kernelProcessCPU(test) + start := time.Now() + err = kernelTCPWrite(server, payload, buffered) + if err != nil { + test.Fatal(err) + } + err = kernelTCPResult(test, readResult, "paced stream delivery") + if err != nil { + test.Fatal(err) + } + if server.congestion.pacing { + flow := traffic.snapshot(server.key) + var first, last time.Duration + var delivered, rate uint64 + for _, event := range flow.events { + if !event.outgoing || event.sequence < uint64(size+1) || event.end == event.sequence { + continue + } + if delivered == 0 { + first = event.at + } + last = event.at + delivered += event.end - event.sequence + rate = max(rate, event.rate) + } + allowed := rate*uint64(last-first+2*time.Millisecond)/uint64(time.Second) + 2*uint64(server.effectiveMSS) + if delivered == 0 || delivered > allowed { + test.Errorf("pacing envelope: sent=%d bytes over %s at up to %d bytes/s, allowed=%d; recent TCP events: %+v", delivered, last-first, rate, allowed, flow.events) + } + } + traffic.await(test, server, "paced stream ACK processing", func(flow kernelTCPFlow) bool { return flow.processed >= uint64(3*size+1) }) + elapsed := time.Since(start) + cpu := kernelProcessCPU(test) - cpuStart + if cpu > max(20*time.Millisecond, elapsed/3) { + test.Errorf("transfer waiting consumed CPU: stream=%d bytes elapsed=%s CPU=%s", len(payload), elapsed, cpu) + } + }) + } + } +} + +func kernelTCPRecoveryFilter(pattern string, start, end uint64, mss, round int) func(kernelTCPEvent) kernelTCPAction { + attempts := make(map[uint64]int) + highestACK := start + ackDropped := false + return func(event kernelTCPEvent) kernelTCPAction { + if !event.outgoing { + if event.flags != header.TCPFlagAck { + return kernelTCPAction{} + } + duplicate := event.ack == highestACK + highestACK = max(highestACK, event.ack) + if pattern == "duplicate_ack" && duplicate { + return kernelTCPAction{copies: 1 << (round + 1)} + } + if pattern == "ack_loss" && !ackDropped { + ackDropped = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + } + if event.sequence < start || event.sequence >= end || event.end <= event.sequence { + return kernelTCPAction{} + } + attempts[event.sequence]++ + if attempts[event.sequence] != 1 { + return kernelTCPAction{} + } + segment := (event.sequence - start) / uint64(mss) + switch pattern { + case "tail": + return kernelTCPAction{drop: event.end == end} + case "head": + return kernelTCPAction{drop: segment == 0} + case "burst", "duplicate_ack": + return kernelTCPAction{drop: segment < 2} + case "holes": + return kernelTCPAction{drop: segment%3 == 0} + case "reorder": + if segment%3 == 0 { + return kernelTCPAction{delay: 10 * time.Millisecond} + } + } + return kernelTCPAction{} + } +} + +func TestGoKernelTCPRecovery(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, negotiation := range []struct { + name string + config kernelTCPConfig + }{ + {name: "sack_timestamps"}, + {name: "sack", config: kernelTCPConfig{disableTimestamps: true}}, + {name: "legacy", config: kernelTCPConfig{disableSACK: true, disableTimestamps: true}}, + } { + t.Run(congestion+"/"+negotiation.name, func(configTest *testing.T) { + configTest.Parallel() + fixture, traffic := newKernelTCPFixture(configTest, kernelStackConfig{mtu: 1500, congestion: congestion}, negotiation.config) + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, pattern := range []string{"tail", "head", "burst", "holes", "reorder", "duplicate_ack", "ack_loss", "fin"} { + configTest.Run(fmt.Sprintf("ipv6=%v/%s/%s", ipv6, writer, pattern), func(test *testing.T) { + test.Parallel() + client, sender, conn := traffic.pair(test, fixture, ipv6, writer) + mss := int(conn.effectiveMSS) + start := uint64(1) + for round := range 4 { + segments := 8 + if pattern == "tail" && round > 0 { + segments = 1 + } + length := segments*mss - max(round-1, 0)*mss/3 + end := start + uint64(length) + if round > 0 { + traffic.setFilter(conn, kernelTCPRecoveryFilter(pattern, start, end, mss, round)) + } + payload := kernelPayload(length, uint32(61+round)) + client.SetReadDeadline(time.Now().Add(15 * time.Second)) + sender.SetWriteDeadline(time.Now().Add(15 * time.Second)) + err := kernelTCPWrite(sender, payload, writer == "buffer") + if err != nil { + test.Fatal("write:", err) + } + received := make([]byte, length) + n, readErr := io.ReadFull(client, received) + if readErr != nil || !bytes.Equal(received, payload) { + flow := traffic.snapshot(conn.key) + test.Fatalf("round %d: delivered %d/%d bytes: %v; invariant=%q; recent TCP events: %+v", round, n, length, readErr, flow.failure, flow.events[max(0, len(flow.events)-16):]) + } + traffic.await(test, conn, "transfer acknowledgement", func(flow kernelTCPFlow) bool { return flow.processed >= end }) + traffic.setFilter(conn, nil) + start = end + } + if pattern == "fin" { + dropped := false + traffic.setFilter(conn, func(event kernelTCPEvent) kernelTCPAction { + if event.outgoing && event.flags&header.TCPFlagFin != 0 && !dropped { + dropped = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + }) + } + client.SetReadDeadline(time.Now().Add(kernelTCPTimeout)) + err := N.CloseWrite(sender) + if err != nil { + test.Fatal(err) + } + var extra [1]byte + _, err = client.Read(extra[:]) + if err != io.EOF { + test.Fatalf("stream end: %v", err) + } + flow := traffic.await(test, conn, "FIN acknowledgement", func(flow kernelTCPFlow) bool { return flow.fin != 0 && flow.acked >= flow.fin }) + if flow.faults == 0 { + test.Fatal("transfer completed without exercising the network fault") + } + }) + } + } + } + }) + } + } +} + +func TestGoKernelTCPTransmitLifetime(t *testing.T) { + devices := make(chan struct{}, 8) + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, shutdown := range []string{"peer_reset", "stack_close"} { + for _, shape := range []string{"first", "short", "partial", "multiple"} { + t.Run(fmt.Sprintf("%s/ipv6=%v/%s/%s/%s", congestion, ipv6, writer, shutdown, shape), func(test *testing.T) { + test.Parallel() + devices <- struct{}{} + test.Cleanup(func() { <-devices }) + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{}) + client, sender, conn := traffic.pair(test, fixture, ipv6, writer) + barrier := newKernelTCPBarrier(test) + mss := int(conn.effectiveMSS) + prefix := mss + length := 1 + switch shape { + case "first": + prefix = 0 + case "partial": + length = mss - 1 + case "multiple": + length = 2*mss + 1 + } + paused := false + traffic.setFilter(conn, func(event kernelTCPEvent) kernelTCPAction { + if !event.outgoing && event.flags == header.TCPFlagAck { + return kernelTCPAction{drop: true} + } + if event.outgoing && event.end > event.sequence && event.sequence >= uint64(1+prefix) && !paused { + paused = true + return kernelTCPAction{pause: barrier} + } + return kernelTCPAction{} + }) + if prefix > 0 { + err := kernelTCPWrite(sender, kernelPayload(prefix, 71), writer == "buffer") + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(client, make([]byte, prefix)) + if err != nil { + test.Fatal(err) + } + } + written := make(chan error, 1) + go func() { written <- kernelTCPWrite(sender, kernelPayload(length, 73), writer == "buffer") }() + kernelTCPResult(test, barrier.entered, "device write") + closed := make(chan error, 1) + go func() { + if shutdown == "stack_close" { + closed <- fixture.stack.Close() + return + } + closeErr := client.SetLinger(0) + if closeErr == nil { + closeErr = client.Close() + } + closed <- closeErr + }() + if shutdown == "stack_close" { + traffic.await(test, conn, "stack shutdown started", func(kernelTCPFlow) bool { return fixture.stack.closed.Load() }) + } else { + err := kernelTCPResult(test, closed, "peer socket reset") + if err != nil { + test.Fatal(err) + } + } + flow := traffic.snapshot(conn.key) + if flow.writes == 0 || flow.released || flow.failure != "" { + test.Fatalf("shutdown did not overlap an active device write: writes=%d released=%v failure=%q", flow.writes, flow.released, flow.failure) + } + barrier.once.Do(func() { close(barrier.resume) }) + kernelTCPResult(test, written, "write cancellation") + if shutdown == "stack_close" { + err := kernelTCPResult(test, closed, "stack shutdown") + if err != nil { + test.Fatal(err) + } + } + traffic.await(test, conn, "connection reclamation", func(flow kernelTCPFlow) bool { return flow.released }) + }) + } + } + } + } + } +} diff --git a/stack_go_integration_cpu_unix_test.go b/stack_go_integration_cpu_unix_test.go new file mode 100644 index 00000000..70b3cbf6 --- /dev/null +++ b/stack_go_integration_cpu_unix_test.go @@ -0,0 +1,20 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func kernelProcessCPU(t *testing.T) time.Duration { + t.Helper() + var usage unix.Rusage + err := unix.Getrusage(unix.RUSAGE_SELF, &usage) + if err != nil { + t.Fatal(err) + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} diff --git a/stack_go_integration_cpu_windows_test.go b/stack_go_integration_cpu_windows_test.go new file mode 100644 index 00000000..ab3af2e2 --- /dev/null +++ b/stack_go_integration_cpu_windows_test.go @@ -0,0 +1,20 @@ +package tun + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +func kernelProcessCPU(t *testing.T) time.Duration { + t.Helper() + var created, exited, kernel, user windows.Filetime + err := windows.GetProcessTimes(windows.CurrentProcess(), &created, &exited, &kernel, &user) + if err != nil { + t.Fatal(err) + } + kernelTicks := uint64(kernel.HighDateTime)<<32 | uint64(kernel.LowDateTime) + userTicks := uint64(user.HighDateTime)<<32 | uint64(user.LowDateTime) + return time.Duration(kernelTicks+userTicks) * 100 +} diff --git a/stack_go_integration_darwin_test.go b/stack_go_integration_darwin_test.go index daf4b4ec..0db89016 100644 --- a/stack_go_integration_darwin_test.go +++ b/stack_go_integration_darwin_test.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net" - "os/exec" "runtime" "sync/atomic" "testing" @@ -23,45 +22,11 @@ const ( kernelConnectionReset = unix.ECONNRESET ) -func TestGoKernelMemoryPressure(t *testing.T) { - previous := runtime.GOMAXPROCS(4) - defer runtime.GOMAXPROCS(previous) - configs := []kernelStackConfig{{mtu: 1500}, {mtu: 1500, prepare: prepareKernelNetif}, {mtu: 9000, prepare: prepareKernelNetif}} - for _, config := range configs { - t.Run(fmt.Sprintf("mtu=%d/netif=%v", config.mtu, config.prepare != nil), func(configTest *testing.T) { - var pressure atomic.Uint32 - config.pressure = func() MemoryPressure { return MemoryPressure(pressure.Load()) } - fixture := newKernelStackFixture(configTest, config) - configTest.Logf("utun netif mode: %v", fixture.stack.engines[0].platformIO.(*goDarwinIO).netif) - for _, level := range []MemoryPressure{MemoryPressureWarning, MemoryPressureCritical, MemoryPressureNone} { - configTest.Run(fmt.Sprintf("level=%d", level), func(test *testing.T) { - pressure.Store(uint32(level)) - for _, splice := range []bool{false, true} { - test.Run(fmt.Sprintf("splice=%v", splice), func(flowTest *testing.T) { - var client, server net.Conn - if splice { - client, server, _, _ = fixture.splicePair(flowTest, true) - } else { - client, server = fixture.pair(flowTest, true) - } - payload := kernelPayload(8<<20, 79) - response := kernelPayload(8<<20, 83) - result := make(chan error, 1) - go func() { result <- kernelTransfer(client, server, payload, false) }() - err := kernelTransfer(server, client, response, !splice) - if err != nil { - flowTest.Error("download:", err) - } - err = <-result - if err != nil { - flowTest.Error("upload:", err) - } - }) - } - }) - } - }) - } +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, size), + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_SNDBUF, size), + ) } func prepareKernelNetif(t *testing.T, options *Options) { @@ -110,12 +75,12 @@ func configureKernelInterface(t *testing.T, device Tun, options Options) { ipv6 := options.Inet6Address[0] commands := [][]string{ {"/sbin/ifconfig", options.Name, "mtu", fmt.Sprint(options.MTU)}, - {"/sbin/ifconfig", options.Name, "inet", ipv4.Addr().String(), ipv4.Addr().String(), "netmask", "255.255.255.0", "up"}, + {"/sbin/ifconfig", options.Name, "inet", ipv4.Addr().String(), ipv4.Addr().String(), "netmask", net.IP(net.CIDRMask(ipv4.Bits(), 32)).String(), "up"}, {"/sbin/ifconfig", options.Name, "inet6", ipv6.String(), "-dad"}, {"/sbin/route", "-n", "add", "-net", ipv4.Masked().String(), "-interface", options.Name}, } for _, command := range commands { - output, err := exec.Command(command[0], command[1:]...).CombinedOutput() + output, err := kernelCommand(command[0], command[1:]...) if err != nil { t.Fatalf("%v: %s: %v", command, output, err) } @@ -186,10 +151,15 @@ func TestGoKernelDeviceBackpressure(t *testing.T) { test.Fatal("real netif transmit gate never became blocked") } if shutdown { - started := time.Now() - err := fixture.stack.Close() - if err != nil || time.Since(started) > time.Second { - test.Fatalf("close during device backpressure: %v", err) + closed := make(chan error, 1) + go func() { closed <- fixture.stack.Close() }() + select { + case err := <-closed: + if err != nil { + test.Fatal("close during device backpressure:", err) + } + case <-time.After(time.Second): + test.Fatal("close during device backpressure exceeded 1 second") } } interrupted := 0 @@ -239,85 +209,3 @@ func TestGoKernelDeviceBackpressure(t *testing.T) { } } } - -func TestGoKernelMemoryPressureTransition(t *testing.T) { - previous := runtime.GOMAXPROCS(4) - t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) - for _, netif := range []bool{false, true} { - for _, ipv6 := range []bool{false, true} { - for _, splice := range []bool{false, true} { - t.Run(fmt.Sprintf("netif=%v/ipv6=%v/splice=%v", netif, ipv6, splice), func(test *testing.T) { - test.Parallel() - var pressure, observed atomic.Uint32 - config := kernelStackConfig{mtu: 1500, socketBuffer: 128 << 10, upstreamSocketBuffer: 128 << 10} - config.pressure = func() MemoryPressure { - level := pressure.Load() - observed.Or(1 << level) - return MemoryPressure(level) - } - if netif { - config.prepare = prepareKernelNetif - } - fixture := newKernelStackFixture(test, config) - var client, server net.Conn - if splice { - client, server, _, _ = fixture.splicePair(test, ipv6) - } else { - client, server = fixture.pair(test, ipv6) - } - const phaseSize = 4 << 20 - upload := kernelPayload(3*phaseSize, 227) - download := kernelPayload(3*phaseSize, 229) - var uploadAccepted, downloadAccepted atomic.Int64 - written := make(chan error, 2) - go func() { written <- kernelFlowWrite(client, upload, false, &uploadAccepted) }() - go func() { written <- kernelFlowWrite(server, download, !splice, &downloadAccepted) }() - time.Sleep(100 * time.Millisecond) - if uploadAccepted.Load() == int64(len(upload)) || downloadAccepted.Load() == int64(len(download)) { - test.Fatal("pressure transition requires both writers to be active") - } - for phase, level := range []MemoryPressure{MemoryPressureWarning, MemoryPressureCritical, MemoryPressureNone} { - observed.And(^(1 << uint32(level))) - pressure.Store(uint32(level)) - read := make(chan error, 2) - for direction, conn := range []net.Conn{server, client} { - expected := upload - if direction == 1 { - expected = download - } - go func() { - data := make([]byte, phaseSize) - _, err := io.ReadFull(conn, data) - if err == nil && !bytes.Equal(data, expected[phase*phaseSize:(phase+1)*phaseSize]) { - err = E.New("pressure transition payload mismatch") - } - read <- err - }() - } - for range 2 { - err := <-read - if err != nil { - test.Fatal("active stream pressure transition:", err) - } - } - if observed.Load()&(1<= 1< 0 } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: windowUpdate(event)} + }) data := make([]byte, len(payload)) _, err := io.ReadFull(server, data[:available]) if err != nil { test.Fatal(err) } - time.Sleep(100 * time.Millisecond) - output, err := exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "ingress").CombinedOutput() - if err != nil { - test.Fatalf("remove ingress loss: %s: %v", output, err) - } + traffic.await(test, server, "drop Go receive-window update", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && windowUpdate(event) }) + }) + traffic.setFilter(server, nil) server.SetReadDeadline(time.Now().Add(4 * time.Second)) _, err = io.ReadFull(server, data[available:]) if err != nil { @@ -131,7 +142,7 @@ func TestGoKernelReceiveWindowUpdateLoss(t *testing.T) { if err != io.EOF { test.Fatalf("stream end: %v", err) } - err = <-written + err = kernelTCPResult(test, written, "upload after receive-window update loss") if err != nil { test.Fatal(err) } @@ -140,67 +151,36 @@ func TestGoKernelReceiveWindowUpdateLoss(t *testing.T) { } func TestGoKernelSACKReneging(t *testing.T) { - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) for _, ipv6 := range []bool{false, true} { t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500}, kernelTCPConfig{}) client, server := fixture.pair(test, ipv6) - runTC := func(args ...string) { - test.Helper() - output, err := exec.Command("tc", args...).CombinedOutput() - if err != nil { - test.Fatalf("tc %v: %s: %v", args, output, err) - } - } - protocol := "ip" - offset := "24" - if ipv6 { - protocol = "ipv6" - offset = "44" - } - runTC("qdisc", "add", "dev", fixture.options.Name, "ingress") - test.Cleanup(func() { exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "ingress").Run() }) - runTC("filter", "add", "dev", fixture.options.Name, "parent", "ffff:", "protocol", protocol, - "u32", "match", "u32", fmt.Sprintf("0x%08x", server.sendISN+1), "0xffffffff", "at", offset, "action", "drop") - payload := kernelPayload(8*int(server.effectiveMSS), 137) + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: event.outgoing && event.sequence == 1 && event.end > event.sequence} + }) + mss := int(server.effectiveMSS) + payload := kernelPayload(8*mss, 137) _, err := server.Write(payload) if err != nil { test.Fatal(err) } - rawConn, err := client.SyscallConn() - if err != nil { - test.Fatal(err) - } - var info *unix.TCPInfo - var infoErr error - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - err = rawConn.Control(func(descriptor uintptr) { - info, infoErr = unix.GetsockoptTCPInfo(int(descriptor), unix.IPPROTO_TCP, unix.TCP_INFO) + retained := goSackBlock{start: uint64(mss + 1), end: uint64(len(payload) + 1)} + traffic.await(test, server, "kernel SACK of queued out-of-order data", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { + return !event.outgoing && slices.Contains(event.sacks, retained) }) - if err != nil || infoErr != nil { - test.Fatal(E.Errors(err, infoErr)) - } - if info.Rcv_ooopack >= 7 { - break - } - time.Sleep(time.Millisecond) - } - if info == nil || info.Rcv_ooopack < 7 || server.sendUnacked.Load() != 1 { - test.Fatalf("missing out-of-order kernel queue: info=%+v unacked=%d", info, server.sendUnacked.Load()) - } - pruned := kernelTCPStatistic(test, "OfoPruned") + }) err = client.SetReadBuffer(512) if err != nil { test.Fatal(err) } - runTC("qdisc", "del", "dev", fixture.options.Name, "ingress") - deadline = time.Now().Add(2 * time.Second) - for kernelTCPStatistic(test, "OfoPruned") == pruned && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if kernelTCPStatistic(test, "OfoPruned") == pruned { - test.Fatal("kernel did not discard SACKed data") - } + traffic.setFilter(server, nil) + traffic.await(test, server, "kernel reneging on its SACK interval", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { + return !event.outgoing && event.ack >= retained.start && event.ack < retained.end + }) + }) err = client.SetReadBuffer(64 << 10) if err != nil { test.Fatal(err) @@ -209,7 +189,7 @@ func TestGoKernelSACKReneging(t *testing.T) { data := make([]byte, len(payload)) n, err := io.ReadFull(client, data) if err != nil || !bytes.Equal(data, payload) { - test.Fatalf("SACK reneging: received=%d/%d pruned=%d: %v", n, len(payload), kernelTCPStatistic(test, "OfoPruned")-pruned, err) + test.Fatalf("SACK reneging: received=%d/%d: %v", n, len(payload), err) } err = server.CloseWrite() if err != nil { @@ -238,7 +218,7 @@ func TestGoKernelZeroWindowHalfClose(t *testing.T) { }) return E.Errors(err, optionErr) }} - fixture := newKernelStackFixture(test, config) + fixture, traffic := newKernelTCPFixture(test, config, kernelTCPConfig{}) client, server := fixture.pair(test, ipv6) payload := kernelPayload(int(server.sendPermit.Load()-1), 139) _, err := server.Write(payload) @@ -255,15 +235,19 @@ func TestGoKernelZeroWindowHalfClose(t *testing.T) { if server.sendUnacked.Load() != uint64(len(payload)+1) || server.sendPermit.Load() != server.sendUnacked.Load() { test.Fatalf("window did not close after acknowledging all data: length=%d unacked=%d permit=%d", len(payload), server.sendUnacked.Load(), server.sendPermit.Load()) } - output, err := exec.Command("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "loss", "100%").CombinedOutput() - if err != nil { - test.Fatalf("drop FIN acknowledgment: %s: %v", output, err) + finAcknowledgement := func(event kernelTCPEvent) bool { + return !event.outgoing && event.flags&header.TCPFlagAck != 0 && event.ack == uint64(len(payload)+2) } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: finAcknowledgement(event)} + }) err = server.CloseWrite() if err != nil { test.Fatal(err) } - time.Sleep(1600 * time.Millisecond) + traffic.await(test, server, "FIN sent with zero receive window", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.peerWindow == 0 + }) err = client.SetReadBuffer(64 << 10) if err != nil { test.Fatal(err) @@ -273,16 +257,18 @@ func TestGoKernelZeroWindowHalfClose(t *testing.T) { if err != nil || !bytes.Equal(data, payload) { test.Fatalf("half-close data: %v", err) } - time.Sleep(100 * time.Millisecond) - output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "root").CombinedOutput() - if err != nil { - test.Fatalf("restore acknowledgments: %s: %v", output, err) - } + traffic.await(test, server, "drop FIN acknowledgement at a closed receive window", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && finAcknowledgement(event) }) + }) + traffic.setFilter(server, nil) client.SetReadDeadline(time.Now().Add(4 * time.Second)) _, err = client.Read(data[:1]) if err != io.EOF { test.Fatalf("zero-window FIN: %v", err) } + traffic.await(test, server, "recover lost FIN acknowledgement", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.acked >= flow.fin + }) err = client.CloseWrite() if err != nil { test.Fatal(err) @@ -295,33 +281,6 @@ func TestGoKernelZeroWindowHalfClose(t *testing.T) { } } -func kernelTCPStatistic(t *testing.T, name string) uint64 { - t.Helper() - data, err := os.ReadFile("/proc/net/netstat") - if err != nil { - t.Fatal(err) - } - lines := strings.Split(string(data), "\n") - for index := 0; index+1 < len(lines); index += 2 { - fields := strings.Fields(lines[index]) - if len(fields) == 0 || fields[0] != "TcpExt:" { - continue - } - values := strings.Fields(lines[index+1]) - for fieldIndex, field := range fields { - if field == name { - value, parseErr := strconv.ParseUint(values[fieldIndex], 10, 64) - if parseErr != nil { - t.Fatal(parseErr) - } - return value - } - } - } - t.Fatal("missing TCP statistic:", name) - return 0 -} - func TestGoKernelWindowLimitedRecovery(t *testing.T) { for _, ipv6 := range []bool{false, true} { t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { @@ -332,7 +291,7 @@ func TestGoKernelWindowLimitedRecovery(t *testing.T) { if err != nil { test.Fatal(err) } - output, err := exec.Command("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "delay", "50ms").CombinedOutput() + output, err := kernelCommand("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") if err != nil { test.Fatalf("delay ACKs: %s: %v", output, err) } @@ -376,7 +335,7 @@ func TestGoKernelWindowLimitedRecovery(t *testing.T) { case <-time.After(time.Second): test.Fatal("write did not react to deadline") } - output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, "root").CombinedOutput() + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, "root") if err != nil { test.Fatalf("restore ACKs: %s: %v", output, err) } @@ -421,7 +380,7 @@ func TestGoKernelWindowLimitedRecovery(t *testing.T) { } func TestGoKernelZeroWindow(t *testing.T) { - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000}) + fixture, traffic := newKernelTCPFixture(t, kernelStackConfig{mtu: 9000}, kernelTCPConfig{}) for _, ipv6 := range []bool{false, true} { t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { client, server := fixture.pair(test, ipv6) @@ -431,7 +390,7 @@ func TestGoKernelZeroWindow(t *testing.T) { } runTC := func(args ...string) { test.Helper() - output, commandErr := exec.Command("tc", args...).CombinedOutput() + output, commandErr := kernelCommand("tc", args...) if commandErr != nil { test.Fatalf("tc %v: %s: %v", args, output, commandErr) } @@ -453,7 +412,13 @@ func TestGoKernelZeroWindow(t *testing.T) { if server.sendPermit.Load() != unacked || server.sentTail.Load() != uint64(len(payload)+1) || unacked <= 1 || unacked >= server.sentTail.Load() { test.Fatalf("window did not close with only outstanding data: unacked=%d sent=%d buffered=%d permit=%d", unacked, server.sentTail.Load(), server.bufferedTail.Load(), server.sendPermit.Load()) } - runTC("qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "loss", "100%") + windowUpdate := func(event kernelTCPEvent) bool { + return !event.outgoing && event.flags == header.TCPFlagAck && event.window > 0 + } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: windowUpdate(event)} + }) + runTC("qdisc", "del", "dev", fixture.options.Name, "root") err = client.SetReadBuffer(4 << 20) if err != nil { test.Fatal(err) @@ -463,8 +428,10 @@ func TestGoKernelZeroWindow(t *testing.T) { if err != nil { test.Fatal(err) } - time.Sleep(100 * time.Millisecond) - runTC("qdisc", "del", "dev", fixture.options.Name, "root") + traffic.await(test, server, "drop kernel receive-window update", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && windowUpdate(event) }) + }) + traffic.setFilter(server, nil) client.SetReadDeadline(time.Now().Add(4 * time.Second)) _, err = io.ReadFull(client, data[unacked-1:]) if err != nil { diff --git a/stack_go_integration_lifecycle_linux_test.go b/stack_go_integration_lifecycle_linux_test.go index 682bd0e2..e815f63f 100644 --- a/stack_go_integration_lifecycle_linux_test.go +++ b/stack_go_integration_lifecycle_linux_test.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "net" - "os/exec" "regexp" "runtime" "testing" @@ -46,13 +45,13 @@ func TestGoKernelHandshakeLoss(t *testing.T) { if direction == "root" { command = append(command, "handle", "1:", "prio") } - output, err := exec.Command("tc", command...).CombinedOutput() + output, err := kernelCommand("tc", command...) if err != nil { test.Fatalf("handshake qdisc: %s: %v", output, err) } - test.Cleanup(func() { exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, direction).Run() }) - output, err = exec.Command("tc", "filter", "add", "dev", fixture.options.Name, "parent", parent, "protocol", protocol, - "u32", "match", "u8", flags, "0x12", "at", offset, "action", "drop").CombinedOutput() + test.Cleanup(func() { kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, direction) }) + output, err = kernelCommand("tc", "filter", "add", "dev", fixture.options.Name, "parent", parent, "protocol", protocol, + "u32", "match", "u8", flags, "0x12", "at", offset, "action", "drop") if err != nil { test.Fatalf("handshake filter: %s: %v", output, err) } @@ -81,14 +80,14 @@ func TestGoKernelHandshakeLoss(t *testing.T) { test.Fatalf("handshake completed while packets were dropped: %v", result.err) case <-time.After(250 * time.Millisecond): } - output, err = exec.Command("tc", "-s", "filter", "show", "dev", fixture.options.Name, "parent", parent).CombinedOutput() + output, err = kernelCommand("tc", "-s", "filter", "show", "dev", fixture.options.Name, "parent", parent) if err != nil || !regexp.MustCompile(`dropped [1-9][0-9]*`).Match(output) { test.Fatalf("no handshake packet was dropped: %s: %v", output, err) } if cancelHandshake { cancel() } else { - output, err = exec.Command("tc", "qdisc", "del", "dev", fixture.options.Name, direction).CombinedOutput() + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, direction) if err != nil { test.Fatalf("restore handshake traffic: %s: %v", output, err) } @@ -144,7 +143,7 @@ func TestGoKernelLinkInterruption(t *testing.T) { } runTC := func(args ...string) []byte { test.Helper() - output, err := exec.Command("tc", args...).CombinedOutput() + output, err := kernelCommand("tc", args...) if err != nil { test.Fatalf("tc %v: %s: %v", args, output, err) } @@ -228,11 +227,11 @@ func TestGoKernelLinkInterruption(t *testing.T) { runTC("qdisc", "del", "dev", fixture.options.Name, "root") runTC("qdisc", "del", "dev", fixture.options.Name, "ingress") for range 2 { - err := <-written + err := kernelTCPResult(test, written, "write through link outage") if err != nil { test.Fatal("write through link outage:", err) } - err = <-read + err = kernelTCPResult(test, read, "read after link recovery") if err != nil { test.Fatal("read after link recovery:", err) } @@ -243,45 +242,66 @@ func TestGoKernelLinkInterruption(t *testing.T) { } } -func configureKernelLoss(t *testing.T, options Options) { +func configureKernelNetem(t *testing.T, options Options, impairment ...string) { t.Helper() ifbName := "ifb-" + options.Name + impairment = append(impairment, "limit", "4096") commands := [][]string{ {"ip", "link", "add", ifbName, "type", "ifb"}, {"ip", "link", "set", ifbName, "up"}, - {"tc", "qdisc", "add", "dev", options.Name, "root", "netem", "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%", "limit", "4096"}, + append([]string{"tc", "qdisc", "add", "dev", options.Name, "root", "netem"}, impairment...), {"tc", "qdisc", "add", "dev", options.Name, "ingress"}, {"tc", "filter", "add", "dev", options.Name, "parent", "ffff:", "protocol", "all", "u32", "match", "u32", "0", "0", "action", "mirred", "egress", "redirect", "dev", ifbName}, - {"tc", "qdisc", "add", "dev", ifbName, "root", "netem", "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%", "limit", "4096"}, + append([]string{"tc", "qdisc", "add", "dev", ifbName, "root", "netem"}, impairment...), } for index, command := range commands { - output, err := exec.Command(command[0], command[1:]...).CombinedOutput() + output, err := kernelCommand(command[0], command[1:]...) if err != nil { t.Fatalf("%v: %s: %v", command, output, err) } if index == 0 { - t.Cleanup(func() { exec.Command("ip", "link", "delete", ifbName).Run() }) + t.Cleanup(func() { kernelCommand("ip", "link", "delete", ifbName) }) } } } +func kernelBidirectionalDataLoss() func(kernelTCPEvent) kernelTCPAction { + var dropped [2]bool + return func(event kernelTCPEvent) kernelTCPAction { + direction := 0 + if event.outgoing { + direction = 1 + } + if event.length > 0 && !dropped[direction] { + dropped[direction] = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + } +} + func TestGoKernelBidirectionalLoss(t *testing.T) { previous := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(previous) for _, gso := range []bool{false, true} { for _, multiQueue := range []bool{false, true} { t.Run(fmt.Sprintf("gso=%v/mq=%v", gso, multiQueue), func(offloadTest *testing.T) { - fixture := newKernelStackFixture(offloadTest, kernelStackConfig{mtu: 1500, gso: gso, multiQueue: multiQueue, configure: configureKernelLoss}) + fixture, traffic := newKernelTCPFixture(offloadTest, kernelStackConfig{ + mtu: 1500, gso: gso, multiQueue: multiQueue, + configure: func(test *testing.T, options Options) { + configureKernelNetem(test, options, "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%") + }, + }, kernelTCPConfig{}) for _, ipv6 := range []bool{false, true} { for _, splice := range []bool{false, true} { offloadTest.Run(fmt.Sprintf("ipv6=%v/splice=%v", ipv6, splice), func(flowTest *testing.T) { flowTest.Parallel() - var client, server net.Conn + writer := "write" if splice { - client, server, _, _ = fixture.splicePair(flowTest, ipv6) - } else { - client, server = fixture.pair(flowTest, ipv6) + writer = "splice" } + client, server, conn := traffic.pair(flowTest, fixture, ipv6, writer) + traffic.setFilter(conn, kernelBidirectionalDataLoss()) payload := kernelPayload(1<<20, 43) response := kernelPayload(1<<20, 47) result := make(chan error, 1) @@ -290,10 +310,14 @@ func TestGoKernelBidirectionalLoss(t *testing.T) { if err != nil { flowTest.Error("download:", err) } - err = <-result + err = kernelTCPResult(flowTest, result, "upload through bidirectional loss") if err != nil { flowTest.Error("upload:", err) } + observed := traffic.snapshot(conn.key) + if observed.faults != 2 { + flowTest.Fatalf("bidirectional transfer observed %d targeted data drops, expected 2", observed.faults) + } }) } } diff --git a/stack_go_integration_lifecycle_test.go b/stack_go_integration_lifecycle_test.go index 7ef2708b..28f3fd76 100644 --- a/stack_go_integration_lifecycle_test.go +++ b/stack_go_integration_lifecycle_test.go @@ -4,13 +4,16 @@ package tun import ( "bytes" + "context" "errors" "fmt" "io" "net" "os" + "os/exec" "runtime" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -102,25 +105,6 @@ func TestGoKernelHandshake(t *testing.T) { func TestGoKernelHalfClose(t *testing.T) { fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) for _, ipv6 := range []bool{false, true} { - for _, splice := range []bool{false, true} { - t.Run(fmt.Sprintf("ipv6=%v/splice=%v", ipv6, splice), func(test *testing.T) { - test.Parallel() - var client, server net.Conn - if splice { - client, server, _, _ = fixture.splicePair(test, ipv6) - } else { - client, server = fixture.pair(test, ipv6) - } - err := kernelTransfer(client, server, kernelPayload(256<<10, 163), false) - if err != nil { - test.Fatal("request and FIN:", err) - } - err = kernelTransfer(server, client, kernelPayload(2<<20, 167), false) - if err != nil { - test.Fatal("response after peer FIN:", err) - } - }) - } t.Run(fmt.Sprintf("ipv6=%v/close_read", ipv6), func(test *testing.T) { test.Parallel() client, server := fixture.pair(test, ipv6) @@ -229,9 +213,26 @@ func TestGoKernelClose(t *testing.T) { }) for _, splice := range []bool{false, true} { t.Run(fmt.Sprintf("busy_shutdown/splice=%v", splice), func(scenarioTest *testing.T) { - fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500, multiQueue: runtime.GOOS == "linux"}) - payload := kernelPayload(8<<20, 33) - completed := make(chan error, 32) + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{ + mtu: 1500, multiQueue: runtime.GOOS == "linux", + upstreamSocketBuffer: 4096, + dialer: net.Dialer{Control: func(_, _ string, rawConn syscall.RawConn) error { + var optionErr error + err := rawConn.Control(func(descriptor uintptr) { + optionErr = kernelSetSocketBuffers(descriptor, 4096) + }) + return E.Errors(err, optionErr) + }}, + }) + payload := kernelPayload(32749, 33) + type pendingResult struct { + connection int + operation string + kernelIOResult + } + completed := make(chan pendingResult, 32) + reading := make(chan kernelIOResult, 16) + var progress [16]atomic.Int64 for index := range 16 { var client, conn net.Conn if splice { @@ -239,46 +240,94 @@ func TestGoKernelClose(t *testing.T) { } else { client, conn = fixture.pair(scenarioTest, index%2 != 0) } - client.SetDeadline(time.Time{}) - client.(*net.TCPConn).SetReadBuffer(4096) + client.SetDeadline(time.Now().Add(2 * time.Second)) conn.SetDeadline(time.Time{}) - go func() { conn.Write(payload); completed <- nil }() go func() { - _, err := conn.Read(make([]byte, 1)) - if err == nil { - completed <- E.New("pending read completed without close error") - return + for { + n, err := conn.Write(payload) + progress[index].Add(int64(n)) + if err != nil { + completed <- pendingResult{connection: index, operation: "write", kernelIOResult: kernelIOResult{n: n, err: err}} + return + } } - completed <- nil }() + go func() { + var probe [1]byte + n, err := io.ReadFull(conn, probe[:]) + if err == nil && probe[0] != 0x71 { + err = E.New("shutdown read probe corrupted") + } + reading <- kernelIOResult{n: n, err: err} + n, err = conn.Read(probe[:]) + completed <- pendingResult{connection: index, operation: "read", kernelIOResult: kernelIOResult{n: n, err: err}} + }() + _, err := client.Write([]byte{0x71}) + if err != nil { + scenarioTest.Fatal("start blocked reader:", err) + } + var probe [1]byte + _, err = io.ReadFull(client, probe[:]) + if err != nil || probe[0] != payload[0] { + scenarioTest.Fatalf("start blocked writer: byte=%x error=%v", probe[0], err) + } } - time.Sleep(30 * time.Millisecond) - started := time.Now() - err := fixture.stack.Close() - if err != nil { - scenarioTest.Fatal(err) + for range 16 { + select { + case result := <-reading: + if result.n != 1 || result.err != nil { + scenarioTest.Fatalf("start blocked reader: %+v", result) + } + case <-time.After(time.Second): + scenarioTest.Fatal("reader did not consume peer probe") + } + } + var previousProgress [16]int64 + stalledAt := time.Now() + backpressureDeadline := time.NewTimer(time.Second) + defer backpressureDeadline.Stop() + observations := time.NewTicker(10 * time.Millisecond) + defer observations.Stop() + for time.Since(stalledAt) < 40*time.Millisecond { + select { + case result := <-completed: + scenarioTest.Fatalf("I/O completed before stack shutdown: %+v", result) + case <-observations.C: + for index := range progress { + accepted := progress[index].Load() + if accepted != previousProgress[index] { + previousProgress[index] = accepted + stalledAt = time.Now() + } + } + case <-backpressureDeadline.C: + scenarioTest.Fatal("writers did not encounter backpressure within 1 second") + } } - if time.Since(started) > time.Second { - scenarioTest.Error("stack close exceeded 1 second") + closed := make(chan error, 1) + go func() { closed <- fixture.stack.Close() }() + select { + case err := <-closed: + if err != nil { + scenarioTest.Fatal(err) + } + case <-time.After(time.Second): + scenarioTest.Fatal("stack close did not complete within 1 second") } deadline := time.NewTimer(2 * time.Second) defer deadline.Stop() for range 32 { select { - case operationErr := <-completed: - if operationErr != nil { - scenarioTest.Error(operationErr) + case result := <-completed: + if !(E.IsClosed(result.err) || errors.Is(result.err, kernelConnectionReset)) || E.IsTimeout(result.err) || + result.operation == "read" && result.n != 0 || + result.operation == "write" && result.n >= len(payload) { + scenarioTest.Errorf("pending I/O after stack shutdown: %+v", result) } case <-deadline.C: scenarioTest.Fatal("pending I/O did not unblock after stack shutdown") } } - for _, engine := range fixture.stack.engines { - inUse := engine.slabPool.inUse.Load() - if inUse != 0 { - scenarioTest.Errorf("shutdown retained %d allocated slabs", inUse) - } - } }) } } @@ -288,6 +337,14 @@ type kernelIOResult struct { err error } +func kernelCommand(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + command := exec.CommandContext(ctx, name, args...) + command.WaitDelay = time.Second + return command.CombinedOutput() +} + func TestGoKernelDeadline(t *testing.T) { fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000}) for _, ipv6 := range []bool{false, true} { diff --git a/stack_go_integration_linux_test.go b/stack_go_integration_linux_test.go index b2626aaf..6f5022b8 100644 --- a/stack_go_integration_linux_test.go +++ b/stack_go_integration_linux_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" "golang.org/x/sys/unix" ) @@ -16,6 +17,13 @@ const ( kernelConnectionReset = unix.ECONNRESET ) +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, size), + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_SNDBUF, size), + ) +} + func configureKernelInterface(t *testing.T, device Tun, options Options) { t.Helper() link, err := netlink.LinkByName(options.Name) diff --git a/stack_go_integration_packet_flow_test.go b/stack_go_integration_packet_flow_test.go index 1cfe5154..cbae7aaa 100644 --- a/stack_go_integration_packet_flow_test.go +++ b/stack_go_integration_packet_flow_test.go @@ -16,6 +16,7 @@ import ( "github.com/sagernet/sing/common/buf" "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" ) @@ -34,37 +35,44 @@ func TestGoKernelPacketFragments(t *testing.T) { const sessions = 4 for phase := range 4 { start := make(chan struct{}) - var workers sync.WaitGroup + release := sync.OnceFunc(func() { close(start) }) + test.Cleanup(release) + completed := make(chan error, sessions) for index := range sessions { client, conn, destination := fixture.packetPair(test, ipv6) payload := kernelPayload(16385, uint32(phase*sessions+index+1)) - workers.Go(func() { + go func() { defer client.Close() defer conn.Close() - select { - case <-start: - case <-conn.doneChan: - return - } + <-start client.SetReadDeadline(time.Now().Add(time.Second)) options := N.NewReadWaitOptions(nil, conn) buffer := options.NewBufferSize(len(payload)) - _, _ = buffer.Write(payload) + copy(buffer.Extend(len(payload)), payload) options.PostReturn(buffer) err := conn.WritePacket(buffer, destination) if err != nil { - test.Error(err) + completed <- E.Cause(err, "write fragments for session ", index) return } storage := make([]byte, 65535) n, err := client.Read(storage) - if err != nil || !bytes.Equal(storage[:n], payload) { - test.Errorf("phase=%d session=%d: reassembled %d/%d bytes: %v", phase, index, n, len(payload), err) + if err == nil && !bytes.Equal(storage[:n], payload) { + err = E.New("reassembled ", n, "/", len(payload), " bytes with corrupted payload") + } + if err != nil { + err = E.Cause(err, "receive fragments for session ", index) } - }) + completed <- err + }() + } + release() + for range sessions { + err := kernelTCPResult(test, completed, "fragmented UDP delivery") + if err != nil { + test.Errorf("phase=%d: %v", phase, err) + } } - close(start) - workers.Wait() if test.Failed() { return } diff --git a/stack_go_integration_packet_offload_linux_test.go b/stack_go_integration_packet_offload_linux_test.go index c742db78..37c5aaf2 100644 --- a/stack_go_integration_packet_offload_linux_test.go +++ b/stack_go_integration_packet_offload_linux_test.go @@ -20,12 +20,12 @@ import ( ) func TestGoKernelPacketOffload(t *testing.T) { - allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator} + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator, epoch: time.Now()} buf.DefaultAllocator = allocator t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) for _, multiQueue := range []bool{false, true} { t.Run(fmt.Sprintf("mq=%v", multiQueue), func(queueTest *testing.T) { - fixture := newKernelStackFixture(queueTest, kernelStackConfig{mtu: 1500, gso: true, multiQueue: multiQueue}) + fixture := newKernelStackFixture(queueTest, kernelStackConfig{mtu: 1500, gso: true, multiQueue: multiQueue, prepareStack: allocator.prepareStack}) for _, ipv6 := range []bool{false, true} { for _, mode := range []string{"direct", "socks", "socks-fqdn"} { queueTest.Run(fmt.Sprintf("ipv6=%v/%s", ipv6, mode), func(scenarioTest *testing.T) { @@ -41,7 +41,6 @@ func TestGoKernelPacketOffload(t *testing.T) { scenarioTest.Fatal(err) } defer peer.Close() - peer.SetDeadline(time.Now().Add(3 * time.Second)) peer.SetReadBuffer(1 << 20) upstream, err := net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) if err != nil { @@ -100,6 +99,10 @@ func TestGoKernelPacketOffload(t *testing.T) { scenarioTest.Fatal("UDP splice refused") } for cycle := range 2 { + deadline := time.Now().Add(3 * time.Second) + client.SetDeadline(deadline) + conn.SetReadDeadline(deadline) + peer.SetDeadline(deadline) payloads := make([][]byte, 20) wirePackets := make([][]byte, len(payloads)) for index := range payloads { diff --git a/stack_go_integration_packet_test.go b/stack_go_integration_packet_test.go index 0cd29e27..af889661 100644 --- a/stack_go_integration_packet_test.go +++ b/stack_go_integration_packet_test.go @@ -20,7 +20,6 @@ import ( "github.com/sagernet/sing-tun/gtcpip/header" "github.com/sagernet/sing/common/buf" "github.com/sagernet/sing/common/bufio" - E "github.com/sagernet/sing/common/exceptions" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" "github.com/sagernet/sing/protocol/socks" @@ -267,7 +266,9 @@ func TestGoKernelPacket(t *testing.T) { var serverAddress M.Socksaddr serverAddress, socksPackets = newKernelSocksEchoServer(scenarioTest, address) socksClient := socks.NewClient(N.SystemDialer, serverAddress, socks.Version5, "", "") - associateConn, err := socksClient.ListenPacket(context.Background(), socksDestination) + handshakeContext, cancelHandshake := context.WithTimeout(context.Background(), 3*time.Second) + associateConn, err := socksClient.ListenPacket(handshakeContext, socksDestination) + cancelHandshake() if err != nil { scenarioTest.Fatal(err) } @@ -288,14 +289,6 @@ func TestGoKernelPacket(t *testing.T) { N.PutPacketBuffer(cached) scenarioTest.Fatal(err) } - if !config.multiQueue { - offload = &kernelSocksOffload{ - PacketOffload: offload, - engine: conn.writer.(*GoPacketConn).engine, - cached: 2, - test: scenarioTest, - } - } accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{ NAT: PacketNAT{Origin: destination, Destination: socksDestination}, Cached: []*N.PacketBuffer{cached}, @@ -412,41 +405,6 @@ func TestGoKernelPacket(t *testing.T) { } } -type kernelSocksOffload struct { - N.PacketOffload - engine *goEngine - cached int - test *testing.T -} - -func (o *kernelSocksOffload) EncodePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - if o.cached > 0 { - o.cached-- - } else { - borrowed := false - for _, frame := range o.engine.frames { - if buffer == frame.buffer { - borrowed = true - break - } - } - if !borrowed { - for _, entry := range o.engine.reassemblyEntries { - if buffer == entry.buffer { - borrowed = true - break - } - } - } - if !borrowed { - err := E.New("SOCKS splice copied the TUN payload before encoding") - o.test.Error(err) - return err - } - } - return o.PacketOffload.EncodePacket(buffer, destination) -} - type kernelSocksPacket struct { payload []byte destination M.Socksaddr @@ -463,6 +421,7 @@ func newKernelSocksEchoServer(t *testing.T, address net.IP) (M.Socksaddr, chan k t.Fatal(err) } t.Cleanup(func() { listener.Close() }) + ctx := t.Context() server := &kernelSocksEchoServer{packets: make(chan kernelSocksPacket, 16)} go func() { for { @@ -471,8 +430,10 @@ func newKernelSocksEchoServer(t *testing.T, address net.IP) (M.Socksaddr, chan k return } go func() { - handleErr := socks.HandleConnectionEx(context.Background(), conn, std_bufio.NewReader(conn), nil, server, server, 0, M.SocksaddrFromNet(conn.RemoteAddr()), nil) + stopContext := context.AfterFunc(ctx, func() { conn.Close() }) + handleErr := socks.HandleConnectionEx(ctx, conn, std_bufio.NewReader(conn), nil, server, server, 0, M.SocksaddrFromNet(conn.RemoteAddr()), nil) if handleErr != nil { + stopContext() conn.Close() } }() @@ -508,7 +469,12 @@ func (s *kernelSocksEchoServer) NewPacketConnectionEx(ctx context.Context, conn buffer.Release() return } - s.packets <- kernelSocksPacket{payload: bytes.Clone(buffer.Bytes()), destination: packetDestination} + select { + case s.packets <- kernelSocksPacket{payload: bytes.Clone(buffer.Bytes()), destination: packetDestination}: + case <-ctx.Done(): + buffer.Release() + return + } err = conn.WritePacket(buffer, packetDestination) if err != nil { return @@ -525,7 +491,42 @@ var ( type kernelPacketAllocator struct { buf.Allocator - active atomic.Int64 + active atomic.Int64 + epoch time.Time + lastActivity atomic.Int64 +} + +type kernelPacketIO struct { + goPlatformIO + allocator *kernelPacketAllocator +} + +func (p *kernelPacketIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + readable, count, err := p.goPlatformIO.wait(timeout, events) + if readable || count > 0 { + now := time.Since(p.allocator.epoch).Nanoseconds() + for { + previous := p.allocator.lastActivity.Load() + if now <= previous || p.allocator.lastActivity.CompareAndSwap(previous, now) { + break + } + } + } + return readable, count, err +} + +func (a *kernelPacketAllocator) prepareStack(stack *Go) { + queues := stack.queueFactory + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + platforms, err := queues(current) + if err != nil { + return nil, err + } + for index, platform := range platforms { + platforms[index] = &kernelPacketIO{goPlatformIO: platform, allocator: a} + } + return platforms, nil + } } func (a *kernelPacketAllocator) Get(size int) []byte { @@ -546,17 +547,27 @@ func (a *kernelPacketAllocator) Put(buffer []byte) error { func (a *kernelPacketAllocator) waitIdle(t *testing.T) { t.Helper() - deadline := time.Now().Add(time.Second) - for a.active.Load() != 0 && time.Now().Before(deadline) { + deadline := time.Now().Add(5 * time.Second) + for { + lastActivity := a.lastActivity.Load() + quiet := time.Since(a.epoch) - time.Duration(lastActivity) + active := a.active.Load() + unchanged := a.lastActivity.Load() == lastActivity + if quiet >= goReadBufferIdle && active == 0 && unchanged { + return + } + if quiet >= goReadBufferIdle+time.Second && active != 0 && unchanged { + t.Fatalf("connections retain %d pooled buffers after %s without native I/O", active, quiet) + } + if time.Now().After(deadline) { + t.Fatalf("idle wait deadline reached: last native I/O %s ago, %d pooled buffers retained", quiet, active) + } time.Sleep(time.Millisecond) } - if active := a.active.Load(); active != 0 { - t.Fatalf("idle connections retain %d pooled buffers", active) - } } func TestGoKernelPacketBatchIdle(t *testing.T) { - allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator} + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator, epoch: time.Now()} buf.DefaultAllocator = allocator t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) configs := []kernelStackConfig{{mtu: 1500}, {mtu: 65535}} @@ -564,6 +575,7 @@ func TestGoKernelPacketBatchIdle(t *testing.T) { configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) } for _, config := range configs { + config.prepareStack = allocator.prepareStack t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { fixture := newKernelStackFixture(configurationTest, config) for _, ipv6 := range []bool{false, true} { @@ -588,7 +600,6 @@ func TestGoKernelPacketBatchIdle(t *testing.T) { defer peer.Close() peer.SetReadBuffer(1 << 20) peer.SetWriteBuffer(1 << 20) - peer.SetDeadline(time.Now().Add(5 * time.Second)) var upstream *net.UDPConn if mode != "unconnected" { upstream, err = net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) @@ -619,6 +630,12 @@ func TestGoKernelPacketBatchIdle(t *testing.T) { } } for cycle := range 3 { + deadline := time.Now().Add(3 * time.Second) + client.SetDeadline(deadline) + conn.SetReadDeadline(deadline) + if peer != nil { + peer.SetDeadline(deadline) + } payloads := make([][]byte, 24) for index := range payloads { size := 1200 @@ -628,7 +645,7 @@ func TestGoKernelPacketBatchIdle(t *testing.T) { payloads[index] = kernelPayload(size, uint32(cycle*31+index+1)) _, err := client.Write(payloads[index]) if err != nil { - scenarioTest.Fatal(err) + scenarioTest.Fatalf("cycle %d upload packet %d: %v", cycle, index, err) } } uploaded := make([]bool, len(payloads)) diff --git a/stack_go_integration_tcp_test.go b/stack_go_integration_tcp_test.go new file mode 100644 index 00000000..cdc425af --- /dev/null +++ b/stack_go_integration_tcp_test.go @@ -0,0 +1,530 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "fmt" + "net" + "slices" + "sync" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + N "github.com/sagernet/sing/common/network" +) + +const kernelTCPTimeout = 3 * time.Second + +type kernelTCPEvent struct { + at time.Duration + rate uint64 + outgoing bool + sequence uint64 + end uint64 + ack uint64 + window uint16 + length int + sacks []goSackBlock + flags header.TCPFlags + dropped bool + delayed bool + delivery bool + copies int +} + +type kernelTCPAction struct { + drop bool + copies int + delay time.Duration + pause *kernelTCPBarrier +} + +type kernelTCPBarrier struct { + entered chan struct{} + resume chan struct{} + once sync.Once +} + +func newKernelTCPBarrier(t *testing.T) *kernelTCPBarrier { + barrier := &kernelTCPBarrier{entered: make(chan struct{}), resume: make(chan struct{})} + t.Cleanup(func() { barrier.once.Do(func() { close(barrier.resume) }) }) + return barrier +} + +type kernelTCPConfig struct { + receiveDelay time.Duration + disableSACK bool + disableTimestamps bool + checkpoint func(string, *GoConn) +} + +type kernelTCPFlow struct { + conn *GoConn + isn uint32 + sent uint64 + acked uint64 + fin uint64 + peerFIN bool + peerWindow uint16 + failure string + events []kernelTCPEvent + processed uint64 + closed bool + released bool + writes int + faults uint64 + filter func(kernelTCPEvent) kernelTCPAction +} + +type kernelTCPHarness struct { + access sync.Mutex + flows map[flowKey]*kernelTCPFlow + changed chan struct{} + done chan struct{} + epoch time.Time + config kernelTCPConfig + workers sync.WaitGroup + stopped bool +} + +func newKernelTCPFixture(t *testing.T, config kernelStackConfig, tcpConfig kernelTCPConfig) (*kernelStackFixture, *kernelTCPHarness) { + t.Helper() + harness := &kernelTCPHarness{ + flows: make(map[flowKey]*kernelTCPFlow), changed: make(chan struct{}, 1), done: make(chan struct{}), + epoch: time.Now(), config: tcpConfig, + } + prepare := config.prepareStack + config.prepareStack = func(stack *Go) { + if prepare != nil { + prepare(stack) + } + queues := stack.queueFactory + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + platforms, err := queues(current) + if err != nil { + return nil, err + } + for index, platform := range platforms { + platforms[index] = &kernelTCPIO{goPlatformIO: platform, harness: harness} + } + return platforms, nil + } + operations := *stack.congestion + initialize := operations.init + operations.init = func(conn *GoConn) { + if initialize != nil { + initialize(conn) + } + harness.access.Lock() + harness.flows[conn.key] = &kernelTCPFlow{conn: conn, isn: conn.sendISN} + harness.access.Unlock() + } + release := operations.release + operations.release = func(conn *GoConn) { + harness.access.Lock() + flow := harness.flows[conn.key] + flow.released = true + if flow.writes != 0 && flow.failure == "" { + flow.failure = fmt.Sprintf("connection resources released with %d device writes in progress", flow.writes) + } + harness.access.Unlock() + if release != nil { + release(conn) + } + } + checkpoint := tcpConfig.checkpoint + if checkpoint != nil { + acked := operations.packetsAcked + operations.packetsAcked = func(conn *GoConn, sample *goAckSample) { + if sample.packetsAcked > 0 { + checkpoint("ack", conn) + } + if acked != nil { + acked(conn, sample) + } + } + avoid := operations.congAvoid + if avoid != nil { + operations.congAvoid = func(conn *GoConn, packets uint32) { + checkpoint("control", conn) + avoid(conn, packets) + checkpoint("controlled", conn) + } + } + control := operations.congControl + if control != nil { + operations.congControl = func(conn *GoConn, flags goAckFlags, sample *goRateSample) { + checkpoint("control", conn) + control(conn, flags, sample) + checkpoint("controlled", conn) + } + } + } + stack.congestion = &operations + } + fixture := newKernelStackFixture(t, config) + t.Cleanup(func() { + harness.access.Lock() + harness.stopped = true + close(harness.done) + harness.access.Unlock() + harness.workers.Wait() + }) + return fixture, harness +} + +func (h *kernelTCPHarness) eventLocked(packet []byte, outgoing bool) (*kernelTCPFlow, kernelTCPEvent) { + parsed, valid := parseForwardPacket(packet) + if !valid || parsed.protocol != uint8(header.TCPProtocolNumber) { + return nil, kernelTCPEvent{} + } + key := parsed.flowKey() + if outgoing { + key = key.reversed() + } + flow := h.flows[key] + if flow == nil { + return nil, kernelTCPEvent{} + } + tcp := header.TCP(parsed.transport) + flags := tcp.Flags() + event := kernelTCPEvent{at: time.Since(h.epoch), outgoing: outgoing, flags: flags, window: tcp.WindowSize(), length: len(tcp.Payload())} + if outgoing { + event.rate = flow.conn.pacingRate.Load() + event.sequence = uint64(uint32(tcp.SequenceNumber() - flow.isn)) + event.end = event.sequence + uint64(len(tcp.Payload())) + if flags&(header.TCPFlagSyn|header.TCPFlagFin) != 0 { + event.end++ + } + } else { + event.ack = uint64(uint32(tcp.AckNumber() - flow.isn)) + for _, block := range header.ParseTCPOptions(tcp.Options()).SACKBlocks { + event.sacks = append(event.sacks, goSackBlock{ + start: uint64(uint32(block.Start) - flow.isn), + end: uint64(uint32(block.End) - flow.isn), + }) + } + } + return flow, event +} + +func (h *kernelTCPHarness) recordLocked(flow *kernelTCPFlow, event kernelTCPEvent) { + if event.outgoing { + if !event.delivery && event.end > event.sequence && event.end <= flow.processed && flow.peerWindow != 0 && flow.failure == "" { + flow.failure = fmt.Sprintf("retransmitted acknowledged sequence [%d,%d), processed ACK %d", event.sequence, event.end, flow.processed) + } + flow.sent = max(flow.sent, event.end) + if event.flags&header.TCPFlagFin != 0 { + flow.fin = event.end + } + } else if !event.dropped && !event.delayed { + flow.peerWindow = event.window + if event.flags&header.TCPFlagAck != 0 { + flow.acked = max(flow.acked, event.ack) + } + flow.peerFIN = flow.peerFIN || event.flags&header.TCPFlagFin != 0 + } + if !event.delivery && (event.dropped || event.delayed || event.copies > 1) { + flow.faults++ + } + if len(flow.events) == 128 { + copy(flow.events, flow.events[1:]) + flow.events = flow.events[:127] + } + flow.events = append(flow.events, event) + select { + case h.changed <- struct{}{}: + default: + } +} + +func (h *kernelTCPHarness) pair(t *testing.T, fixture *kernelStackFixture, ipv6 bool, writer string) (*net.TCPConn, net.Conn, *GoConn) { + t.Helper() + var client *net.TCPConn + var sender net.Conn + if writer == "splice" { + client, sender, _, _ = fixture.splicePair(t, ipv6) + } else { + client, sender = fixture.pair(t, ipv6) + } + key := flowKey{protocol: uint8(header.TCPProtocolNumber), source: client.LocalAddr().(*net.TCPAddr).AddrPort(), destination: client.RemoteAddr().(*net.TCPAddr).AddrPort()} + return client, sender, h.snapshot(key).conn +} + +func (h *kernelTCPHarness) setFilter(conn *GoConn, filter func(kernelTCPEvent) kernelTCPAction) { + h.access.Lock() + h.flows[conn.key].filter = filter + h.access.Unlock() +} + +func (h *kernelTCPHarness) inspectLocked(platform *kernelTCPIO) { + for _, flow := range h.flows { + conn := flow.conn + if conn.engine.platformIO != platform { + continue + } + flow.processed = conn.sendUnacked.Load() + flow.closed = conn.connState.Load() >= goConnStateDead + if flow.closed || flow.failure != "" { + continue + } + flight := conn.flight + if uint64(flight.sackedOut)+uint64(flight.lostOut) > uint64(flight.packetsOut) || + uint64(flight.sackedOut)+uint64(flight.retransmitOut) > uint64(flight.packetsOut) { + flow.failure = fmt.Sprintf("invalid flight accounting: %+v", flight) + } + if conn.sendReleased.Load() > min(flow.processed, conn.transmittedTail.Load()) { + flow.failure = "send buffers released before transmission and acknowledgement completed" + } + } +} + +func (h *kernelTCPHarness) snapshot(key flowKey) kernelTCPFlow { + h.access.Lock() + defer h.access.Unlock() + flow := *h.flows[key] + flow.events = slices.Clone(flow.events) + return flow +} + +func (h *kernelTCPHarness) await(t *testing.T, conn *GoConn, description string, ready func(kernelTCPFlow) bool) kernelTCPFlow { + t.Helper() + timer := time.NewTimer(kernelTCPTimeout) + defer timer.Stop() + tick := time.NewTicker(time.Millisecond) + defer tick.Stop() + for { + flow := h.snapshot(conn.key) + if flow.failure != "" { + t.Fatalf("%s: %s; recent TCP events: %+v", description, flow.failure, flow.events[max(0, len(flow.events)-16):]) + } + if ready(flow) { + return flow + } + select { + case <-h.changed: + case <-tick.C: + case <-timer.C: + t.Fatalf("%s timed out: sent=%d ACK=%d processed=%d FIN=%d peerFIN=%v connection=%d; recent TCP events: %+v", description, flow.sent, flow.acked, flow.processed, flow.fin, flow.peerFIN, conn.connState.Load(), flow.events[max(0, len(flow.events)-16):]) + } + } +} + +type kernelTCPIO struct { + goPlatformIO + harness *kernelTCPHarness +} + +func (p *kernelTCPIO) close() error { + p.harness.access.Lock() + p.harness.inspectLocked(p) + p.harness.access.Unlock() + return p.goPlatformIO.close() +} + +func (p *kernelTCPIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + p.harness.access.Lock() + p.harness.inspectLocked(p) + p.harness.access.Unlock() + if p.harness.config.checkpoint != nil { + p.harness.config.checkpoint("idle", nil) + } + readable, count, err := p.goPlatformIO.wait(timeout, events) + if readable && p.harness.config.receiveDelay > 0 { + timer := time.NewTimer(p.harness.config.receiveDelay) + select { + case <-timer.C: + case <-p.harness.done: + } + timer.Stop() + } + return readable, count, err +} + +func (p *kernelTCPIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + if p.harness.config.checkpoint != nil { + p.harness.config.checkpoint("receive", nil) + } + n, drained, err := p.goPlatformIO.readBurst(frames, options) + p.harness.access.Lock() + kept := 0 + for index := range n { + frame := frames[index] + packet := frame.buffer.Bytes() + p.rewriteSynOptions(packet) + flow, event := p.harness.eventLocked(packet, false) + if flow != nil { + action := kernelTCPAction{} + if flow.filter != nil { + action = flow.filter(event) + } + event.dropped = action.drop + event.delayed = action.delay > 0 + event.copies = max(action.copies, 1) + p.harness.recordLocked(flow, event) + if action.drop { + continue + } + if action.delay > 0 { + p.delayLocked(flow, event, packet, frame.meta, action.delay) + continue + } + for copyIndex := 1; copyIndex < action.copies; copyIndex++ { + flow.conn.engine.inject(packet, frame.meta) + } + } + frames[kept] = frame + kept++ + } + p.harness.access.Unlock() + return kept, drained, err +} + +func (p *kernelTCPIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + return p.write(frame, meta, false) +} + +func (p *kernelTCPIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + return p.write([][]byte{packet}, meta, false) +} + +func (p *kernelTCPIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { + return p.write(frame, meta, true) +} + +func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, data bool) error { + packet := slices.Concat(frame...) + p.harness.access.Lock() + flow, event := p.harness.eventLocked(packet, true) + action := kernelTCPAction{} + if flow != nil { + flow.writes++ + if flow.filter != nil { + action = flow.filter(event) + } + } + if action.pause != nil { + p.harness.access.Unlock() + close(action.pause.entered) + select { + case <-action.pause.resume: + case <-p.harness.done: + } + p.harness.access.Lock() + } + var err error + event.copies = max(action.copies, 1) + switch { + case action.drop: + event.dropped = true + case action.delay > 0: + event.delayed = true + p.delayLocked(flow, event, packet, meta, action.delay) + default: + for range event.copies { + if data { + err = p.goPlatformIO.writeData(frame, meta) + } else { + err = p.goPlatformIO.writeFrame(frame, meta) + } + if err != nil { + break + } + } + } + if flow != nil { + if err == nil { + p.harness.recordLocked(flow, event) + } + flow.writes-- + } + if data { + p.goPlatformIO.flush() + } + p.harness.access.Unlock() + return err +} + +func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, packet []byte, meta ForwardFrameMeta, delay time.Duration) { + if p.harness.stopped { + return + } + packet = slices.Clone(packet) + p.harness.workers.Go(func() { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-p.harness.done: + return + case <-timer.C: + } + p.harness.access.Lock() + defer p.harness.access.Unlock() + if p.harness.stopped || flow.conn.connState.Load() >= goConnStateDead { + return + } + event.at = time.Since(p.harness.epoch) + event.delayed = false + event.delivery = true + for range event.copies { + if event.outgoing { + err := p.goPlatformIO.writeData([][]byte{packet}, meta) + p.goPlatformIO.flush() + if err != nil && flow.failure == "" { + flow.failure = fmt.Sprintf("deliver delayed packet: %v", err) + } + } else { + flow.conn.engine.inject(packet, meta) + } + } + p.harness.recordLocked(flow, event) + }) +} + +func (p *kernelTCPIO) rewriteSynOptions(packet []byte) { + if !p.harness.config.disableSACK && !p.harness.config.disableTimestamps { + return + } + parsed, valid := parseForwardPacket(packet) + if !valid || !parsed.isPureTCPSyn() { + return + } + tcp := header.TCP(parsed.transport) + options := tcp.Options() + for index := 0; index < len(options); { + kind := options[index] + if kind == header.TCPOptionEOL { + break + } + if kind == header.TCPOptionNOP { + index++ + continue + } + if index+1 >= len(options) { + break + } + length := int(options[index+1]) + if length < 2 || index+length > len(options) { + break + } + if kind == header.TCPOptionSACKPermitted && p.harness.config.disableSACK || kind == header.TCPOptionTS && p.harness.config.disableTimestamps { + for offset := index; offset < index+length; offset++ { + options[offset] = header.TCPOptionNOP + } + } + index += length + } + var source, destination []byte + if parsed.ipVersion == 4 { + ip := header.IPv4(packet) + source, destination = ip.SourceAddressSlice(), ip.DestinationAddressSlice() + } else { + ip := header.IPv6(packet) + source, destination = ip.SourceAddressSlice(), ip.DestinationAddressSlice() + } + pseudo := header.PseudoHeaderChecksum(header.TCPProtocolNumber, source, destination, uint16(len(tcp))) + tcp.SetChecksum(0) + tcp.SetChecksum(^checksum.Checksum(tcp, pseudo)) +} diff --git a/stack_go_integration_test.go b/stack_go_integration_test.go index 41a9f86a..a6e248a3 100644 --- a/stack_go_integration_test.go +++ b/stack_go_integration_test.go @@ -6,12 +6,10 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "io" "net" "net/netip" - "os" "runtime" "sync" "sync/atomic" @@ -29,12 +27,13 @@ import ( type kernelStackConfig struct { stackFactory func(StackOptions) (Stack, error) + congestion string + prepareStack func(*Go) mtu uint32 gso bool multiQueue bool prepare func(*testing.T, *Options) configure func(*testing.T, Options) - pressure func() MemoryPressure ctx context.Context handshake func(*GoConn) error udpMapping NATMapping @@ -67,6 +66,9 @@ var kernelInterfaceSequence atomic.Uint32 func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackFixture { t.Helper() index := kernelInterfaceSequence.Add(1) + if index >= 1<<14 { + t.Fatal("test interface address range exhausted") + } fixture := &kernelStackFixture{ tcp: make(map[uint16]chan kernelAccept), udp: make(map[uint16]chan N.PacketConn), @@ -79,7 +81,7 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF MTU: config.mtu, GSO: config.gso, MultiQueue: config.multiQueue, - Inet4Address: []netip.Prefix{netip.MustParsePrefix(fmt.Sprintf("198.19.%d.1/24", 200+index%50))}, + Inet4Address: []netip.Prefix{netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 18 + byte(index>>13), byte(index >> 5), byte(index<<3) | 1}), 29)}, Inet6Address: []netip.Prefix{netip.MustParsePrefix(fmt.Sprintf("fd73:ab91:%x::1/64", index))}, EXP_ExternalConfiguration: true, EXP_MultiPendingPackets: true, @@ -114,8 +116,8 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF stackOptions := StackOptions{ Context: config.ctx, Tun: device, TunOptions: fixture.options, Handler: fixture, Logger: logger.NOP(), UDPTimeout: time.Minute, ICMPTimeout: time.Minute, - MemoryPressure: config.pressure, - UDPMapping: config.udpMapping, UDPFiltering: config.udpFiltering, + UDPMapping: config.udpMapping, UDPFiltering: config.udpFiltering, + TCPCongestionControl: config.congestion, } var stack Stack if config.stackFactory != nil { @@ -123,10 +125,20 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF if err != nil { t.Fatal(err) } + fixture.stack, _ = stack.(*Go) } else { - fixture.stack = NewGo(stackOptions) + fixture.stack, err = NewGo(stackOptions) + if err != nil { + t.Fatal(err) + } stack = fixture.stack } + if config.prepareStack != nil { + if fixture.stack == nil { + t.Fatal("prepareStack requires the Go stack") + } + config.prepareStack(fixture.stack) + } err = stack.Start() if err != nil { t.Fatal(err) @@ -355,8 +367,10 @@ func TestGoKernelStream(t *testing.T) { fixture := newKernelStackFixture(configurationTest, config) for _, ipv6 := range []bool{false, true} { configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Parallel() for _, splice := range []bool{false, true} { addressTest.Run(fmt.Sprintf("duplex/splice=%v", splice), func(scenarioTest *testing.T) { + scenarioTest.Parallel() var client, server net.Conn var upload, download *atomic.Int64 if splice { @@ -381,56 +395,8 @@ func TestGoKernelStream(t *testing.T) { } }) } - addressTest.Run("read_deadline_recovery", func(scenarioTest *testing.T) { - client, server := fixture.pair(scenarioTest, ipv6) - server.SetReadDeadline(time.Now().Add(20 * time.Millisecond)) - data := make([]byte, 7) - _, err := server.Read(data) - if !errors.Is(err, os.ErrDeadlineExceeded) { - scenarioTest.Fatalf("read deadline: %v", err) - } - server.SetReadDeadline(time.Now().Add(time.Second)) - _, err = client.Write([]byte("resumed")) - if err != nil { - scenarioTest.Fatal(err) - } - _, err = io.ReadFull(server, data) - if err != nil || string(data) != "resumed" { - scenarioTest.Fatalf("recovery: %q %v", data, err) - } - }) - addressTest.Run("write_deadline_recovery", func(scenarioTest *testing.T) { - client, server := fixture.pair(scenarioTest, ipv6) - client.SetReadBuffer(4096) - server.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) - payload := kernelPayload(8<<20, 31) - n, err := server.Write(payload) - if !errors.Is(err, os.ErrDeadlineExceeded) { - scenarioTest.Fatalf("write %d/%d deadline: %v", n, len(payload), err) - } - server.SetWriteDeadline(time.Now().Add(8 * time.Second)) - client.SetReadBuffer(4 << 20) - written := make(chan error, 1) - go func() { - _, writeErr := server.Write(payload[n:]) - if writeErr == nil { - writeErr = server.CloseWrite() - } - written <- writeErr - }() - data, readErr := io.ReadAll(client) - if readErr != nil { - scenarioTest.Fatalf("initial accepted=%d received=%d: %v; buffered=%d sent=%d transmitted=%d unacked=%d permit=%d", n, len(data), readErr, server.bufferedTail.Load(), server.sentTail.Load(), server.transmittedTail.Load(), server.sendUnacked.Load(), server.sendPermit.Load()) - } - if !bytes.Equal(data, payload) { - scenarioTest.Fatalf("deadline recovery: received %d/%d", len(data), len(payload)) - } - writeErr := <-written - if writeErr != nil { - scenarioTest.Fatal(writeErr) - } - }) addressTest.Run("wait_read_buffer", func(scenarioTest *testing.T) { + scenarioTest.Parallel() client, server := fixture.pair(scenarioTest, ipv6) server.InitializeReadWaiter(N.ReadWaitOptions{FrontHeadroom: 91, RearHeadroom: 73}) payload := kernelPayload(256<<10, 17) @@ -625,7 +591,7 @@ func kernelFlowRead(conn net.Conn, payload []byte, buffered bool) error { func TestGoKernelSequenceWrap(t *testing.T) { previous := runtime.GOMAXPROCS(4) t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, gso: runtime.GOOS == "linux"}) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000, gso: runtime.GOOS == "linux"}) for _, ipv6 := range []bool{false, true} { for _, mode := range []string{"write", "buffer", "splice"} { t.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { diff --git a/stack_go_integration_windows_test.go b/stack_go_integration_windows_test.go index 8e6f27b1..000ad004 100644 --- a/stack_go_integration_windows_test.go +++ b/stack_go_integration_windows_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/sagernet/sing-tun/internal/winipcfg" + E "github.com/sagernet/sing/common/exceptions" "golang.org/x/sys/windows" ) @@ -13,6 +14,13 @@ const ( kernelConnectionReset = windows.WSAECONNRESET ) +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + windows.SetsockoptInt(windows.Handle(descriptor), windows.SOL_SOCKET, windows.SO_RCVBUF, size), + windows.SetsockoptInt(windows.Handle(descriptor), windows.SOL_SOCKET, windows.SO_SNDBUF, size), + ) +} + func configureKernelInterface(t *testing.T, device Tun, options Options) { t.Helper() luid := winipcfg.LUID(device.(*NativeTun).adapter.LUID()) diff --git a/stack_go_io.go b/stack_go_io.go index a28549bf..38a859da 100644 --- a/stack_go_io.go +++ b/stack_go_io.go @@ -45,6 +45,7 @@ type goPlatformIO interface { unregisterSocket(socket *goSocket) readBurst(frames []goFrame, options N.ReadWaitOptions) (count int, drained bool, err error) writeFrame(frame [][]byte, meta ForwardFrameMeta) error + writePacket(packet []byte, meta ForwardFrameMeta) error writeData(frame [][]byte, meta ForwardFrameMeta) error writePacketBatch(frames []goUDPFrame) error releaseReadBuffers() @@ -70,23 +71,14 @@ func (w *goWriteback) WriteReturnPackets(packets [][]byte) error { prefix := w.platformIO.transmitPrefix() var writeErr error for _, packet := range packets { - writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeFrame([][]byte{packet[prefix:]}, ForwardFrameMeta{}))) + err := goIgnoreDropped(w.platformIO.writePacket(packet[prefix:], ForwardFrameMeta{})) + if err != nil { + writeErr = E.Errors(writeErr, err) + } } return writeErr } -func newGoReadBuffers(packetSize int, count int, options N.ReadWaitOptions) []*buf.Buffer { - slotSize := options.FrontHeadroom + packetSize + options.RearHeadroom - buffers := make([]*buf.Buffer, count) - for index := range buffers { - buffer := buf.NewSize(slotSize) - buffer.Resize(options.FrontHeadroom, 0) - buffer.Reserve(options.RearHeadroom) - buffers[index] = buffer - } - return buffers -} - func goIgnoreDropped(err error) error { if err == errGoFrameDropped { return nil diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go index 217b9be2..b15d09e8 100644 --- a/stack_go_io_darwin.go +++ b/stack_go_io_darwin.go @@ -386,7 +386,9 @@ func receiveMessageBatch(fd int, headers []rawfile.MsgHdrX) (int, unix.Errno) { } func (o *goDarwinIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { - err := o.writePacket(frame) + o.transmitAccess.Lock() + err := o.writePacketLocked(frame) + o.transmitAccess.Unlock() if err == errGoTransmitBlocked { o.droppedEngineFrames.record(o.stack.logger, "engine frames") return errGoFrameDropped @@ -394,6 +396,11 @@ func (o *goDarwinIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { return err } +func (o *goDarwinIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + func (o *goDarwinIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { o.transmitAccess.Lock() defer o.transmitAccess.Unlock() @@ -511,6 +518,7 @@ func (o *goDarwinIO) flushLocked() { return } room := o.gateRoom(pending) + sentFrom := o.batchStart for room > 0 { messages := o.batchMessages[o.batchStart : o.batchStart+room] n, errno := rawfile.NonBlockingSendMMsg(o.tunFd, messages) @@ -530,6 +538,7 @@ func (o *goDarwinIO) flushLocked() { o.gateSubmitted += int64(n) room -= n } + clear(o.batchIovecs[sentFrom*goDarwinBatchIovecs : o.batchStart*goDarwinBatchIovecs]) if o.batchStart == o.batchCount { o.batchStart = 0 o.batchCount = 0 @@ -622,6 +631,7 @@ func (o *goDarwinIO) spillHeld() { length += copy(spill[length:], bytesFromIovec(iovec)) } iovecs[1] = rawfile.IovecFromBytes(spill[:length]) + clear(iovecs[2:]) message.Msg.Iovlen = 2 o.batchSpilled[slot] = true } @@ -660,16 +670,11 @@ func (o *goDarwinIO) compactBatch() { o.batchMessages[target].Msg.Iovlen = int32(len(targetIovecs)) o.batchSpilled[target] = o.batchSpilled[slot] } + clear(o.batchIovecs[(o.batchCount-o.batchStart)*goDarwinBatchIovecs : o.batchCount*goDarwinBatchIovecs]) o.batchCount -= o.batchStart o.batchStart = 0 } -func (o *goDarwinIO) writePacket(frame [][]byte) error { - o.transmitAccess.Lock() - defer o.transmitAccess.Unlock() - return o.writePacketLocked(frame) -} - func (o *goDarwinIO) writePacketLocked(frame [][]byte) error { defer func() { clear(o.transmitIovecs) }() if o.gateRoom(1) == 0 { diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go index 1a1d276c..746f0b1e 100644 --- a/stack_go_io_linux.go +++ b/stack_go_io_linux.go @@ -373,6 +373,11 @@ func (o *goLinuxIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { } } +func (o *goLinuxIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { delay := goTransmitBackoffMin waited := time.Duration(0) @@ -410,7 +415,7 @@ func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Errno { var headerStorage [virtioNetHdrLen]byte var iovecStorage [goPacketBatchSize + 2]unix.Iovec - iovecs := iovecStorage[:0] + iovecCount := 0 if o.device.vnetHeader { prefix := goEmptyVirtioHeader[:] if meta != (ForwardFrameMeta{}) { @@ -430,7 +435,8 @@ func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Er } vector := unix.Iovec{Base: &prefix[0]} vector.SetLen(len(prefix)) - iovecs = append(iovecs, vector) + iovecStorage[iovecCount] = vector + iovecCount++ } for _, segment := range frame { if len(segment) == 0 { @@ -438,10 +444,11 @@ func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Er } vector := unix.Iovec{Base: &segment[0]} vector.SetLen(len(segment)) - iovecs = append(iovecs, vector) + iovecStorage[iovecCount] = vector + iovecCount++ } //nolint:staticcheck - _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) + _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecStorage[0])), uintptr(iovecCount)) return errno } @@ -523,25 +530,25 @@ func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { udpLength := uint16(header.UDPMinimumSize + payloadLength) var sourceAddress, destinationAddress []byte if frame.length == header.IPv4MinimumSize+header.UDPMinimumSize { - ipHdr := header.IPv4(packetHeader[:header.IPv4MinimumSize]) + ipHdr := header.IPv4(frame.header[:header.IPv4MinimumSize]) ipHdr.SetTotalLength(uint16(frame.length + payloadLength)) ipHdr.SetChecksum(0) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) sourceAddress = ipHdr.SourceAddressSlice() destinationAddress = ipHdr.DestinationAddressSlice() } else { - ipHdr := header.IPv6(packetHeader[:header.IPv6MinimumSize]) + ipHdr := header.IPv6(frame.header[:header.IPv6MinimumSize]) ipHdr.SetPayloadLength(udpLength) sourceAddress = ipHdr.SourceAddressSlice() destinationAddress = ipHdr.DestinationAddressSlice() } - udpHdr := header.UDP(packetHeader[frame.length-header.UDPMinimumSize : frame.length]) + udpHdr := header.UDP(frame.header[frame.length-header.UDPMinimumSize : frame.length]) udpHdr.SetLength(udpLength) udpHdr.SetChecksum(header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpLength)) meta.gsoType = goUDPGSOType meta.gsoSize = uint16(len(frame.payload)) } - segments[0] = packetHeader[:frame.length] + segments[0] = frame.header[:frame.length] for packetIndex := index; packetIndex < end; packetIndex++ { segments[packetIndex-index+1] = frames[packetIndex].payload } @@ -553,6 +560,7 @@ func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { switch errno { case unix.EINVAL, unix.EIO, unix.EOPNOTSUPP: o.device.udpTransmitDisabled.Store(true) + frame.header = packetHeader continue } } diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index 5dc8472d..b4e953c2 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -16,8 +16,6 @@ import ( const goEngineInlineTransmit = false -const goSessionRingCapacity = 0x800000 - const ( goCompletionKeyTun uintptr = iota + 1 goCompletionKeyWake @@ -59,10 +57,6 @@ func (o *goWindowsIO) start() error { return E.New("go: unsupported TUN implementation") } o.tun = nativeTun - err := nativeTun.resizeSessionRing(goSessionRingCapacity) - if err != nil { - return E.Cause(err, "go: resize wintun session ring") - } iocp, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 1) if err != nil { return E.Cause(err, "go: create completion port") @@ -165,12 +159,12 @@ func (o *goWindowsIO) wait(timeout time.Duration, events []goSocketEvent) (bool, waitMillis = uint32((timeout + time.Millisecond - 1) / time.Millisecond) } var removed uint32 - err := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) - if err != nil { - if err == windows.WAIT_TIMEOUT { + errno := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) + if errno != 0 { + if errno == windows.WAIT_TIMEOUT { return tunReadable, 0, nil } - return false, 0, E.Cause(err, "go: wait for completion") + return false, 0, E.Cause(errno, "go: wait for completion") } socketCount := 0 for index := range removed { @@ -342,7 +336,10 @@ func (o *goWindowsIO) writePacketBatch(frames []goUDPFrame) error { frame := &frames[index] segments[0] = frame.header[:frame.length] segments[1] = frame.payload - writeError = E.Errors(writeError, o.writeFrame(segments[:], frame.meta)) + err := o.writeFrame(segments[:], frame.meta) + if err != nil { + writeError = E.Errors(writeError, err) + } } return writeError } @@ -356,6 +353,11 @@ func (o *goWindowsIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { return err } +func (o *goWindowsIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { backoff := goTransmitBackoffMin waited := time.Duration(0) diff --git a/stack_go_profile_default.go b/stack_go_profile_default.go index ca476e3d..e8cca441 100644 --- a/stack_go_profile_default.go +++ b/stack_go_profile_default.go @@ -2,6 +2,8 @@ package tun +import "time" + const ( goReceiveCapacityBase = 64 << 10 goReceiveCapacityMax = 4 << 20 @@ -10,6 +12,7 @@ const ( goFlowCapacity = 16384 goReceiveBatchMin = 8 goReadBatch = 64 + goReadBufferIdle = time.Second goDescriptorRingCapacity = 2048 goReassemblyEntries = 8 goSlabPoolLowWater = 128 diff --git a/stack_go_profile_ios.go b/stack_go_profile_ios.go index e6732bac..4482bd3d 100644 --- a/stack_go_profile_ios.go +++ b/stack_go_profile_ios.go @@ -1,5 +1,7 @@ package tun +import "time" + const ( goReceiveCapacityBase = 32 << 10 goReceiveCapacityMax = 256 << 10 @@ -8,6 +10,7 @@ const ( goFlowCapacity = 1024 goReceiveBatchMin = 4 goReadBatch = 32 + goReadBufferIdle = 250 * time.Millisecond goDescriptorRingCapacity = 512 goReassemblyEntries = 4 goSlabPoolLowWater = 8 diff --git a/stack_go_ring.go b/stack_go_ring.go index 769ccf03..2df22056 100644 --- a/stack_go_ring.go +++ b/stack_go_ring.go @@ -2,6 +2,7 @@ package tun import ( "slices" + "sort" "sync" "sync/atomic" @@ -420,18 +421,26 @@ func (s *goTransmitStore) releaseAll() { } const ( - goDescriptorRetransmitted uint8 = 1 << iota + goDescriptorRetransmitted uint16 = 1 << iota goDescriptorNoSample goDescriptorSacked goDescriptorDropped goDescriptorAmend + goDescriptorRetransmitOut + goDescriptorRated + goDescriptorAppLimited + goDescriptorTxStart + goDescriptorLost ) type goSentDescriptor struct { - endOffset uint64 - sentAt int32 - flags uint8 - _ [3]byte + endOffset uint64 + sentAt int32 + deliveredStamp int32 + firstSentStamp int32 + delivered uint32 + flags uint16 + _ [2]byte } const ( @@ -445,12 +454,13 @@ type goDescriptorBlock [goDescriptorBlockCapacity]goSentDescriptor // head. Blocks are allocated on demand and returned after their final entry is // consumed. The directory stays fixed so neither side has to resize shared state. type goDescriptorRing struct { - blocks [goDescriptorRingCapacity / goDescriptorBlockCapacity]atomic.Pointer[goDescriptorBlock] - pool *goDescriptorPool - head atomic.Uint32 - tail atomic.Uint32 - pending goSentDescriptor - pendingActive bool + blocks [goDescriptorRingCapacity / goDescriptorBlockCapacity]atomic.Pointer[goDescriptorBlock] + pool *goDescriptorPool + head atomic.Uint32 + tail atomic.Uint32 + overflowed atomic.Bool + overflowAccess sync.Mutex + overflow []goSentDescriptor } type goDescriptorPool struct { @@ -500,36 +510,28 @@ func (r *goDescriptorRing) release() { } r.head.Store(0) r.tail.Store(0) - r.pending = goSentDescriptor{} - r.pendingActive = false + r.overflow = nil + r.overflowed.Store(false) } -func (r *goDescriptorRing) push(endOffset uint64, sentAt int32, flags uint8) { - if flags&goDescriptorAmend != 0 { - r.pushAmendment(endOffset, sentAt, flags) +func (r *goDescriptorRing) push(entry goSentDescriptor) { + if !r.overflowed.Load() && r.store(entry) { return } - if r.pendingActive { - r.pending.endOffset = endOffset - r.pending.sentAt = sentAt - r.pending.flags |= flags - } else { - r.pending = goSentDescriptor{endOffset: endOffset, sentAt: sentAt, flags: flags} - r.pendingActive = true - } - if r.store(r.pending) { - r.pendingActive = false + r.overflowAccess.Lock() + defer r.overflowAccess.Unlock() + if !r.overflowed.Load() && r.store(entry) { + return } -} - -func (r *goDescriptorRing) pushAmendment(endOffset uint64, length int32, flags uint8) { - if r.pendingActive { - if !r.store(r.pending) { + if entry.flags&goDescriptorAmend != 0 && len(r.overflow) > 0 { + last := &r.overflow[len(r.overflow)-1] + if last.flags&goDescriptorAmend != 0 && last.endOffset == entry.endOffset && last.sentAt == entry.sentAt { + last.flags |= entry.flags return } - r.pendingActive = false } - r.store(goSentDescriptor{endOffset: endOffset, sentAt: length, flags: flags}) + r.overflow = append(r.overflow, entry) + r.overflowed.Store(true) } func (r *goDescriptorRing) store(entry goSentDescriptor) bool { @@ -577,28 +579,44 @@ func (s *goScoreboard) append(entry goSentDescriptor) { } func (s *goScoreboard) drain(ring *goDescriptorRing, unacked uint64) { + s.drainRing(ring, unacked) + if !ring.overflowed.Load() { + return + } + ring.overflowAccess.Lock() + defer ring.overflowAccess.Unlock() + s.drainRing(ring, unacked) + for _, entry := range ring.overflow { + s.drainEntry(entry, unacked) + } + ring.overflow = ring.overflow[:0] + ring.overflowed.Store(false) +} + +func (s *goScoreboard) drainRing(ring *goDescriptorRing, unacked uint64) { head := ring.head.Load() tail := ring.tail.Load() for head != tail { entry := ring.take(head) head++ - if entry.flags&goDescriptorAmend != 0 { - s.amend(entry.endOffset-uint64(entry.sentAt), entry.endOffset, entry.flags&^goDescriptorAmend) - continue - } - if entry.endOffset > unacked { - s.append(entry) - } + s.drainEntry(entry, unacked) } ring.head.Store(head) } -func (s *goScoreboard) amend(start uint64, end uint64, flags uint8) { - for index := range s.entries { +func (s *goScoreboard) drainEntry(entry goSentDescriptor, unacked uint64) { + if entry.flags&goDescriptorAmend != 0 { + s.amend(entry.endOffset-uint64(entry.sentAt), entry.endOffset, entry.flags&^goDescriptorAmend) + } else if entry.endOffset > unacked { + s.append(entry) + } +} + +func (s *goScoreboard) amend(start uint64, end uint64, flags uint16) { + first := sort.Search(len(s.entries), func(index int) bool { return s.entries[index].endOffset > start }) + for index := first; index < len(s.entries) && s.entries[index].endOffset <= end; index++ { entry := &s.entries[index] - if entry.endOffset > start && entry.endOffset <= end { - entry.flags |= flags - } + entry.flags |= flags } } @@ -612,9 +630,6 @@ func (s *goScoreboard) advance(acked uint64) { } remaining := copy(s.entries, s.entries[dropped:]) s.entries = s.entries[:remaining] - if remaining == 0 { - s.reset() - } } func (s *goScoreboard) startOf(index int, unacked uint64) uint64 { @@ -638,6 +653,12 @@ func (s *goScoreboard) reset() { s.entries = nil } +func (s *goScoreboard) trim() { + if len(s.entries) == 0 && cap(s.entries) > goScoreboardInlineCapacity { + s.entries = nil + } +} + const goMaxSackBlocks = 4 const goMaxSackBlocksSent = 3 @@ -711,7 +732,7 @@ func (s *goRangeSet) first() (goRange, bool) { func (s *goRangeSet) removeBelow(offset uint64) { kept := 0 for index := range s.count { - if s.ranges[index].end <= offset { + if s.ranges[index].end < offset || s.ranges[index].end == offset && !s.ranges[index].fin { continue } s.ranges[kept] = s.ranges[index] @@ -723,13 +744,15 @@ func (s *goRangeSet) removeBelow(offset uint64) { s.count = kept } -func (s *goRangeSet) covers(start uint64, end uint64) bool { +func (s *goRangeSet) overlap(start uint64, end uint64) (uint64, uint64, bool) { for index := range s.count { - if s.ranges[index].start <= start && end <= s.ranges[index].end { - return true + candidate := s.ranges[index] + if candidate.end <= start || candidate.start >= end { + continue } + return max(start, candidate.start), min(end, candidate.end), true } - return false + return 0, 0, false } func (s *goRangeSet) bytes() uint64 { @@ -746,10 +769,10 @@ func (s *goRangeSet) blocks(out *[goMaxSackBlocks]goSackBlock, base int) int { if emitted == goMaxSackBlocksSent { break } - if start == 0 { + if start == 0 || slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { continue } - found := slices.IndexFunc(s.ranges[:s.count], func(candidate goRange) bool { return candidate.start == start }) + found := slices.IndexFunc(s.ranges[:s.count], func(candidate goRange) bool { return candidate.start == start && candidate.end > start }) if found < 0 { continue } @@ -758,7 +781,7 @@ func (s *goRangeSet) blocks(out *[goMaxSackBlocks]goSackBlock, base int) int { } for index := 0; index < s.count && emitted < goMaxSackBlocksSent; index++ { start := s.ranges[index].start - if slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { + if start == s.ranges[index].end || slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { continue } out[emitted] = goSackBlock{start: s.ranges[index].start, end: s.ranges[index].end} diff --git a/stack_go_socket_windows.go b/stack_go_socket_windows.go index ae90e86d..83521795 100644 --- a/stack_go_socket_windows.go +++ b/stack_go_socket_windows.go @@ -22,6 +22,26 @@ const ( type goIOVector = windows.WSABuf +var ( + modws2_32 = windows.NewLazySystemDLL("ws2_32.dll") + procWSARecv = modws2_32.NewProc("WSARecv") + procWSARecvFrom = modws2_32.NewProc("WSARecvFrom") + procWSASend = modws2_32.NewProc("WSASend") + procWSASendTo = modws2_32.NewProc("WSASendTo") +) + +const goSocketError = uintptr(^uint32(0)) + +func goSocketErrno(r1 uintptr, errno syscall.Errno) syscall.Errno { + if r1 != goSocketError { + return 0 + } + if errno == 0 { + return syscall.EINVAL + } + return errno +} + type goSocket struct { handle windows.Handle entry *goAFDEntry @@ -122,18 +142,20 @@ func (s *goSocket) readVector(iovecs []goIOVector) (int, syscall.Errno) { received uint32 flags uint32 ) - err := windows.WSARecv(s.handle, &iovecs[0], uint32(len(iovecs)), &received, &flags, nil, nil) - if err != nil { - return 0, err.(syscall.Errno) + r1, _, e1 := syscall.SyscallN(procWSARecv.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs)), uintptr(unsafe.Pointer(&received)), uintptr(unsafe.Pointer(&flags)), 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno } return int(received), 0 } func (s *goSocket) writeVector(iovecs []goIOVector) (int, syscall.Errno) { var sent uint32 - err := windows.WSASend(s.handle, &iovecs[0], uint32(len(iovecs)), &sent, 0, nil, nil) - if err != nil { - return 0, err.(syscall.Errno) + r1, _, e1 := syscall.SyscallN(procWSASend.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs)), uintptr(unsafe.Pointer(&sent)), 0, 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno } return int(sent), 0 } @@ -144,9 +166,10 @@ func (s *goSocket) write(data []byte) (int, syscall.Errno) { vector.Buf = &data[0] } var sent uint32 - err := windows.WSASend(s.handle, &vector, 1, &sent, 0, nil, nil) - if err != nil { - return 0, err.(syscall.Errno) + r1, _, e1 := syscall.SyscallN(procWSASend.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&sent)), 0, 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno } return int(sent), 0 } @@ -178,11 +201,8 @@ func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) vector.Buf = &data[0] } var sent uint32 - err := windows.WSASendTo(s.handle, &vector, 1, &sent, 0, &storage, length, nil, nil) - if err != nil { - return err.(syscall.Errno) - } - return 0 + r1, _, e1 := syscall.SyscallN(procWSASendTo.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&sent)), 0, uintptr(unsafe.Pointer(&storage)), uintptr(length), 0, 0) + return goSocketErrno(r1, e1) } func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { @@ -193,9 +213,10 @@ func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errn flags uint32 ) vector := windows.WSABuf{Len: uint32(len(buffer)), Buf: &buffer[0]} - err := windows.WSARecvFrom(s.handle, &vector, 1, &received, &flags, &storage, &length, nil, nil) - if err != nil { - return 0, netip.AddrPort{}, err.(syscall.Errno) + r1, _, e1 := syscall.SyscallN(procWSARecvFrom.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&received)), uintptr(unsafe.Pointer(&flags)), uintptr(unsafe.Pointer(&storage)), uintptr(unsafe.Pointer(&length)), 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, netip.AddrPort{}, errno } return int(received), M.SocksaddrFromNetIP(M.AddrPortFromRawSockaddr(&storage.Addr)).Unwrap().AddrPort(), 0 } diff --git a/stack_go_splice.go b/stack_go_splice.go index 24cce2d1..990ceae6 100644 --- a/stack_go_splice.go +++ b/stack_go_splice.go @@ -426,12 +426,12 @@ func (e *goEngine) spliceTransmit(conn *GoConn) bool { if conn.blockedValid || !conn.transmitOwner.CompareAndSwap(0, 1) { return true } - _, blocked := conn.transmitLoop(false, 0) + _, result := conn.transmitLoop(false, 0) conn.transmitOwner.Store(0) if conn.connState.Load() >= goConnStateAborted { return false } - if blocked { + if result == goTransmitBlocked { e.handleTransmitBlocked(conn) } return true @@ -470,6 +470,9 @@ func (e *goEngine) spliceRetryBlocked(conn *GoConn) { e.spliceAbort(conn, E.Cause(err, "go: write tun")) return } + if conn.congestion.pacing { + conn.advancePacing(e.now(), segment.length) + } conn.blockedValid = false conn.blockedFrame = nil conn.transmittedTail.Store(end) diff --git a/stack_go_tcp_ack.go b/stack_go_tcp_ack.go new file mode 100644 index 00000000..dffa5e4e --- /dev/null +++ b/stack_go_tcp_ack.go @@ -0,0 +1,1193 @@ +package tun + +import ( + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" +) + +// Mirrors the ACK processing and loss recovery state machine of net/ipv4/tcp_input.c. + +type goSackState struct { + firstStamp int32 + lastStamp int32 + sampled bool +} + +func (e *goEngine) drainDescriptors(conn *GoConn) { + before := len(conn.scoreboard.entries) + conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) + entries := conn.scoreboard.entries + for index := before; index < len(entries); index++ { + descriptor := &entries[index] + sentAt := conn.descriptorSentTime(descriptor) + if descriptor.flags&goDescriptorTxStart != 0 { + descriptor.flags &^= goDescriptorTxStart + conn.rackEndOffset = 0 + conn.rackAdvanced = false + if conn.congestion.windowEventTxStart != nil { + var idle int64 + if conn.lastSendStamp >= 0 { + idle = max(sentAt-conn.lastSendStamp, 0) + } + conn.congestion.windowEventTxStart(conn, idle) + } + } + conn.lastSendStamp = sentAt + } + e.refreshPacketsOut(conn) +} + +func (e *goEngine) summarizeFlight(conn *GoConn) { + var summary goFlightSummary + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.flags&goDescriptorRetransmitted != 0 { + summary.retransmitted++ + } + if descriptor.flags&goDescriptorSacked != 0 { + summary.sackedOut++ + continue + } + if descriptor.flags&goDescriptorLost != 0 { + summary.lostOut++ + } + if descriptor.flags&goDescriptorRetransmitOut != 0 { + summary.retransmitOut++ + } + } + if !conn.sackPermitted { + summary.sackedOut = conn.renoSacked + } + conn.flight = summary + e.refreshPacketsOut(conn) +} + +func (e *goEngine) refreshPacketsOut(conn *GoConn) { + entries := conn.scoreboard.entries + start := conn.sendUnacked.Load() + if len(entries) > 0 { + start = entries[len(entries)-1].endOffset + } + sent := conn.sentTail.Load() + if sent > start { + start = sent + } + conn.flight.packetsOut = conn.dataSegmentsOut.Load() - conn.packetCreditBase.Load() + conn.flight.packetsEnd = start +} + +func (e *goEngine) accountRemoved(conn *GoConn, flags uint16) { + conn.packetCreditBase.Add(1) + if flags&goDescriptorRetransmitted != 0 { + conn.flight.retransmitted-- + } + if flags&goDescriptorSacked != 0 { + conn.flight.sackedOut-- + return + } + if flags&goDescriptorLost != 0 { + conn.flight.lostOut-- + } + if flags&goDescriptorRetransmitOut != 0 { + conn.flight.retransmitOut-- + } +} + +func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, payloadLength int, flags header.TCPFlags, timestampEcho uint32, hasTimestamp bool) bool { + e.refreshCoarseTime() + e.drainDescriptors(conn) + ackOffset := conn.sendOffset(tcpHdr.AckNumber()) + limit := int64(conn.sentTail.Load()) + if conn.finSent { + limit++ + } + if ackOffset > limit { + e.markAck(conn, true) + return false + } + priorUnacked := conn.sendUnacked.Load() + segmentAck := ackOffset + if ackOffset < int64(priorUnacked) { + ackOffset = int64(priorUnacked) + } + if conn.state == goTCPSynReceived { + if conn.connState.Load() != goConnStateEngaged { + return false + } + if segmentAck != int64(conn.sentTail.Load()) { + e.sendReset(conn) + e.detachConn(conn, E.New("go: handshake ack out of range"), goDeathImmediate) + return false + } + e.establishConn(conn) + } + if hasTimestamp { + conn.lastTimestampEcho = timestampEcho + } + var ackFlags goAckFlags + if payloadLength > 0 || flags&(header.TCPFlagSyn|header.TCPFlagFin) != 0 { + ackFlags |= goAckData + } + rawCount := 0 + if conn.sackPermitted { + rawCount = goParseSackBlocks(tcpHdr, int(tcpHdr.DataOffset()), conn, &e.sackRaw) + } + if segmentAck < int64(priorUnacked) { + if rawCount > 0 { + e.processOldAckSack(conn, priorUnacked, segmentAck, rawCount) + } + return true + } + windowRaw := tcpHdr.WindowSize() + newWindow := uint64(windowRaw) << conn.peerWindowShift + if segmentAck > int64(priorUnacked) || segOffset > conn.windowLeft1 || (segOffset == conn.windowLeft1 && (newWindow > conn.peerWindow || newWindow == 0)) { + ackFlags |= goAckWindowUpdate + conn.windowLeft1 = segOffset + conn.windowLeft2 = segmentAck + conn.peerWindow = newWindow + conn.maxPeerWindow = max(conn.maxPeerWindow, newWindow) + conn.lastPeerWindow = windowRaw + } + if ackOffset > int64(priorUnacked) { + ackFlags |= goAckUnackedAdvanced + conn.retransmitAttempts = 0 + } + priorFack := conn.highestSacked + hadOutstanding := conn.hasOutstanding() + priorDelivered := conn.delivered + priorLost := conn.lost + sample := &e.rateSample + *sample = goRateSample{roundTripMicros: -1, priorInFlight: conn.flight.inFlight()} + var sackState goSackState + if rawCount > 0 { + raw := e.sackRaw[:rawCount] + if goIsDuplicateSack(raw, ackOffset) { + e.applyDuplicateSack(conn, raw[0], priorUnacked, &ackFlags) + raw = raw[1:] + } + sackCount := goSelectSackBlocks(raw, uint64(ackOffset), conn.sentTail.Load(), &e.sackScratch) + if sackCount > 0 { + e.applySackBlocks(conn, e.sackScratch[:sackCount], &ackFlags, sample, &sackState) + } + } + ackFlags |= e.cleanRetransmitQueue(conn, uint64(ackOffset), priorFack, priorUnacked, sample, &sackState, timestampEcho, hasTimestamp) + e.rackUpdateReorderWindow(conn, sample) + if conn.probeHighSeq != 0 { + e.processProbeAck(conn, uint64(ackOffset), ackFlags) + } + rexmit := goRexmitNone + if !hadOutstanding { + if ackFlags&goAckDSACK != 0 { + rexmit = e.fastretransAlert(conn, priorUnacked, 0, &ackFlags) + } + } else { + if ackFlags&goAckNotDuplicate == 0 || ackFlags&goAckAlert != 0 || conn.congestionState != goCongestionOpen { + numDupack := uint32(0) + if ackFlags&(goAckUnackedAdvanced|goAckNotDuplicate|goAckDSACK) == 0 { + numDupack = 1 + } + rexmit = e.fastretransAlert(conn, priorUnacked, numDupack, &ackFlags) + } + sample.ackDelayed = ackFlags&goAckMaybeDelayed != 0 + delivered := conn.delivered - priorDelivered + e.rateGenerate(conn, sample, delivered, conn.lost-priorLost, e.now()) + e.validateWindow(conn) + e.congestionControl(conn, delivered, ackFlags, sample) + e.updateFrameLimit(conn) + } + if ackFlags&goAckUnackedAdvanced != 0 { + e.advanceFinState(conn) + if conn.connState.Load() >= goConnStateAborted { + return true + } + if len(conn.scoreboard.entries) == 0 && conn.sendUnacked.Load() == conn.sentTail.Load() { + e.armIdleRestart(conn) + } + if ackFlags&goAckKeepTimer == 0 { + e.rearmRetransmit(conn) + } + } + e.xmitRecovery(conn, rexmit) + e.updatePersist(conn) + conn.releaseIdleDescriptors() + conn.publishPermit(uint64(ackOffset)) + return true +} + +func (e *goEngine) processOldAckSack(conn *GoConn, priorUnacked uint64, ackOffset int64, rawCount int) { + var ackFlags goAckFlags + sample := goRateSample{roundTripMicros: -1, priorInFlight: conn.flight.inFlight()} + var sackState goSackState + raw := e.sackRaw[:rawCount] + if goIsDuplicateSack(raw, ackOffset) { + e.applyDuplicateSack(conn, raw[0], priorUnacked, &ackFlags) + raw = raw[1:] + } + sackCount := goSelectSackBlocks(raw, priorUnacked, conn.sentTail.Load(), &e.sackScratch) + if sackCount > 0 { + e.applySackBlocks(conn, e.sackScratch[:sackCount], &ackFlags, &sample, &sackState) + } + e.refreshPacketsOut(conn) + rexmit := e.fastretransAlert(conn, priorUnacked, 0, &ackFlags) + e.xmitRecovery(conn, rexmit) + conn.publishPermit(priorUnacked) +} + +func (e *goEngine) applyDuplicateSack(conn *GoConn, block goRawSackBlock, priorUnacked uint64, flags *goAckFlags) { + if block.start < 1 || block.end <= block.start || uint64(block.end) > conn.sentTail.Load() { + return + } + start := uint64(block.start) + end := uint64(block.end) + if end-start > conn.maxPeerWindow { + return + } + packets := (end - start + uint64(conn.effectiveMSS) - 1) / uint64(conn.effectiveMSS) + conn.duplicateSegments += packets + if conn.duplicateSegments > conn.totalRetransmits { + return + } + *flags |= goAckDSACK + if end-start <= uint64(conn.effectiveMSS) && conn.probeHighSeq != 0 && conn.probeRetransmitted && end == conn.probeHighSeq { + *flags |= goAckDSACKProbe + } + if conn.reorderingSeen != 0 && *flags&goAckDSACKProbe == 0 { + conn.rackDSACKSeen = true + } + if end <= priorUnacked { + if conn.undoMarker != 0 && conn.undoRetransmits > 0 && end > conn.undoMarker { + conn.undoRetransmits -= int32(min(packets, uint64(conn.undoRetransmits))) + } + return + } + descriptorStart := priorUnacked + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + descriptorEnd := descriptor.endOffset + if descriptorStart >= end { + break + } + if descriptorStart >= start && descriptorEnd <= end && descriptor.flags&goDescriptorRetransmitted != 0 { + if conn.undoMarker != 0 && conn.undoRetransmits > 0 && descriptorEnd > conn.undoMarker { + conn.undoRetransmits-- + } + if descriptor.flags&goDescriptorRetransmitOut != 0 && descriptor.flags&goDescriptorSacked == 0 { + conn.flight.retransmitOut-- + } + descriptor.flags &^= goDescriptorRetransmitOut + } + descriptorStart = descriptorEnd + } +} + +func (e *goEngine) applySackBlocks(conn *GoConn, blocks []goSackBlock, flags *goAckFlags, sample *goRateSample, sackState *goSackState) { + unacked := conn.sendUnacked.Load() + priorHighest := conn.highestSacked + reorder := conn.sentTail.Load() + start := unacked + scoreboard := &conn.scoreboard + for index := 0; index < len(scoreboard.entries); index++ { + descriptor := &scoreboard.entries[index] + end := descriptor.endOffset + if descriptor.flags&goDescriptorSacked == 0 { + for _, block := range blocks { + if block.start > start || block.end < end { + continue + } + e.rackAdvance(conn, descriptor, end) + if descriptor.flags&goDescriptorRetransmitted == 0 { + if start < priorHighest && start < reorder { + reorder = start + } + if end <= conn.highSeq { + *flags |= goAckOriginalSacked + } + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) == 0 { + if !sackState.sampled { + sackState.firstStamp = descriptor.sentAt + sackState.sampled = true + } + sackState.lastStamp = descriptor.sentAt + } + } + if descriptor.flags&goDescriptorRetransmitOut != 0 { + if descriptor.flags&goDescriptorLost != 0 { + descriptor.flags &^= goDescriptorLost | goDescriptorRetransmitOut + conn.flight.lostOut-- + conn.flight.retransmitOut-- + } else { + conn.flight.retransmitOut-- + } + } else if descriptor.flags&goDescriptorLost != 0 { + descriptor.flags &^= goDescriptorLost + conn.flight.lostOut-- + } + descriptor.flags |= goDescriptorSacked + conn.flight.sackedOut++ + conn.highestSacked = max(conn.highestSacked, end) + *flags |= goAckDataSacked + conn.countDelivered(1) + e.rateDescriptorDelivered(conn, descriptor, sample) + break + } + } + start = end + } + if conn.congestionState != goCongestionLoss || conn.undoMarker != 0 { + e.checkSackReordering(conn, reorder) + } +} + +func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFack uint64, priorUnacked uint64, sample *goRateSample, sackState *goSackState, timestampEcho uint32, hasTimestamp bool) goAckFlags { + var flags goAckFlags + sent := conn.sentTail.Load() + newUnacked := min(ackOffset, sent) + priorSacked := conn.flight.sackedOut + var ( + firstStamp int32 + lastStamp int32 + sampled bool + unsampled bool + packetsAcked uint32 + ) + reorder := sent + start := priorUnacked + entries := conn.scoreboard.entries + for index := range entries { + descriptor := &entries[index] + if descriptor.endOffset > newUnacked { + break + } + if descriptor.flags&goDescriptorRetransmitted != 0 { + flags |= goAckRetransmittedAcked + } else if descriptor.flags&goDescriptorSacked == 0 { + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + unsampled = true + } else { + lastStamp = descriptor.sentAt + if !sampled { + firstStamp = descriptor.sentAt + sampled = true + } + } + if start < reorder { + reorder = start + } + if descriptor.endOffset <= conn.highSeq { + flags |= goAckOriginalSacked + } + } + if descriptor.flags&goDescriptorSacked == 0 && conn.sackPermitted { + conn.countDelivered(1) + if !conn.spuriousRetransmit(descriptor) { + e.rackAdvance(conn, descriptor, descriptor.endOffset) + } + } + packetsAcked++ + e.rateDescriptorDelivered(conn, descriptor, sample) + e.accountRemoved(conn, descriptor.flags) + flags |= goAckDataAcked + start = descriptor.endOffset + } + if conn.finSent && ackOffset > conn.finOffset { + conn.finAcked = true + } + if newUnacked > priorUnacked { + conn.scoreboard.advance(newUnacked) + conn.releaseTransmitted(newUnacked) + conn.sendUnacked.Store(newUnacked) + if len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorSacked != 0 { + flags |= goAckReneging + } + } + e.refreshPacketsOut(conn) + now := e.now() + nowStamp := conn.stamp(now) + sequenceRoundTrip := int64(-1) + sackRoundTrip := int64(-1) + controlRoundTrip := int64(-1) + if sampled && !unsampled && flags&goAckRetransmittedAcked == 0 { + sequenceRoundTrip = goStampMicros(nowStamp - firstStamp) + controlRoundTrip = goStampMicros(nowStamp - lastStamp) + if packetsAcked == 1 && priorSacked == 0 && newUnacked-priorUnacked < uint64(conn.effectiveMSS) && + sample.priorDelivered+1 == conn.delivered && flags&goAckAlert == 0 { + flags |= goAckMaybeDelayed + } + } + if sackState.sampled { + sackRoundTrip = goStampMicros(nowStamp - sackState.firstStamp) + controlRoundTrip = goStampMicros(nowStamp - sackState.lastStamp) + } + e.ackUpdateRoundTrip(conn, flags, sequenceRoundTrip, sackRoundTrip, controlRoundTrip, sample, timestampEcho, hasTimestamp, now) + if flags&goAckDataAcked != 0 { + if !conn.sackPermitted { + e.removeRenoSacks(conn, packetsAcked) + if flags&goAckRetransmittedAcked != 0 { + flags &^= goAckOriginalSacked + } + } else if reorder < priorFack { + e.checkSackReordering(conn, reorder) + } + conn.probeAttempts = 0 + } + if conn.congestion.packetsAcked != nil { + ackSample := &e.ackSample + *ackSample = goAckSample{ + packetsAcked: packetsAcked, + roundTripMicros: sample.roundTripMicros, + inFlightBytes: uint64(conn.effectiveMSS) * uint64(conn.delivered-sample.priorDelivered), + } + conn.congestion.packetsAcked(conn, ackSample) + } + return flags +} + +func (e *goEngine) ackUpdateRoundTrip(conn *GoConn, flags goAckFlags, sequenceRoundTrip int64, sackRoundTrip int64, controlRoundTrip int64, sample *goRateSample, timestampEcho uint32, hasTimestamp bool, now int64) { + if sequenceRoundTrip < 0 { + sequenceRoundTrip = sackRoundTrip + } + if sequenceRoundTrip < 0 && hasTimestamp && timestampEcho != 0 && flags&goAckDataAcked != 0 { + elapsed := goTimestampAt(now) - timestampEcho + if int32(elapsed) >= 0 { + sequenceRoundTrip = max(int64(elapsed)< conn.roundTripMin.get() { + return + } + conn.roundTripMin.runningMin(uint32(goMinRoundTripWindow/time.Millisecond), uint32(now/int64(time.Millisecond)), max(roundTripMicros, 1)) +} + +func (e *goEngine) limitRenoSacked(conn *GoConn) bool { + holes := min(max(conn.flight.lostOut, 1), conn.flight.packetsOut) + if conn.renoSacked+holes > conn.flight.packetsOut { + conn.renoSacked = conn.flight.packetsOut - holes + conn.flight.sackedOut = conn.renoSacked + return true + } + return false +} + +func (e *goEngine) checkRenoReordering(conn *GoConn, addend uint32) { + if !e.limitRenoSacked(conn) { + return + } + conn.reordering = min(conn.flight.packetsOut+addend, goMaxReordering) + conn.reorderingSeen++ +} + +func (e *goEngine) addRenoSack(conn *GoConn, numDupack uint32) { + if numDupack == 0 { + return + } + prior := conn.renoSacked + conn.renoSacked += numDupack + conn.flight.sackedOut = conn.renoSacked + e.checkRenoReordering(conn, 0) + if conn.renoSacked > prior { + conn.countDelivered(conn.renoSacked - prior) + } +} + +func (e *goEngine) removeRenoSacks(conn *GoConn, acked uint32) { + if acked > 0 { + conn.countDelivered(max(acked-min(acked, conn.renoSacked), 1)) + if acked-1 >= conn.renoSacked { + conn.renoSacked = 0 + } else { + conn.renoSacked -= acked - 1 + } + } + conn.flight.sackedOut = conn.renoSacked + e.checkRenoReordering(conn, acked) +} + +func (e *goEngine) processProbeAck(conn *GoConn, ackOffset uint64, flags goAckFlags) { + if ackOffset < conn.probeHighSeq { + return + } + switch { + case !conn.probeRetransmitted: + conn.probeHighSeq = 0 + case flags&goAckDSACKProbe != 0: + conn.probeHighSeq = 0 + case ackOffset > conn.probeHighSeq: + e.initWindowReduction(conn) + conn.setCAState(goCongestionCWR) + e.endWindowReduction(conn) + e.tryKeepOpen(conn) + case flags&(goAckUnackedAdvanced|goAckNotDuplicate|goAckDataSacked) == 0: + conn.probeHighSeq = 0 + } +} + +func (e *goEngine) markLost(conn *GoConn, descriptor *goSentDescriptor) { + if descriptor.flags&goDescriptorSacked != 0 || descriptor.endOffset > conn.transmittedTail.Load() { + return + } + if descriptor.flags&goDescriptorLost != 0 { + if descriptor.flags&goDescriptorRetransmitOut != 0 { + descriptor.flags &^= goDescriptorRetransmitOut + conn.flight.retransmitOut-- + conn.lost++ + } + return + } + descriptor.flags |= goDescriptorLost + conn.flight.lostOut++ + conn.lost++ +} + +func (e *goEngine) accountSplit(conn *GoConn, flags uint16) { + conn.packetCreditBase.Add(^uint32(0)) + conn.flight.packetsOut++ + if flags&goDescriptorRetransmitted != 0 { + conn.flight.retransmitted++ + } + if flags&goDescriptorSacked != 0 { + conn.flight.sackedOut++ + return + } + if flags&goDescriptorLost != 0 { + conn.flight.lostOut++ + } + if flags&goDescriptorRetransmitOut != 0 { + conn.flight.retransmitOut++ + } +} + +func (c *GoConn) descriptorSentTime(descriptor *goSentDescriptor) int64 { + now := c.engine.now() + units := (now - c.epoch) / goTimeUnit + age := int64(int32(units) - descriptor.sentAt) + return c.epoch + (units-age)*goTimeUnit +} + +func (c *GoConn) descriptorTimestamp(descriptor *goSentDescriptor) uint32 { + return goTimestampAt(c.descriptorSentTime(descriptor)) +} + +func (c *GoConn) spuriousRetransmit(descriptor *goSentDescriptor) bool { + return descriptor.flags&goDescriptorRetransmitted != 0 && c.echoedTimestampBefore(c.descriptorTimestamp(descriptor)) +} + +func (e *goEngine) rackAdvance(conn *GoConn, descriptor *goSentDescriptor, endOffset uint64) { + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + return + } + roundTrip := goStampMicros(conn.stamp(e.now()) - descriptor.sentAt) + if roundTrip < int64(conn.minRoundTripMicros()) && descriptor.flags&goDescriptorRetransmitted != 0 { + return + } + conn.rackAdvanced = true + conn.rackRoundTripMicros = roundTrip + if conn.rackEndOffset == 0 || goStampAfter(descriptor.sentAt, conn.rackStamp, endOffset, conn.rackEndOffset) { + conn.rackStamp = descriptor.sentAt + conn.rackEndOffset = endOffset + } +} + +func (e *goEngine) rackReorderWindow(conn *GoConn) int64 { + if conn.reorderingSeen == 0 { + if conn.congestionState >= goCongestionRecovery { + return 0 + } + if conn.flight.sackedOut >= conn.reordering { + return 0 + } + } + return min(int64(conn.minRoundTripMicros()>>2)*int64(conn.reorderWindowSteps), int64(conn.smoothedRoundTrip)) +} + +func (e *goEngine) rackDescriptorTimeout(conn *GoConn, descriptor *goSentDescriptor, reorderWindow int64, now int64) int64 { + return conn.rackRoundTripMicros + reorderWindow - goStampMicros(conn.stamp(now)-descriptor.sentAt) +} + +func (e *goEngine) rackDetectLoss(conn *GoConn) int64 { + if conn.rackEndOffset == 0 { + return 0 + } + now := e.now() + reorderWindow := e.rackReorderWindow(conn) + timeout := int64(0) + ordered := conn.flight.retransmitted == 0 + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.flags&goDescriptorSacked != 0 { + continue + } + if descriptor.flags&goDescriptorLost != 0 && descriptor.flags&goDescriptorRetransmitOut == 0 { + continue + } + if !goStampAfter(conn.rackStamp, descriptor.sentAt, conn.rackEndOffset, descriptor.endOffset) { + if ordered { + break + } + continue + } + remaining := e.rackDescriptorTimeout(conn, descriptor, reorderWindow, now) + if remaining <= 0 { + e.markLost(conn, descriptor) + } else { + timeout = max(timeout, remaining) + } + } + return timeout +} + +func (e *goEngine) rackMarkLost(conn *GoConn) bool { + if !conn.rackAdvanced { + return false + } + conn.rackAdvanced = false + timeout := e.rackDetectLoss(conn) + if timeout == 0 { + return false + } + conn.reorderDeadline = e.now() + (timeout+goRackTimeoutMinMicros)*int64(time.Microsecond) + e.rearmTimer(conn) + return true +} + +func (e *goEngine) expireReorder(conn *GoConn) { + conn.reorderDeadline = 0 + if !conn.hasOutstandingData() { + e.rearmTimer(conn) + return + } + e.drainDescriptors(conn) + e.summarizeFlight(conn) + priorInFlight := conn.flight.inFlight() + priorLost := conn.lost + e.rackDetectLoss(conn) + if priorInFlight != conn.flight.inFlight() { + if conn.congestionState != goCongestionRecovery { + e.enterRecovery(conn) + if conn.congestion.congControl == nil { + e.reduceWindow(conn, 1, conn.lost-priorLost, 0) + } + } + e.xmitRetransmitQueue(conn) + conn.publishPermit(conn.sendUnacked.Load()) + } + if conn.retransmitDeadline == 0 { + e.rearmRetransmit(conn) + } else { + e.rearmTimer(conn) + } +} + +func (e *goEngine) rackUpdateReorderWindow(conn *GoConn, sample *goRateSample) { + if !conn.sackPermitted || !sample.valid { + return + } + if int32(sample.priorDelivered-conn.rackLastDelivered) < 0 { + conn.rackDSACKSeen = false + } + if conn.rackDSACKSeen { + conn.reorderWindowSteps = min(0xff, conn.reorderWindowSteps+1) + conn.rackDSACKSeen = false + conn.rackLastDelivered = conn.delivered + conn.reorderWindowPersist = goRackRecoveryThreshold + } else if conn.reorderWindowPersist == 0 { + conn.reorderWindowSteps = 1 + } +} + +func (e *goEngine) identifyPacketLoss(conn *GoConn, flags goAckFlags) { + entries := conn.scoreboard.entries + if len(entries) == 0 { + return + } + if !conn.sackPermitted { + if (conn.congestionState < goCongestionRecovery && conn.renoSacked >= conn.reordering) || (conn.congestionState == goCongestionRecovery && flags&goAckUnackedAdvanced != 0) { + if entries[0].flags&goDescriptorLost == 0 { + e.markLost(conn, &entries[0]) + } + } + } else { + e.rackMarkLost(conn) + } +} + +func (e *goEngine) timeToRecover(conn *GoConn) bool { + return conn.flight.lostOut != 0 +} + +func (e *goEngine) processLoss(conn *GoConn, flags *goAckFlags, numDupack uint32) uint8 { + unacked := conn.sendUnacked.Load() + sent := conn.sentTail.Load() + recovered := unacked >= conn.highSeq + if *flags&goAckUnackedAdvanced != 0 && e.tryUndoLoss(conn, *flags, false) { + return goRexmitNone + } + if conn.frto { + if *flags&goAckOriginalSacked != 0 && e.tryUndoLoss(conn, *flags, true) { + return goRexmitNone + } + if sent > conn.highSeq { + if *flags&goAckDataSacked != 0 || numDupack > 0 { + conn.frto = false + } + } else if *flags&goAckUnackedAdvanced != 0 && !recovered { + conn.highSeq = sent + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + if conn.hasDataWaiting() && windowEnd > sent { + return goRexmitNew + } + conn.frto = false + } + } + if recovered { + e.tryUndoRecovery(conn, *flags) + return goRexmitNone + } + if !conn.sackPermitted { + if sent > conn.highSeq && numDupack > 0 { + e.addRenoSack(conn, numDupack) + } else if *flags&goAckUnackedAdvanced != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + } + return goRexmitLost +} + +func (e *goEngine) fastretransAlert(conn *GoConn, priorUnacked uint64, numDupack uint32, flags *goAckFlags) uint8 { + unacked := conn.sendUnacked.Load() + if conn.flight.packetsOut == 0 && conn.renoSacked != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + if *flags&goAckReneging != 0 && *flags&goAckUnackedAdvanced != 0 { + delay := max(int64(conn.smoothedRoundTrip)/2, int64(10*time.Millisecond/time.Microsecond)) + conn.retransmitDeadline = e.now() + delay*int64(time.Microsecond) + e.rearmTimer(conn) + *flags |= goAckKeepTimer + return goRexmitNone + } + if conn.congestionState == goCongestionOpen { + conn.retransmitStamp = 0 + } else if unacked >= conn.highSeq { + switch conn.congestionState { + case goCongestionCWR: + if unacked != conn.highSeq { + e.endWindowReduction(conn) + conn.setCAState(goCongestionOpen) + } + case goCongestionRecovery: + if !conn.sackPermitted { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + if e.tryUndoRecovery(conn, *flags) { + return goRexmitNone + } + e.endWindowReduction(conn) + } + } + switch conn.congestionState { + case goCongestionRecovery: + if *flags&goAckUnackedAdvanced == 0 { + if !conn.sackPermitted { + e.addRenoSack(conn, numDupack) + } + } else if e.tryUndoPartial(conn, *flags, priorUnacked) { + return goRexmitNone + } + if e.tryUndoDSACK(conn) { + e.tryToOpen(conn) + } + e.identifyPacketLoss(conn, *flags) + if conn.congestionState != goCongestionRecovery { + if !e.timeToRecover(conn) { + return goRexmitNone + } + e.enterRecovery(conn) + } + return goRexmitLost + case goCongestionLoss: + rexmit := e.processLoss(conn, flags, numDupack) + e.identifyPacketLoss(conn, *flags) + if conn.congestionState != goCongestionOpen { + return rexmit + } + } + if !conn.sackPermitted { + if *flags&goAckUnackedAdvanced != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + e.addRenoSack(conn, numDupack) + } + if conn.congestionState <= goCongestionDisorder { + e.tryUndoDSACK(conn) + } + e.identifyPacketLoss(conn, *flags) + if !e.timeToRecover(conn) { + e.tryToOpen(conn) + return goRexmitNone + } + e.enterRecovery(conn) + return goRexmitLost +} + +func (e *goEngine) xmitRecovery(conn *GoConn, rexmit uint8) { + if rexmit == goRexmitNone || rexmit == goRexmitNew { + return + } + e.xmitRetransmitQueue(conn) +} + +func (e *goEngine) xmitRetransmitQueue(conn *GoConn) { + if conn.flight.packetsOut == 0 { + return + } + start := conn.sendUnacked.Load() + sent := 0 + for index := 0; index < len(conn.scoreboard.entries); index++ { + descriptor := &conn.scoreboard.entries[index] + end := descriptor.endOffset + if conn.congestionWindow <= conn.flight.inFlight() || conn.flight.retransmitOut >= conn.flight.lostOut { + return + } + if descriptor.flags&(goDescriptorSacked|goDescriptorRetransmitOut) != 0 || descriptor.flags&goDescriptorLost == 0 { + start = end + continue + } + if conn.congestion.pacing { + stamp := conn.pacingStamp.Load() + if stamp > e.now()+goWheelTick { + conn.pacingDeadline = stamp - goWheelTick + e.rearmTimer(conn) + return + } + } + if e.transmitRetransmit(conn, start, int(end-start), false) == 0 { + return + } + sent++ + if sent == goRecoveryBurst { + return + } + start = conn.scoreboard.entries[index].endOffset + } +} + +func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int, probe bool) int { + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + limit := min(conn.transmittedTail.Load(), windowEnd) + if conn.peerWindow == 0 || offset >= limit { + return 0 + } + length = int(min(uint64(length), uint64(conn.effectiveMSS), limit-offset)) + for _, descriptor := range conn.scoreboard.entries { + if descriptor.endOffset > offset { + length = min(length, int(descriptor.endOffset-offset)) + break + } + } + segment := goSegment{offset: offset, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} + if !e.writeConnControl(conn, &segment) { + return 0 + } + conn.totalRetransmits++ + conn.advancePacing(e.now(), length) + end := offset + uint64(length) + marked := e.markRetransmitted(conn, offset, end, probe) + if !probe && marked > 0 { + conn.flight.retransmitOut += marked + if conn.inWindowReduction() { + conn.reductionRetransmits += marked + } + if conn.undoRetransmits < 0 { + conn.undoRetransmits = 0 + } + conn.undoRetransmits += int32(marked) + } + if conn.retransmitStamp == 0 { + conn.retransmitStamp = goTimestampAt(e.now()) + } + return length +} + +func (e *goEngine) markRetransmitted(conn *GoConn, start uint64, end uint64, probe bool) uint32 { + previous := conn.sendUnacked.Load() + stamp := conn.stamp(e.now()) + marked := uint32(0) + appLimited := conn.appLimited.Load() != 0 + scoreboard := &conn.scoreboard + for index := 0; index < len(scoreboard.entries); index++ { + descriptor := &scoreboard.entries[index] + descriptorEnd := descriptor.endOffset + if previous >= end { + break + } + if descriptorEnd <= start { + previous = descriptorEnd + continue + } + if previous < start { + flags := descriptor.flags + scoreboard.split(index, start) + e.accountSplit(conn, flags) + index++ + descriptor = &scoreboard.entries[index] + } + if end < descriptorEnd { + flags := descriptor.flags + scoreboard.split(index, end) + e.accountSplit(conn, flags) + descriptor = &scoreboard.entries[index] + descriptorEnd = end + } + if descriptor.flags&goDescriptorRetransmitted == 0 { + conn.flight.retransmitted++ + } + descriptor.flags |= goDescriptorRetransmitted + descriptor.flags &^= goDescriptorDropped | goDescriptorNoSample + if !probe && descriptor.flags&goDescriptorRetransmitOut == 0 { + descriptor.flags |= goDescriptorRetransmitOut + marked++ + } + descriptor.sentAt = stamp + descriptor.delivered = conn.delivered + descriptor.deliveredStamp = conn.deliveredStamp + descriptor.firstSentStamp = conn.firstSentStamp.Load() + descriptor.flags |= goDescriptorRated + if appLimited { + descriptor.flags |= goDescriptorAppLimited + } else { + descriptor.flags &^= goDescriptorAppLimited + } + previous = descriptorEnd + } + return marked +} + +func (c *GoConn) publishPermit(ackOffset uint64) { + unacked := c.sendUnacked.Load() + limit := max(c.flight.packetsEnd, unacked) + inFlight := c.flight.inFlight() + if c.congestionWindow > inFlight { + limit += uint64(c.congestionWindow-inFlight) * uint64(c.effectiveMSS) + } + windowEnd := ackOffset + c.peerWindow + c.permitWindowBound.Store(limit <= windowEnd) + permit := max(min(limit, windowEnd), unacked) + c.sendPacketPermit.Store(c.packetCreditBase.Load() + c.congestionWindow + c.flight.leftOut() - c.flight.retransmitOut) + c.sendPermit.Store(permit) + c.wakeTransmitter() + c.wakeWriter() +} + +func (c *GoConn) gsoMaxSize() uint32 { + mss := uint32(c.effectiveMSS) + if c.engine.platformIO.transmitSegmentOffload() { + return max(goGSOMaxPayload/mss*mss, mss) + } + return max(0xffff/mss*mss, mss) +} + +func (e *goEngine) updateFrameLimit(conn *GoConn) { + rate := conn.pacingRate.Load() + if rate == 0 { + conn.frameLimit.Store(0) + return + } + bytes := rate >> goPacingShift + shift := conn.minRoundTripMicros() >> goTSORoundTripShift + gsoMax := uint64(conn.gsoMaxSize()) + if shift < 64 { + bytes += gsoMax >> shift + } + bytes = min(bytes, gsoMax) + minSegments := uint32(goMinTSOSegments) + if conn.congestion.minTSOSegments != nil { + minSegments = conn.congestion.minTSOSegments(conn) + } + mss := uint64(conn.effectiveMSS) + segments := max(uint32(bytes/mss), minSegments) + conn.frameLimit.Store(uint32(min(uint64(segments)*mss, gsoMax))) +} + +func (e *goEngine) handlePacingRequest(conn *GoConn) { + if conn.connState.Load() >= goConnStateAborted { + return + } + conn.pacingDeadline = conn.pacingRequest.Load() + e.rearmTimer(conn) +} + +func (e *goEngine) expirePacing(conn *GoConn) { + conn.pacingDeadline = 0 + e.rearmTimer(conn) + if conn.flight.lostOut > conn.flight.retransmitOut && conn.congestionWindow > conn.flight.inFlight() { + e.xmitRetransmitQueue(conn) + conn.publishPermit(conn.sendUnacked.Load()) + return + } + conn.wakeTransmitter() +} + +func (e *goEngine) armProbe(conn *GoConn) { + if conn.probeAttempts >= goProbeAttempts || !conn.hasOutstanding() || !conn.sackPermitted || (conn.congestionState != goCongestionOpen && conn.congestionState != goCongestionCWR) { + conn.probeDeadline = 0 + return + } + timeout := int32(goProbeNoSampleMicros) + if conn.smoothedRoundTrip > 0 { + timeout = max(2*conn.smoothedRoundTrip, goProbeFloorMicros) + } + if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS) { + timeout += goProbeDelayedAck + } + if timeout >= conn.retransmitTimeout { + conn.probeDeadline = 0 + return + } + conn.probeDeadline = e.now() + int64(timeout)*int64(time.Microsecond) +} + +func (e *goEngine) expireProbe(conn *GoConn, now int64) { + conn.probeDeadline = 0 + if !conn.hasOutstanding() { + e.rearmTimer(conn) + return + } + e.drainDescriptors(conn) + if conn.probeHighSeq != 0 { + deadline := conn.retransmitDeadline + if len(conn.scoreboard.entries) > 0 { + elapsed := goStampMicros(conn.stamp(now) - conn.scoreboard.entries[0].sentAt) + deadline = now + max(int64(conn.retransmitTimeout)-elapsed, 1)*int64(time.Microsecond) + } + if deadline == 0 { + deadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + } + conn.retransmitDeadline = deadline + e.rearmTimer(conn) + return + } + conn.probeAttempts++ + e.summarizeFlight(conn) + sent := conn.sentTail.Load() + committed := min(sent, conn.transmittedTail.Load()) + unacked := conn.sendUnacked.Load() + pending := conn.bufferedTail.Load() - sent + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + switch { + case pending > 0 && committed == sent && windowEnd > sent && conn.writerActive.Load() == 0: + length := min(pending, uint64(conn.effectiveMSS), windowEnd-sent) + conn.probeRetransmitted = false + conn.probeHighSeq = sent + length + permit := conn.sendPermit.Load() + if packets := conn.dataSegmentsOut.Load(); int32(conn.sendPacketPermit.Load()-packets) <= 0 { + conn.sendPacketPermit.Store(packets + 1) + } + if permit < sent+length { + conn.sendPermit.Store(sent + length) + } + conn.wakeTransmitter() + case committed > unacked: + entries := conn.scoreboard.entries + for len(entries) > 0 && entries[len(entries)-1].endOffset > committed { + entries = entries[:len(entries)-1] + } + start := unacked + end := committed + if len(entries) > 0 { + last := len(entries) - 1 + end = entries[last].endOffset + start = conn.scoreboard.startOf(last, unacked) + if end < committed { + start = end + end = committed + } + } + if end-start > uint64(conn.effectiveMSS) { + start = end - uint64(conn.effectiveMSS) + } + if e.transmitRetransmit(conn, start, int(end-start), true) > 0 { + conn.probeRetransmitted = true + conn.probeHighSeq = committed + } + case conn.finSent && !conn.finAcked: + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + if e.writeConnControl(conn, &segment) { + conn.probeRetransmitted = true + conn.probeHighSeq = conn.sendNext() + } + } + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.armProbe(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { + if !conn.hasOutstanding() { + conn.retransmitDeadline = 0 + conn.retransmitArmed.Store(false) + e.rearmTimer(conn) + return + } + if conn.persistNeeded() { + e.updatePersist(conn) + return + } + conn.probeDeadline = 0 + conn.probeAttempts = 0 + conn.reorderDeadline = 0 + if conn.hasOutstandingData() && conn.sendUnacked.Load() >= conn.transmittedTail.Load() { + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + return + } + if conn.retransmitAttempts+1 >= goRetransmitLimit && conn.persistDeadline == 0 { + conn.retransmitDeadline = 0 + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: retransmit limit"), goDeathImmediate) + return + } + e.drainDescriptors(conn) + e.enterLoss(conn) + conn.retransmitAttempts++ + unacked := conn.sendUnacked.Load() + if conn.hasOutstandingData() { + length := int(min(conn.sentTail.Load()-unacked, uint64(conn.effectiveMSS))) + e.transmitRetransmit(conn, unacked, length, false) + } else { + e.retransmitFrom(conn, unacked) + } + conn.retransmitTimeout = min(conn.retransmitTimeout*2, goRTOMaxMicros) + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + e.updateFrameLimit(conn) + conn.publishPermit(unacked) +} diff --git a/stack_go_tcp_bbr.go b/stack_go_tcp_bbr.go new file mode 100644 index 00000000..a034b4f3 --- /dev/null +++ b/stack_go_tcp_bbr.go @@ -0,0 +1,598 @@ +package tun + +import ( + "math/bits" + "math/rand/v2" + "time" +) + +// Mirrors net/ipv4/tcp_bbr.c with HZ = 1000. + +const ( + goBBRBandwidthScale = 24 + goBBRBandwidthUnit = 1 << goBBRBandwidthScale + goBBRScale = 8 + goBBRUnit = 1 << goBBRScale + goBBRCycleLength = 8 +) + +const ( + goBBRModeStartup uint8 = iota + goBBRModeDrain + goBBRModeProbeBandwidth + goBBRModeProbeRoundTrip +) + +const ( + goBBRBandwidthRounds = goBBRCycleLength + 2 + goBBRMinRoundTripWindowMs = 10 * 1000 + goBBRProbeRoundTripModeMs = 200 + goBBRMinTSORate = 1200000 + goBBRPacingMarginPercent = 1 + goBBRHighGain = goBBRUnit*2885/1000 + 1 + goBBRDrainGain = goBBRUnit * 1000 / 2885 + goBBRWindowGain = goBBRUnit * 2 + goBBRCycleRandom = 7 + goBBRWindowMinTarget = 4 + goBBRFullBandwidthThresh = goBBRUnit * 5 / 4 + goBBRFullBandwidthCount = 3 + goBBRLongTermMinRounds = 4 + goBBRLongTermLossThresh = 50 + goBBRLongTermBandwidthRatio = goBBRUnit / 8 + goBBRLongTermBandwidthDiff = 4000 / 8 + goBBRLongTermMaxRounds = 48 + goBBRExtraAckedGain = goBBRUnit + goBBRExtraAckedWindowRounds = 5 + goBBRAckEpochResetThresh = 1 << 20 + goBBRExtraAckedMaxMicros = 100 * 1000 +) + +var goBBRPacingGains = [goBBRCycleLength]uint32{ + goBBRUnit * 5 / 4, + goBBRUnit * 3 / 4, + goBBRUnit, goBBRUnit, goBBRUnit, + goBBRUnit, goBBRUnit, goBBRUnit, +} + +type goBBRState struct { + minRoundTripMicros uint32 + minRoundTripStamp uint32 + probeRoundTripDone uint32 + bandwidth goMinMax + roundTripCount uint32 + nextRoundDelivered uint32 + cycleStamp int64 + mode uint8 + previousCAState uint8 + packetConservation bool + roundStart bool + idleRestart bool + probeRoundTripRoundDone bool + longTermSampling bool + longTermRounds uint32 + longTermUseBandwidth bool + longTermBandwidth uint32 + longTermLastDelivered uint32 + longTermLastStamp int64 + longTermLastLost uint32 + pacingGain uint32 + windowGain uint32 + fullBandwidthReached bool + fullBandwidthCount uint32 + cycleIndex uint32 + seenRoundTrip bool + priorWindow uint32 + fullBandwidth uint32 + ackEpochStamp int64 + extraAcked [2]uint32 + ackEpochAcked uint32 + extraAckedWindowRounds uint32 + extraAckedWindowIndex int +} + +func goBBRJiffies(c *GoConn) uint32 { + return uint32(c.engine.now() / int64(time.Millisecond)) +} + +func goBBRAfter(a uint32, b uint32) bool { + return int32(a-b) > 0 +} + +func (s *goBBRState) maxBandwidth() uint32 { + return s.bandwidth.get() +} + +func (s *goBBRState) currentBandwidth() uint32 { + if s.longTermUseBandwidth { + return s.longTermBandwidth + } + return s.maxBandwidth() +} + +func (s *goBBRState) extraAckedMax() uint32 { + return max(s.extraAcked[0], s.extraAcked[1]) +} + +func goBBRRateBytesPerSecond(c *GoConn, rate uint64, gain uint32) uint64 { + rate *= uint64(c.effectiveMSS) + rate *= uint64(gain) + rate >>= goBBRScale + rate *= uint64(time.Second/time.Microsecond) / 100 * (100 - goBBRPacingMarginPercent) + return rate >> goBBRBandwidthScale +} + +func goBBRInitPacingRateFromRoundTrip(c *GoConn, s *goBBRState) { + roundTrip := uint64(time.Millisecond / time.Microsecond) + if c.smoothedRoundTrip > 0 { + roundTrip = uint64(max(c.smoothedRoundTrip, 1)) + s.seenRoundTrip = true + } + bandwidth := uint64(c.congestionWindow) * goBBRBandwidthUnit / roundTrip + c.pacingRate.Store(goBBRRateBytesPerSecond(c, bandwidth, goBBRHighGain)) +} + +func goBBRSetPacingRate(c *GoConn, s *goBBRState, bandwidth uint32, gain uint32) { + rate := goBBRRateBytesPerSecond(c, uint64(bandwidth), gain) + if !s.seenRoundTrip && c.smoothedRoundTrip > 0 { + goBBRInitPacingRateFromRoundTrip(c, s) + } + if s.fullBandwidthReached || rate > c.pacingRate.Load() { + c.pacingRate.Store(rate) + } +} + +func goBBRMinTSOSegments(c *GoConn) uint32 { + if c.pacingRate.Load() < goBBRMinTSORate>>3 { + return 1 + } + return 2 +} + +func goBBRTSOSegmentsGoal(c *GoConn) uint32 { + bytes := min(c.pacingRate.Load()>>goPacingShift, uint64(0xffff-1-goHeaderScratchSize)) + segments := max(uint32(bytes/uint64(c.effectiveMSS)), goBBRMinTSOSegments(c)) + return min(segments, 0x7f) +} + +func (s *goBBRState) saveWindow(c *GoConn) { + if s.previousCAState < goCongestionRecovery && s.mode != goBBRModeProbeRoundTrip { + s.priorWindow = c.congestionWindow + } else { + s.priorWindow = max(s.priorWindow, c.congestionWindow) + } +} + +func goBBRTxStart(c *GoConn, _ int64) { + s := c.congestionPrivate.(*goBBRState) + if c.appLimited.Load() != 0 { + s.idleRestart = true + s.ackEpochStamp = c.engine.now() + s.ackEpochAcked = 0 + switch s.mode { + case goBBRModeProbeBandwidth: + goBBRSetPacingRate(c, s, s.currentBandwidth(), goBBRUnit) + case goBBRModeProbeRoundTrip: + s.checkProbeRoundTripDone(c) + } + } +} + +func (s *goBBRState) bandwidthDelayProduct(bandwidth uint32, gain uint32) uint32 { + if s.minRoundTripMicros == ^uint32(0) { + return goInitialWindow + } + w := uint64(bandwidth) * uint64(s.minRoundTripMicros) + return uint32((((w * uint64(gain)) >> goBBRScale) + goBBRBandwidthUnit - 1) / goBBRBandwidthUnit) +} + +func (s *goBBRState) quantizationBudget(c *GoConn, window uint32) uint32 { + window += 3 * goBBRTSOSegmentsGoal(c) + window = (window + 1) &^ 1 + if s.mode == goBBRModeProbeBandwidth && s.cycleIndex == 0 { + window += 2 + } + return window +} + +func (s *goBBRState) inflight(c *GoConn, bandwidth uint32, gain uint32) uint32 { + return s.quantizationBudget(c, s.bandwidthDelayProduct(bandwidth, gain)) +} + +func (s *goBBRState) packetsInNetAtDeparture(c *GoConn, inflightNow uint32) uint32 { + now := c.engine.now() + departure := max(c.pacingStamp.Load(), now) + intervalMicros := uint64(departure-now) / uint64(time.Microsecond) + intervalDelivered := uint32(uint64(s.currentBandwidth()) * intervalMicros >> goBBRBandwidthScale) + inflightAtDeparture := inflightNow + if s.pacingGain > goBBRUnit { + inflightAtDeparture += goBBRTSOSegmentsGoal(c) + } + if intervalDelivered >= inflightAtDeparture { + return 0 + } + return inflightAtDeparture - intervalDelivered +} + +func (s *goBBRState) ackAggregationWindow() uint32 { + if goBBRExtraAckedGain == 0 || !s.fullBandwidthReached { + return 0 + } + maxAggregation := uint32(uint64(s.currentBandwidth()) * goBBRExtraAckedMaxMicros / goBBRBandwidthUnit) + aggregation := (goBBRExtraAckedGain * s.extraAckedMax()) >> goBBRScale + return min(aggregation, maxAggregation) +} + +func (s *goBBRState) setWindowToRecoverOrRestore(c *GoConn, sample *goRateSample, acked uint32) (uint32, bool) { + previousState := s.previousCAState + state := c.congestionState + window := c.congestionWindow + inFlight := c.flight.inFlight() + if sample.losses > 0 { + window = uint32(max(int64(window)-int64(sample.losses), 1)) + } + if state == goCongestionRecovery && previousState != goCongestionRecovery { + s.packetConservation = true + s.nextRoundDelivered = c.delivered + window = inFlight + acked + } else if previousState >= goCongestionRecovery && state < goCongestionRecovery { + window = max(window, s.priorWindow) + s.packetConservation = false + } + s.previousCAState = state + if s.packetConservation { + return max(window, inFlight+acked), true + } + return window, false +} + +func (s *goBBRState) setWindow(c *GoConn, sample *goRateSample, acked uint32, bandwidth uint32, gain uint32) { + window := c.congestionWindow + if acked != 0 { + var conserving bool + window, conserving = s.setWindowToRecoverOrRestore(c, sample, acked) + if !conserving { + target := s.bandwidthDelayProduct(bandwidth, gain) + target += s.ackAggregationWindow() + target = s.quantizationBudget(c, target) + if s.fullBandwidthReached { + window = min(window+acked, target) + } else if window < target || c.delivered < goInitialWindow { + window += acked + } + window = max(window, goBBRWindowMinTarget) + } + } + c.setCongestionWindow(window) + if s.mode == goBBRModeProbeRoundTrip { + c.setCongestionWindow(min(c.congestionWindow, goBBRWindowMinTarget)) + } +} + +func (s *goBBRState) isNextCyclePhase(c *GoConn, sample *goRateSample) bool { + fullLength := c.deliveredTime-s.cycleStamp > int64(s.minRoundTripMicros)*int64(time.Microsecond) + if s.pacingGain == goBBRUnit { + return fullLength + } + inflight := s.packetsInNetAtDeparture(c, sample.priorInFlight) + bandwidth := s.maxBandwidth() + if s.pacingGain > goBBRUnit { + return fullLength && (sample.losses > 0 || inflight >= s.inflight(c, bandwidth, s.pacingGain)) + } + return fullLength || inflight <= s.inflight(c, bandwidth, goBBRUnit) +} + +func (s *goBBRState) advanceCyclePhase(c *GoConn) { + s.cycleIndex = (s.cycleIndex + 1) & (goBBRCycleLength - 1) + s.cycleStamp = c.deliveredTime +} + +func (s *goBBRState) updateCyclePhase(c *GoConn, sample *goRateSample) { + if s.mode == goBBRModeProbeBandwidth && s.isNextCyclePhase(c, sample) { + s.advanceCyclePhase(c) + } +} + +func (s *goBBRState) resetProbeBandwidthMode(c *GoConn) { + s.mode = goBBRModeProbeBandwidth + s.cycleIndex = goBBRCycleLength - 1 - rand.Uint32N(goBBRCycleRandom) + s.advanceCyclePhase(c) +} + +func (s *goBBRState) resetMode(c *GoConn) { + if !s.fullBandwidthReached { + s.mode = goBBRModeStartup + } else { + s.resetProbeBandwidthMode(c) + } +} + +func (s *goBBRState) resetLongTermInterval(c *GoConn) { + s.longTermLastStamp = c.deliveredTime + s.longTermLastDelivered = c.delivered + s.longTermLastLost = c.lost + s.longTermRounds = 0 +} + +func (s *goBBRState) resetLongTermSampling(c *GoConn) { + s.longTermBandwidth = 0 + s.longTermUseBandwidth = false + s.longTermSampling = false + s.resetLongTermInterval(c) +} + +func (s *goBBRState) longTermIntervalDone(c *GoConn, bandwidth uint32) { + if s.longTermBandwidth != 0 { + var diff uint32 + if bandwidth > s.longTermBandwidth { + diff = bandwidth - s.longTermBandwidth + } else { + diff = s.longTermBandwidth - bandwidth + } + if uint64(diff)*goBBRUnit <= uint64(goBBRLongTermBandwidthRatio)*uint64(s.longTermBandwidth) || + goBBRRateBytesPerSecond(c, uint64(diff), goBBRUnit) <= goBBRLongTermBandwidthDiff { + s.longTermBandwidth = (bandwidth + s.longTermBandwidth) >> 1 + s.longTermUseBandwidth = true + s.pacingGain = goBBRUnit + s.longTermRounds = 0 + return + } + } + s.longTermBandwidth = bandwidth + s.resetLongTermInterval(c) +} + +func (s *goBBRState) longTermSample(c *GoConn, sample *goRateSample) { + if s.longTermUseBandwidth { + if s.mode == goBBRModeProbeBandwidth && s.roundStart { + s.longTermRounds++ + if s.longTermRounds >= goBBRLongTermMaxRounds { + s.resetLongTermSampling(c) + s.resetProbeBandwidthMode(c) + } + } + return + } + if !s.longTermSampling { + if sample.losses == 0 { + return + } + s.resetLongTermInterval(c) + s.longTermSampling = true + } + if sample.appLimited { + s.resetLongTermSampling(c) + return + } + if s.roundStart { + s.longTermRounds++ + } + if s.longTermRounds < goBBRLongTermMinRounds { + return + } + if s.longTermRounds > 4*goBBRLongTermMinRounds { + s.resetLongTermSampling(c) + return + } + if sample.losses == 0 { + return + } + lost := c.lost - s.longTermLastLost + delivered := c.delivered - s.longTermLastDelivered + if delivered == 0 || uint64(lost)<= int64(^uint32(0))/1000 { + s.resetLongTermSampling(c) + return + } + bandwidth := uint64(delivered) * goBBRBandwidthUnit / (uint64(t) * 1000) + s.longTermIntervalDone(c, uint32(bandwidth)) +} + +func (s *goBBRState) updateBandwidth(c *GoConn, sample *goRateSample) { + s.roundStart = false + if sample.delivered < 0 || sample.intervalMicros <= 0 { + return + } + if int32(sample.priorDelivered-s.nextRoundDelivered) >= 0 { + s.nextRoundDelivered = c.delivered + s.roundTripCount++ + s.roundStart = true + s.packetConservation = false + } + s.longTermSample(c, sample) + bandwidth := uint64(sample.delivered) * goBBRBandwidthUnit / uint64(sample.intervalMicros) + if !sample.appLimited || bandwidth >= uint64(s.maxBandwidth()) { + s.bandwidth.runningMax(goBBRBandwidthRounds, s.roundTripCount, uint32(min(bandwidth, uint64(^uint32(0))))) + } +} + +func (s *goBBRState) updateAckAggregation(c *GoConn, sample *goRateSample) { + if goBBRExtraAckedGain == 0 || sample.ackedSacked == 0 || sample.delivered < 0 || sample.intervalMicros <= 0 { + return + } + if s.roundStart { + s.extraAckedWindowRounds = min(0x1f, s.extraAckedWindowRounds+1) + if s.extraAckedWindowRounds >= goBBRExtraAckedWindowRounds { + s.extraAckedWindowRounds = 0 + s.extraAckedWindowIndex = 1 - s.extraAckedWindowIndex + s.extraAcked[s.extraAckedWindowIndex] = 0 + } + } + epochMicros := (c.deliveredTime - s.ackEpochStamp) / int64(time.Microsecond) + high, low := bits.Mul64(uint64(s.currentBandwidth()), uint64(max(epochMicros, 0))) + expectedAcked := uint32(goBBRAckEpochResetThresh) + if high == 0 && low < uint64(goBBRAckEpochResetThresh)*goBBRBandwidthUnit { + expectedAcked = uint32(low / goBBRBandwidthUnit) + } + if s.ackEpochAcked <= expectedAcked || s.ackEpochAcked+sample.ackedSacked >= goBBRAckEpochResetThresh { + s.ackEpochAcked = 0 + s.ackEpochStamp = c.deliveredTime + expectedAcked = 0 + } + s.ackEpochAcked = min(0xfffff, s.ackEpochAcked+sample.ackedSacked) + extraAcked := min(s.ackEpochAcked-expectedAcked, c.congestionWindow) + if extraAcked > s.extraAcked[s.extraAckedWindowIndex] { + s.extraAcked[s.extraAckedWindowIndex] = extraAcked + } +} + +func (s *goBBRState) checkFullBandwidthReached(sample *goRateSample) { + if s.fullBandwidthReached || !s.roundStart || sample.appLimited { + return + } + threshold := uint32(uint64(s.fullBandwidth) * goBBRFullBandwidthThresh >> goBBRScale) + if s.maxBandwidth() >= threshold { + s.fullBandwidth = s.maxBandwidth() + s.fullBandwidthCount = 0 + return + } + s.fullBandwidthCount++ + s.fullBandwidthReached = s.fullBandwidthCount >= goBBRFullBandwidthCount +} + +func (s *goBBRState) checkDrain(c *GoConn) { + if s.mode == goBBRModeStartup && s.fullBandwidthReached { + s.mode = goBBRModeDrain + c.slowStartThreshold = s.inflight(c, s.maxBandwidth(), goBBRUnit) + } + if s.mode == goBBRModeDrain && s.packetsInNetAtDeparture(c, c.flight.inFlight()) <= s.inflight(c, s.maxBandwidth(), goBBRUnit) { + s.resetProbeBandwidthMode(c) + } +} + +func (s *goBBRState) checkProbeRoundTripDone(c *GoConn) { + if s.probeRoundTripDone == 0 || !goBBRAfter(goBBRJiffies(c), s.probeRoundTripDone) { + return + } + s.minRoundTripStamp = goBBRJiffies(c) + c.setCongestionWindow(max(c.congestionWindow, s.priorWindow)) + s.resetMode(c) +} + +func (s *goBBRState) updateMinRoundTrip(c *GoConn, sample *goRateSample) { + now := goBBRJiffies(c) + filterExpired := goBBRAfter(now, s.minRoundTripStamp+goBBRMinRoundTripWindowMs) + if sample.roundTripMicros >= 0 && (uint32(sample.roundTripMicros) < s.minRoundTripMicros || (filterExpired && !sample.ackDelayed)) { + s.minRoundTripMicros = uint32(sample.roundTripMicros) + s.minRoundTripStamp = now + } + if goBBRProbeRoundTripModeMs > 0 && filterExpired && !s.idleRestart && s.mode != goBBRModeProbeRoundTrip { + s.mode = goBBRModeProbeRoundTrip + s.saveWindow(c) + s.probeRoundTripDone = 0 + } + if s.mode == goBBRModeProbeRoundTrip { + inFlight := c.flight.inFlight() + c.appLimited.Store(max(c.delivered+inFlight, 1)) + if s.probeRoundTripDone == 0 && inFlight <= goBBRWindowMinTarget { + s.probeRoundTripDone = now + goBBRProbeRoundTripModeMs + s.probeRoundTripRoundDone = false + s.nextRoundDelivered = c.delivered + } else if s.probeRoundTripDone != 0 { + if s.roundStart { + s.probeRoundTripRoundDone = true + } + if s.probeRoundTripRoundDone { + s.checkProbeRoundTripDone(c) + } + } + } + if sample.delivered > 0 { + s.idleRestart = false + } +} + +func (s *goBBRState) updateGains() { + switch s.mode { + case goBBRModeStartup: + s.pacingGain = goBBRHighGain + s.windowGain = goBBRHighGain + case goBBRModeDrain: + s.pacingGain = goBBRDrainGain + s.windowGain = goBBRHighGain + case goBBRModeProbeBandwidth: + if s.longTermUseBandwidth { + s.pacingGain = goBBRUnit + } else { + s.pacingGain = goBBRPacingGains[s.cycleIndex] + } + s.windowGain = goBBRWindowGain + case goBBRModeProbeRoundTrip: + s.pacingGain = goBBRUnit + s.windowGain = goBBRUnit + } +} + +func goBBRMain(c *GoConn, _ goAckFlags, sample *goRateSample) { + s := c.congestionPrivate.(*goBBRState) + s.updateBandwidth(c, sample) + s.updateAckAggregation(c, sample) + s.updateCyclePhase(c, sample) + s.checkFullBandwidthReached(sample) + s.checkDrain(c) + s.updateMinRoundTrip(c, sample) + s.updateGains() + bandwidth := s.currentBandwidth() + goBBRSetPacingRate(c, s, bandwidth, s.pacingGain) + s.setWindow(c, sample, sample.ackedSacked, bandwidth, s.windowGain) +} + +func goBBRInit(c *GoConn) { + s := new(goBBRState) + c.congestionPrivate = s + c.slowStartThreshold = goInfiniteSlowStartThreshold + s.nextRoundDelivered = c.delivered + s.previousCAState = goCongestionOpen + s.minRoundTripMicros = c.minRoundTripMicros() + s.minRoundTripStamp = goBBRJiffies(c) + s.bandwidth.reset(s.roundTripCount, 0) + goBBRInitPacingRateFromRoundTrip(c, s) + s.resetLongTermSampling(c) + s.mode = goBBRModeStartup + s.ackEpochStamp = c.engine.now() +} + +func goBBRUndoWindow(c *GoConn) uint32 { + s := c.congestionPrivate.(*goBBRState) + s.fullBandwidth = 0 + s.fullBandwidthCount = 0 + s.resetLongTermSampling(c) + return c.congestionWindow +} + +func goBBRSlowStartThreshold(c *GoConn) uint32 { + s := c.congestionPrivate.(*goBBRState) + s.saveWindow(c) + return c.slowStartThreshold +} + +func goBBRSetState(c *GoConn, newState uint8) { + if newState == goCongestionLoss { + s := c.congestionPrivate.(*goBBRState) + s.previousCAState = goCongestionLoss + s.fullBandwidth = 0 + s.roundStart = true + s.longTermSample(c, &goRateSample{losses: 1}) + } +} + +var goBBROps = goCongestionOps{ + name: "bbr", + init: goBBRInit, + congControl: goBBRMain, + undoWindow: goBBRUndoWindow, + windowEventTxStart: goBBRTxStart, + slowStartThreshold: goBBRSlowStartThreshold, + minTSOSegments: goBBRMinTSOSegments, + setState: goBBRSetState, + pacing: true, +} + +func init() { + goRegisterCongestionControl(&goBBROps) +} diff --git a/stack_go_tcp_cong.go b/stack_go_tcp_cong.go new file mode 100644 index 00000000..da900d76 --- /dev/null +++ b/stack_go_tcp_cong.go @@ -0,0 +1,707 @@ +package tun + +import ( + "time" + + E "github.com/sagernet/sing/common/exceptions" +) + +// Mirrors net/ipv4/tcp_cong.c and the congestion control core of net/ipv4/tcp_input.c. + +const ( + goCongestionOpen uint8 = iota + goCongestionDisorder + goCongestionCWR + goCongestionRecovery + goCongestionLoss +) + +const ( + goCongestionEventWindowRestart uint8 = iota + goCongestionEventCompleteCWR + goCongestionEventLoss +) + +const ( + goInfiniteSlowStartThreshold = 0x7fffffff + goInitialReordering = 3 + goMaxReordering = 300 + goMinRoundTripWindow = 300 * time.Second + goPacingSlowStartRatio = 200 + goPacingAvoidanceRatio = 120 + goMinTSOSegments = 2 + goTSORoundTripShift = 9 + goPacingShift = 10 + goPacingUnpacedSegments = 10 + goRackRecoveryThreshold = 16 + goRackTimeoutMinMicros = 2000 +) + +type goAckFlags uint32 + +const ( + goAckData goAckFlags = 1 << iota + goAckWindowUpdate + goAckDataAcked + goAckRetransmittedAcked + goAckDataSacked + goAckOriginalSacked + goAckUnackedAdvanced + goAckDSACK + goAckMaybeDelayed + goAckDSACKProbe + goAckReneging + goAckKeepTimer +) + +const ( + goAckNotDuplicate = goAckData | goAckWindowUpdate | goAckDataAcked + goAckAlert = goAckDataSacked | goAckDSACK + goAckForwardProgress = goAckDataAcked | goAckDataSacked +) + +const ( + goRexmitNone uint8 = iota + goRexmitLost + goRexmitNew +) + +type goAckSample struct { + packetsAcked uint32 + roundTripMicros int64 + inFlightBytes uint64 +} + +type goRateSample struct { + valid bool + priorStamp int32 + priorDelivered uint32 + delivered int64 + intervalMicros int64 + sendIntervalMicros int64 + receiveIntervalMicros int64 + roundTripMicros int64 + losses uint32 + ackedSacked uint32 + priorInFlight uint32 + lastEndOffset uint64 + appLimited bool + retransmitted bool + ackDelayed bool +} + +type goFlightSummary struct { + packetsEnd uint64 + packetsOut uint32 + sackedOut uint32 + lostOut uint32 + retransmitOut uint32 + retransmitted uint32 +} + +func (s *goFlightSummary) inFlight() uint32 { + return s.packetsOut - s.sackedOut - s.lostOut + s.retransmitOut +} + +func (s *goFlightSummary) leftOut() uint32 { + return s.sackedOut + s.lostOut +} + +type goCongestionOps struct { + name string + init func(conn *GoConn) + release func(conn *GoConn) + slowStartThreshold func(conn *GoConn) uint32 + congAvoid func(conn *GoConn, acked uint32) + congControl func(conn *GoConn, flags goAckFlags, sample *goRateSample) + undoWindow func(conn *GoConn) uint32 + setState func(conn *GoConn, state uint8) + windowEvent func(conn *GoConn, event uint8) + windowEventTxStart func(conn *GoConn, idleNanos int64) + packetsAcked func(conn *GoConn, sample *goAckSample) + minTSOSegments func(conn *GoConn) uint32 + pacing bool +} + +var goCongestionRegistry = make(map[string]*goCongestionOps) + +const goDefaultCongestionControl = "cubic" + +func goRegisterCongestionControl(ops *goCongestionOps) { + if ops.slowStartThreshold == nil || ops.undoWindow == nil || (ops.congAvoid == nil && ops.congControl == nil) { + panic("go: congestion control " + ops.name + " does not implement required ops") + } + if _, exists := goCongestionRegistry[ops.name]; exists { + panic("go: congestion control " + ops.name + " already registered") + } + goCongestionRegistry[ops.name] = ops +} + +func goLookupCongestionControl(name string) (*goCongestionOps, error) { + if name == "" { + name = goDefaultCongestionControl + } + ops, found := goCongestionRegistry[name] + if !found { + return nil, E.New("go: unknown congestion control: ", name) + } + return ops, nil +} + +func (c *GoConn) initCongestionControl(ops *goCongestionOps) { + c.congestion = ops + c.congestionWindow = goInitialWindow + c.slowStartThreshold = goInfiniteSlowStartThreshold + c.congestionClamp = max(uint32(goMaxCongestionSize/uint64(c.effectiveMSS)), 2) + c.reordering = goInitialReordering + c.reorderWindowSteps = 1 + c.deliveredTime = c.engine.now() + c.undoRetransmits = -1 + c.priorSlowStartThreshold = 0 + c.roundTripMin.reset(0, ^uint32(0)) + if ops.init != nil { + ops.init(c) + } +} + +func (c *GoConn) releaseCongestionControl() { + if c.congestion != nil && c.congestion.release != nil { + c.congestion.release(c) + } + c.congestionPrivate = nil +} + +func (c *GoConn) setCAState(state uint8) { + if c.congestion.setState != nil { + c.congestion.setState(c, state) + } + c.congestionState = state +} + +func (c *GoConn) congestionEvent(event uint8) { + if c.congestion.windowEvent != nil { + c.congestion.windowEvent(c, event) + } +} + +func (c *GoConn) inSlowStart() bool { + return c.congestionWindow < c.slowStartThreshold +} + +func (c *GoConn) inWindowReduction() bool { + return c.congestionState == goCongestionCWR || c.congestionState == goCongestionRecovery +} + +func (c *GoConn) currentSlowStartThreshold() uint32 { + if c.inWindowReduction() { + return c.slowStartThreshold + } + return max(c.slowStartThreshold, c.congestionWindow>>1+c.congestionWindow>>2) +} + +func (c *GoConn) setCongestionWindow(window uint32) { + c.congestionWindow = min(max(window, 1), c.congestionClamp) +} + +func (c *GoConn) isWindowLimited() bool { + if c.windowLimited { + return true + } + if c.inSlowStart() { + return c.congestionWindow < 2*c.maxPacketsOut + } + return false +} + +func (c *GoConn) minRoundTripMicros() uint32 { + return c.roundTripMin.get() +} + +func (c *GoConn) slowStart(acked uint32) uint32 { + window := min(c.congestionWindow+acked, c.slowStartThreshold) + acked -= window - c.congestionWindow + c.setCongestionWindow(window) + return acked +} + +func (c *GoConn) congAvoidAI(w uint32, acked uint32) { + if c.congestionWindowCount >= w { + c.congestionWindowCount = 0 + c.setCongestionWindow(c.congestionWindow + 1) + } + c.congestionWindowCount += acked + if c.congestionWindowCount >= w { + delta := c.congestionWindowCount / w + c.congestionWindowCount -= delta * w + c.setCongestionWindow(c.congestionWindow + delta) + } +} + +func goRenoCongAvoid(c *GoConn, acked uint32) { + if !c.isWindowLimited() { + return + } + if c.inSlowStart() { + acked = c.slowStart(acked) + if acked == 0 { + return + } + } + c.congAvoidAI(c.congestionWindow, acked) +} + +func goRenoSlowStartThreshold(c *GoConn) uint32 { + return max(c.congestionWindow>>1, 2) +} + +func goRenoUndoWindow(c *GoConn) uint32 { + return max(c.congestionWindow, c.priorCongestionWindow) +} + +var goRenoOps = goCongestionOps{ + name: "reno", + slowStartThreshold: goRenoSlowStartThreshold, + congAvoid: goRenoCongAvoid, + undoWindow: goRenoUndoWindow, +} + +func init() { + goRegisterCongestionControl(&goRenoOps) +} + +func (e *goEngine) initUndo(conn *GoConn) { + conn.undoMarker = conn.sendUnacked.Load() + conn.undoRetransmits = int32(conn.flight.retransmitOut) + if conn.probeHighSeq != 0 && conn.probeRetransmitted { + conn.undoRetransmits++ + } + if conn.undoRetransmits == 0 { + conn.undoRetransmits = -1 + } +} + +func (e *goEngine) initWindowReduction(conn *GoConn) { + conn.highSeq = conn.sentTail.Load() + conn.probeHighSeq = 0 + conn.congestionWindowCount = 0 + conn.priorCongestionWindow = conn.congestionWindow + conn.reductionDelivered = 0 + conn.reductionRetransmits = 0 + conn.reductionSegmentsOut = conn.dataSegmentsOut.Load() + conn.slowStartThreshold = conn.congestion.slowStartThreshold(conn) +} + +func (c *GoConn) reductionOut() uint32 { + return c.reductionRetransmits + c.dataSegmentsOut.Load() - c.reductionSegmentsOut +} + +func (e *goEngine) reduceWindow(conn *GoConn, newlyAckedSacked uint32, newlyLost uint32, flags goAckFlags) { + if newlyAckedSacked == 0 || conn.priorCongestionWindow == 0 { + return + } + inFlight := conn.flight.inFlight() + conn.reductionDelivered += newlyAckedSacked + out := conn.reductionOut() + var sendCount int64 + delta := int64(conn.slowStartThreshold) - int64(inFlight) + if delta < 0 { + dividend := uint64(conn.slowStartThreshold)*uint64(conn.reductionDelivered) + uint64(conn.priorCongestionWindow) - 1 + sendCount = int64(dividend/uint64(conn.priorCongestionWindow)) - int64(out) + } else { + sendCount = max(int64(conn.reductionDelivered)-int64(out), int64(newlyAckedSacked)) + if flags&goAckUnackedAdvanced != 0 && newlyLost == 0 { + sendCount++ + } + sendCount = min(delta, sendCount) + } + if out == 0 { + sendCount = max(sendCount, 1) + } else { + sendCount = max(sendCount, 0) + } + conn.setCongestionWindow(inFlight + uint32(sendCount)) +} + +func (e *goEngine) endWindowReduction(conn *GoConn) { + if conn.congestion.congControl != nil { + return + } + if conn.slowStartThreshold < goInfiniteSlowStartThreshold && (conn.congestionState == goCongestionCWR || conn.undoMarker != 0) { + conn.setCongestionWindow(conn.slowStartThreshold) + conn.congestionWindowStamp = e.now() + } + conn.congestionEvent(goCongestionEventCompleteCWR) +} + +func (e *goEngine) enterCWR(conn *GoConn) { + conn.priorSlowStartThreshold = 0 + if conn.congestionState < goCongestionCWR { + conn.undoMarker = 0 + e.initWindowReduction(conn) + conn.setCAState(goCongestionCWR) + } +} + +func (e *goEngine) enterRecovery(conn *GoConn) { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + conn.priorSlowStartThreshold = 0 + e.initUndo(conn) + if !conn.inWindowReduction() { + conn.priorSlowStartThreshold = conn.currentSlowStartThreshold() + e.initWindowReduction(conn) + } + conn.setCAState(goCongestionRecovery) +} + +func (e *goEngine) enterLoss(conn *GoConn) { + newRecovery := conn.congestionState < goCongestionRecovery + unacked := conn.sendUnacked.Load() + sent := conn.sentTail.Load() + reneging := len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorSacked != 0 + if reneging { + conn.highestSacked = unacked + conn.sackReneging = true + } else if !conn.sackPermitted { + conn.renoSacked = 0 + } + now := e.now() + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if reneging { + descriptor.flags &^= goDescriptorSacked + } else if index != 0 && e.rackDescriptorTimeout(conn, descriptor, 0, now) > 0 { + continue + } + e.markLost(conn, descriptor) + } + e.summarizeFlight(conn) + if conn.congestionState <= goCongestionDisorder || unacked >= conn.highSeq || (conn.congestionState == goCongestionLoss && conn.retransmitAttempts == 0) { + conn.priorSlowStartThreshold = conn.currentSlowStartThreshold() + conn.priorCongestionWindow = conn.congestionWindow + conn.slowStartThreshold = conn.congestion.slowStartThreshold(conn) + conn.congestionEvent(goCongestionEventLoss) + e.initUndo(conn) + } + conn.setCongestionWindow(conn.flight.inFlight() + 1) + conn.congestionWindowCount = 0 + conn.congestionWindowStamp = e.now() + if conn.congestionState <= goCongestionDisorder && conn.flight.sackedOut >= goInitialReordering { + conn.reordering = min(conn.reordering, goInitialReordering) + } + conn.setCAState(goCongestionLoss) + conn.highSeq = sent + conn.probeHighSeq = 0 + conn.frto = conn.sackPermitted && (newRecovery || conn.retransmitAttempts > 0) +} + +func (e *goEngine) anyRetransmitDone(conn *GoConn) bool { + if conn.flight.retransmitOut > 0 { + return true + } + return len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorRetransmitted != 0 +} + +func (c *GoConn) packetDelayed(flags goAckFlags) bool { + if c.retransmitStamp != 0 { + return c.echoedTimestampBefore(c.retransmitStamp) + } + if !c.sackPermitted && c.sendUnacked.Load() >= c.highSeq { + return false + } + return true +} + +func (c *GoConn) echoedTimestampBefore(stamp uint32) bool { + return c.timestampsEnabled && c.lastTimestampEcho != 0 && int32(c.lastTimestampEcho-stamp) < 0 +} + +func (c *GoConn) mayUndo(flags goAckFlags) bool { + return c.undoMarker != 0 && (c.undoRetransmits == 0 || c.packetDelayed(flags)) +} + +func (e *goEngine) undoWindowReduction(conn *GoConn, unmarkLoss bool) { + if unmarkLoss { + for index := range conn.scoreboard.entries { + conn.scoreboard.entries[index].flags &^= goDescriptorLost + } + e.summarizeFlight(conn) + } + if conn.priorSlowStartThreshold != 0 { + conn.setCongestionWindow(conn.congestion.undoWindow(conn)) + if conn.priorSlowStartThreshold > conn.slowStartThreshold { + conn.slowStartThreshold = conn.priorSlowStartThreshold + } + } + conn.congestionWindowStamp = e.now() + conn.undoMarker = 0 + conn.rackAdvanced = true +} + +func (e *goEngine) nonSackPreventingReopen(conn *GoConn) bool { + if !conn.sackPermitted && conn.sendUnacked.Load() == conn.highSeq { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + return true + } + return false +} + +func (e *goEngine) tryUndoRecovery(conn *GoConn, flags goAckFlags) bool { + if conn.mayUndo(flags) { + e.undoWindowReduction(conn, false) + } else if conn.reorderWindowPersist > 0 { + conn.reorderWindowPersist-- + } + if e.nonSackPreventingReopen(conn) { + return true + } + conn.setCAState(goCongestionOpen) + conn.sackReneging = false + return false +} + +func (e *goEngine) tryUndoDSACK(conn *GoConn) bool { + if conn.undoMarker != 0 && conn.undoRetransmits == 0 { + conn.reorderWindowPersist = min(goRackRecoveryThreshold, conn.reorderWindowPersist+1) + e.undoWindowReduction(conn, false) + return true + } + return false +} + +func (e *goEngine) tryUndoLoss(conn *GoConn, flags goAckFlags, frtoUndo bool) bool { + if !frtoUndo && !conn.mayUndo(flags) { + return false + } + e.undoWindowReduction(conn, true) + conn.retransmitAttempts = 0 + if e.nonSackPreventingReopen(conn) { + return true + } + if frtoUndo || conn.sackPermitted { + conn.setCAState(goCongestionOpen) + conn.sackReneging = false + } + return true +} + +func (e *goEngine) tryUndoPartial(conn *GoConn, flags goAckFlags, priorUnacked uint64) bool { + if conn.undoMarker == 0 || !conn.packetDelayed(flags) { + return false + } + e.checkSackReordering(conn, priorUnacked) + if conn.flight.retransmitOut > 0 { + return true + } + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + e.undoWindowReduction(conn, true) + e.tryKeepOpen(conn) + return false +} + +func (e *goEngine) tryKeepOpen(conn *GoConn) { + state := goCongestionOpen + if conn.flight.leftOut() != 0 || e.anyRetransmitDone(conn) { + state = goCongestionDisorder + } + if conn.congestionState != state { + conn.setCAState(state) + conn.highSeq = conn.sentTail.Load() + } +} + +func (e *goEngine) tryToOpen(conn *GoConn) { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + if conn.congestionState != goCongestionCWR { + e.tryKeepOpen(conn) + } +} + +func (e *goEngine) checkSackReordering(conn *GoConn, lowOffset uint64) { + fack := conn.highestSacked + if lowOffset >= fack { + return + } + metric := fack - lowOffset + mss := uint64(conn.effectiveMSS) + if metric > uint64(conn.reordering)*mss { + conn.reordering = uint32(min((metric+mss-1)/mss, goMaxReordering)) + } + conn.reorderingSeen++ +} + +func (e *goEngine) mayRaiseWindow(conn *GoConn, flags goAckFlags) bool { + if conn.reordering > goInitialReordering { + return flags&goAckForwardProgress != 0 + } + return flags&goAckDataAcked != 0 +} + +func (e *goEngine) congestionControl(conn *GoConn, ackedSacked uint32, flags goAckFlags, sample *goRateSample) { + if conn.congestion.congControl != nil { + conn.congestion.congControl(conn, flags, sample) + return + } + if conn.inWindowReduction() { + e.reduceWindow(conn, ackedSacked, sample.losses, flags) + } else if e.mayRaiseWindow(conn, flags) { + conn.congestion.congAvoid(conn, ackedSacked) + conn.congestionWindowStamp = e.now() + } + e.updatePacingRate(conn) +} + +func (e *goEngine) updatePacingRate(conn *GoConn) { + rate := uint64(conn.effectiveMSS) * (uint64(time.Second/time.Microsecond) / 100) + if conn.congestionWindow < conn.slowStartThreshold/2 { + rate *= goPacingSlowStartRatio + } else { + rate *= goPacingAvoidanceRatio + } + rate *= uint64(max(conn.congestionWindow, conn.flight.packetsOut)) + if conn.smoothedRoundTrip > 0 { + rate /= uint64(conn.smoothedRoundTrip) + } + conn.pacingRate.Store(rate) +} + +func (e *goEngine) validateWindow(conn *GoConn) { + limited := conn.windowLimitedSince.Swap(false) + peak := conn.peakFlight.Swap(0) + if peak == 0 && !limited { + return + } + peakPackets := uint32(peak) + if conn.sendUnacked.Load() >= conn.windowUsageSeq || limited || (!conn.windowLimited && peakPackets > conn.maxPacketsOut) { + conn.windowLimited = limited + conn.maxPacketsOut = peakPackets + conn.windowUsageSeq = conn.sentTail.Load() + } + now := e.now() + if conn.isWindowLimited() { + conn.congestionWindowUsed = 0 + conn.congestionWindowStamp = now + return + } + if peakPackets > conn.congestionWindowUsed { + conn.congestionWindowUsed = peakPackets + } + if conn.congestion.congControl == nil && now-conn.congestionWindowStamp >= int64(conn.retransmitTimeout)*int64(time.Microsecond) { + e.applicationLimitedWindow(conn, now) + } +} + +func (e *goEngine) armIdleRestart(conn *GoConn) { + if conn.congestion.congControl != nil || conn.congestionState != goCongestionOpen || conn.congestionWindow <= goInitialWindow { + conn.idleDeadline = 0 + return + } + conn.idleDeadline = e.now() + int64(conn.retransmitTimeout)*int64(time.Microsecond) +} + +func (e *goEngine) expireIdleRestart(conn *GoConn, now int64) { + conn.idleDeadline = 0 + if conn.hasOutstanding() || conn.congestionState != goCongestionOpen || conn.congestion.congControl != nil { + e.rearmTimer(conn) + return + } + conn.congestionEvent(goCongestionEventWindowRestart) + conn.slowStartThreshold = conn.currentSlowStartThreshold() + restart := min(uint32(goInitialWindow), conn.congestionWindow) + conn.setCongestionWindow(max(conn.congestionWindow>>1, restart)) + conn.congestionWindowStamp = now + conn.congestionWindowUsed = 0 + e.summarizeFlight(conn) + conn.publishPermit(conn.sendUnacked.Load()) + e.armIdleRestart(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) applicationLimitedWindow(conn *GoConn, now int64) { + if conn.congestionState == goCongestionOpen { + used := max(conn.congestionWindowUsed, uint32(goInitialWindow)) + if used < conn.congestionWindow { + conn.slowStartThreshold = conn.currentSlowStartThreshold() + conn.setCongestionWindow((conn.congestionWindow + used) >> 1) + } + conn.congestionWindowUsed = 0 + } + conn.congestionWindowStamp = now +} + +type goMinMaxSample struct { + time uint32 + value uint32 +} + +// Mirrors lib/win_minmax.c. +type goMinMax struct { + samples [3]goMinMaxSample +} + +func (m *goMinMax) get() uint32 { + return m.samples[0].value +} + +func (m *goMinMax) reset(time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + m.samples[0] = sample + m.samples[1] = sample + m.samples[2] = sample + return value +} + +func (m *goMinMax) subwindowUpdate(window uint32, sample goMinMaxSample) uint32 { + dt := sample.time - m.samples[0].time + if dt > window { + m.samples[0] = m.samples[1] + m.samples[1] = m.samples[2] + m.samples[2] = sample + if sample.time-m.samples[0].time > window { + m.samples[0] = m.samples[1] + m.samples[1] = m.samples[2] + m.samples[2] = sample + } + } else if m.samples[1].time == m.samples[0].time && dt > window/4 { + m.samples[2] = sample + m.samples[1] = sample + } else if m.samples[2].time == m.samples[1].time && dt > window/2 { + m.samples[2] = sample + } + return m.samples[0].value +} + +func (m *goMinMax) runningMax(window uint32, time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + if value >= m.samples[0].value || time-m.samples[2].time > window { + return m.reset(time, value) + } + if value >= m.samples[1].value { + m.samples[2] = sample + m.samples[1] = sample + } else if value >= m.samples[2].value { + m.samples[2] = sample + } + return m.subwindowUpdate(window, sample) +} + +func (m *goMinMax) runningMin(window uint32, time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + if value <= m.samples[0].value || time-m.samples[2].time > window { + return m.reset(time, value) + } + if value <= m.samples[1].value { + m.samples[2] = sample + m.samples[1] = sample + } else if value <= m.samples[2].value { + m.samples[2] = sample + } + return m.subwindowUpdate(window, sample) +} diff --git a/stack_go_tcp_cubic.go b/stack_go_tcp_cubic.go new file mode 100644 index 00000000..9530ac66 --- /dev/null +++ b/stack_go_tcp_cubic.go @@ -0,0 +1,298 @@ +package tun + +import ( + "math/bits" + "time" +) + +// Mirrors net/ipv4/tcp_cubic.c with HZ = 1000. + +const ( + goCubicBetaScale = 1024 + goCubicHZ = 10 + goCubicBeta = 717 + goCubicScale = 41 + goCubicHystartAckTrain = 1 + goCubicHystartDelay = 2 + goCubicHystartDetect = goCubicHystartAckTrain | goCubicHystartDelay + goCubicHystartMinSamples = 8 + goCubicHystartDelayMin = 4000 + goCubicHystartDelayMax = 16000 + goCubicHystartLowWindow = 16 + goCubicHystartAckDelta = 2000 + goCubicJiffiesPerSecond = 1000 + goCubicUpdateInterval = goCubicJiffiesPerSecond / 32 +) + +var ( + goCubicBetaScaleFactor = uint32(8 * (goCubicBetaScale + goCubicBeta) / 3 / (goCubicBetaScale - goCubicBeta)) + goCubicRoundTripScale = uint64(goCubicScale * 10) + goCubicFactor = (uint64(1) << (10 + 3*goCubicHZ)) / (goCubicScale * 10) +) + +type goCubicState struct { + count uint32 + lastMaxWindow uint32 + lastWindow uint32 + lastTime uint32 + originPoint uint32 + timeToOrigin uint32 + minDelay uint32 + epochStart uint32 + ackCount uint32 + renoWindow uint32 + sampleCount uint8 + found bool + roundStart int64 + endOffset uint64 + lastAck int64 + currentRoundTrip uint32 +} + +func goCubicClamp(value uint32) uint32 { + return min(max(value, goCubicHystartDelayMin), goCubicHystartDelayMax) +} + +func (s *goCubicState) reset() { + s.count = 0 + s.lastMaxWindow = 0 + s.lastWindow = 0 + s.lastTime = 0 + s.originPoint = 0 + s.timeToOrigin = 0 + s.minDelay = 0 + s.epochStart = 0 + s.ackCount = 0 + s.renoWindow = 0 + s.found = false +} + +func goCubicJiffies(c *GoConn) uint32 { + return uint32(c.engine.now()/int64(time.Millisecond)) + 1 +} + +func goCubicMicros(c *GoConn) int64 { + return c.engine.now() / int64(time.Microsecond) +} + +func (s *goCubicState) hystartReset(c *GoConn) { + now := goCubicMicros(c) + s.roundStart = now + s.lastAck = now + s.endOffset = c.sentTail.Load() + s.currentRoundTrip = ^uint32(0) + s.sampleCount = 0 +} + +func goCubicInit(c *GoConn) { + state := new(goCubicState) + state.reset() + state.hystartReset(c) + c.congestionPrivate = state +} + +func goCubicTxStart(c *GoConn, idleNanos int64) { + state := c.congestionPrivate.(*goCubicState) + delta := idleNanos / int64(time.Millisecond) + if state.epochStart != 0 && delta > 0 { + now := goCubicJiffies(c) + state.epochStart += uint32(delta) + if int32(state.epochStart-now) > 0 { + state.epochStart = now + } + } +} + +var goCubicRootTable = [64]uint8{ + 0, 54, 54, 54, 118, 118, 118, 118, + 123, 129, 134, 138, 143, 147, 151, 156, + 157, 161, 164, 168, 170, 173, 176, 179, + 181, 185, 187, 190, 192, 194, 197, 199, + 200, 202, 204, 206, 209, 211, 213, 215, + 217, 219, 221, 222, 224, 225, 227, 229, + 231, 232, 234, 236, 237, 239, 240, 242, + 244, 245, 246, 248, 250, 251, 252, 254, +} + +func goCubicRoot(a uint64) uint32 { + b := uint32(bits.Len64(a)) + if b < 7 { + return (uint32(goCubicRootTable[a]) + 35) >> 6 + } + b = ((b * 84) >> 8) - 1 + shift := a >> (b * 3) + x := ((uint32(goCubicRootTable[shift]) + 10) << b) >> 6 + x = 2*x + uint32(a/(uint64(x)*uint64(x-1))) + x = (x * 341) >> 10 + return x +} + +func (s *goCubicState) update(c *GoConn, window uint32, acked uint32) { + now := goCubicJiffies(c) + s.ackCount += acked + if s.lastWindow == window && int32(now-s.lastTime) <= goCubicUpdateInterval { + return + } + if s.epochStart == 0 || now != s.lastTime { + s.lastWindow = window + s.lastTime = now + if s.epochStart == 0 { + s.epochStart = now + s.ackCount = acked + s.renoWindow = window + if s.lastMaxWindow <= window { + s.timeToOrigin = 0 + s.originPoint = window + } else { + s.timeToOrigin = goCubicRoot(goCubicFactor * uint64(s.lastMaxWindow-window)) + s.originPoint = s.lastMaxWindow + } + } + t := uint64(now - s.epochStart) + t += (uint64(s.minDelay) + 999) / 1000 + t <<= goCubicHZ + t /= goCubicJiffiesPerSecond + var offset uint64 + if t < uint64(s.timeToOrigin) { + offset = uint64(s.timeToOrigin) - t + } else { + offset = t - uint64(s.timeToOrigin) + } + delta := uint32((goCubicRoundTripScale * offset * offset * offset) >> (10 + 3*goCubicHZ)) + var target uint32 + if t < uint64(s.timeToOrigin) { + target = s.originPoint - delta + } else { + target = s.originPoint + delta + } + if target > window { + s.count = window / (target - window) + } else { + s.count = 100 * window + } + if s.lastMaxWindow == 0 && s.count > 20 { + s.count = 20 + } + } + friendlyDelta := (window * goCubicBetaScaleFactor) >> 3 + for s.ackCount > friendlyDelta { + s.ackCount -= friendlyDelta + s.renoWindow++ + } + if s.renoWindow > window { + maxCount := window / (s.renoWindow - window) + if s.count > maxCount { + s.count = maxCount + } + } + s.count = max(s.count, 2) +} + +func goCubicCongAvoid(c *GoConn, acked uint32) { + state := c.congestionPrivate.(*goCubicState) + if !c.isWindowLimited() { + return + } + if c.inSlowStart() { + acked = c.slowStart(acked) + if acked == 0 { + return + } + } + state.update(c, c.congestionWindow, acked) + c.congAvoidAI(state.count, acked) +} + +func goCubicSlowStartThreshold(c *GoConn) uint32 { + state := c.congestionPrivate.(*goCubicState) + state.epochStart = 0 + window := c.congestionWindow + if window < state.lastMaxWindow { + state.lastMaxWindow = (window * (goCubicBetaScale + goCubicBeta)) / (2 * goCubicBetaScale) + } else { + state.lastMaxWindow = window + } + return max((window*goCubicBeta)/goCubicBetaScale, 2) +} + +func goCubicSetState(c *GoConn, newState uint8) { + if newState == goCongestionLoss { + state := c.congestionPrivate.(*goCubicState) + state.reset() + state.hystartReset(c) + } +} + +func goCubicHystartAckDelay(c *GoConn) uint32 { + rate := c.pacingRate.Load() + if rate == 0 { + return 0 + } + return uint32(min(uint64(time.Second/time.Microsecond)/1000, uint64(c.gsoMaxSize())*4*uint64(time.Second/time.Microsecond)/rate)) +} + +func (s *goCubicState) hystartUpdate(c *GoConn, delay uint32) { + if c.sendUnacked.Load() > s.endOffset { + s.hystartReset(c) + } + if c.congestionWindow < goCubicHystartLowWindow { + return + } + if goCubicHystartDetect&goCubicHystartAckTrain != 0 { + now := goCubicMicros(c) + if now-s.lastAck <= goCubicHystartAckDelta { + s.lastAck = now + threshold := int64(s.minDelay + goCubicHystartAckDelay(c)) + if !c.congestion.pacing { + threshold >>= 1 + } + if now-s.roundStart > threshold { + s.found = true + c.slowStartThreshold = c.congestionWindow + } + } + } + if goCubicHystartDetect&goCubicHystartDelay != 0 { + if s.currentRoundTrip > delay { + s.currentRoundTrip = delay + } + if s.sampleCount < goCubicHystartMinSamples { + s.sampleCount++ + } else if s.currentRoundTrip > s.minDelay+goCubicClamp(s.minDelay>>3) { + s.found = true + c.slowStartThreshold = c.congestionWindow + } + } +} + +func goCubicAcked(c *GoConn, sample *goAckSample) { + state := c.congestionPrivate.(*goCubicState) + if sample.roundTripMicros < 0 { + return + } + if state.epochStart != 0 && int32(goCubicJiffies(c)-state.epochStart) < goCubicJiffiesPerSecond { + return + } + delay := uint32(max(sample.roundTripMicros, 1)) + if state.minDelay == 0 || state.minDelay > delay { + state.minDelay = delay + } + if !state.found && c.inSlowStart() { + state.hystartUpdate(c, delay) + } +} + +var goCubicOps = goCongestionOps{ + name: "cubic", + init: goCubicInit, + slowStartThreshold: goCubicSlowStartThreshold, + congAvoid: goCubicCongAvoid, + setState: goCubicSetState, + undoWindow: goRenoUndoWindow, + windowEventTxStart: goCubicTxStart, + packetsAcked: goCubicAcked, +} + +func init() { + goRegisterCongestionControl(&goCubicOps) +} diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index f68adff8..7228949d 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -22,8 +22,6 @@ const ( goRTOGranularity = int32(goWheelTick / int64(time.Microsecond)) goSynAckRetransmit = 500 * time.Millisecond goSynAckAttempts = 5 - goPersistInitial = 500 * time.Millisecond - goPersistMax = 60 * time.Second goFinLinger = 5 * time.Second goAbortLinger = time.Second goRetransmitLimit = 8 @@ -42,17 +40,10 @@ const ( goMaxCongestionSize = 4 << 20 goResetBurstLimit = 32 goRecoveryBurst = 64 - goDupThresh = 3 goInitialWindow = 10 goAckCoalesceBytes = 256 << 10 ) -const ( - goFRTOInactive uint8 = iota - goFRTOAwaitFirstAck - goFRTOAwaitSecondAck -) - func (p *forwardPacket) isPureTCPSyn() bool { return p.protocol == uint8(header.TCPProtocolNumber) && p.tcpFlags&header.TCPFlagSyn != 0 && p.tcpFlags&header.TCPFlagAck == 0 @@ -162,10 +153,10 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { conn.lastAckSent = 1 conn.lastActivity = now conn.publishedEdge = 1 + conn.receiveCapacity - conn.congestionWindow = goInitialWindow * uint32(conn.effectiveMSS) - conn.slowStartThreshold = goMaxCongestionSize + conn.initCongestionControl(e.stack.congestion) conn.retransmitTimeout = goRTOInitialMicros conn.peerWindow = uint64(tcpHdr.WindowSize()) + conn.maxPeerWindow = conn.peerWindow conn.lastPeerWindow = tcpHdr.WindowSize() conn.consumedTail.Store(1) conn.receiveAvailable.Store(1) @@ -181,7 +172,8 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { conn.bufferedTail.Store(1) conn.sendUnacked.Store(1) conn.sendReleased.Store(1) - conn.sendPermit.Store(1 + min(uint64(conn.congestionWindow), conn.peerWindow)) + conn.sendPermit.Store(1 + min(uint64(conn.congestionWindow)*uint64(conn.effectiveMSS), conn.peerWindow)) + conn.sendPacketPermit.Store(conn.congestionWindow) conn.buildSynAck(synOptions, localMSS) conn.keyed = true e.flows[key] = conn @@ -346,9 +338,16 @@ func (c *GoConn) recordDuplicate(start int64, end int64) { case uint64(end) <= c.receiveNext: case uint64(start) < c.receiveNext: end = int64(c.receiveNext) - case c.oooRanges != nil && c.oooRanges.covers(uint64(start), uint64(end)): default: - return + if c.oooRanges == nil { + return + } + overlapStart, overlapEnd, overlapped := c.oooRanges.overlap(uint64(start), uint64(end)) + if !overlapped { + return + } + start = int64(overlapStart) + end = int64(overlapEnd) } c.dsackStart = uint64(start) c.dsackEnd = uint64(end) @@ -375,80 +374,6 @@ func (c *GoConn) segmentAcceptable(segOffset int64, segmentLength int64) bool { return segOffset < edge && segOffset+segmentLength > receiveNext } -func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, payloadLength int, flags header.TCPFlags, timestampEcho uint32, hasTimestamp bool) bool { - conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) - ackOffset := conn.sendOffset(tcpHdr.AckNumber()) - limit := int64(conn.sentTail.Load()) - if conn.finSent { - limit++ - } - if ackOffset > limit { - e.markAck(conn, true) - return false - } - unacked := int64(conn.sendUnacked.Load()) - segmentAck := ackOffset - if ackOffset < unacked { - ackOffset = unacked - } - windowRaw := tcpHdr.WindowSize() - handshakeAck := conn.state == goTCPSynReceived - if handshakeAck { - if conn.connState.Load() != goConnStateEngaged { - return false - } - if segmentAck != int64(conn.sentTail.Load()) { - e.sendReset(conn) - e.detachConn(conn, E.New("go: handshake ack out of range"), goDeathImmediate) - return false - } - e.establishConn(conn) - } - rawCount := 0 - if conn.sackPermitted { - rawCount = goParseSackBlocks(tcpHdr, int(tcpHdr.DataOffset()), conn, &e.sackRaw) - } - windowChanged := false - if segOffset > conn.windowLeft1 || (segOffset == conn.windowLeft1 && segmentAck >= conn.windowLeft2) { - windowChanged = windowRaw != conn.lastPeerWindow - conn.lastPeerWindow = windowRaw - conn.peerWindow = uint64(windowRaw) << conn.peerWindowShift - conn.windowLeft1 = segOffset - conn.windowLeft2 = segmentAck - } - newSack := false - if rawCount > 0 { - raw := e.sackRaw[:rawCount] - if goIsDuplicateSack(raw, ackOffset) { - e.applyDuplicateSack(conn, raw[0]) - raw = raw[1:] - } - sackCount := goSelectSackBlocks(raw, &e.sackScratch) - if sackCount > 0 { - newSack = e.applySackBlocks(conn, e.sackScratch[:sackCount]) - } - } - if ackOffset > unacked { - e.advanceUnacked(conn, uint64(ackOffset), timestampEcho, hasTimestamp) - } else if segmentAck == unacked && !handshakeAck && conn.hasOutstandingData() && payloadLength == 0 && flags&(header.TCPFlagSyn|header.TCPFlagFin) == 0 && !windowChanged && !newSack { - e.handleDuplicateAck(conn) - } - if conn.frtoState != goFRTOInactive { - e.stepFRTO(conn, uint64(ackOffset), uint64(unacked), newSack) - } - if windowChanged && ackOffset == unacked { - if conn.inRecovery { - e.retransmitHoles(conn) - } else if conn.frtoState == goFRTOInactive { - e.continueRetransmit(conn, uint64(ackOffset)) - } - } - e.updatePersist(conn) - conn.releaseIdleDescriptors() - conn.publishPermit(uint64(ackOffset)) - return true -} - func (e *goEngine) establishConn(conn *GoConn) { conn.state = goTCPEstablished conn.handshakeDeadline = 0 @@ -458,190 +383,6 @@ func (e *goEngine) establishConn(conn *GoConn) { e.wokeHandlerThisBurst = true } -func (e *goEngine) advanceUnacked(conn *GoConn, ackOffset uint64, timestampEcho uint32, hasTimestamp bool) { - sent := conn.sentTail.Load() - previous := conn.sendUnacked.Load() - newUnacked := min(ackOffset, sent) - acked := newUnacked - previous - e.sampleRoundTrip(conn, newUnacked, timestampEcho, hasTimestamp) - conn.scoreboard.advance(newUnacked) - conn.releaseTransmitted(newUnacked) - conn.sendUnacked.Store(newUnacked) - if conn.finSent && ackOffset > conn.finOffset { - conn.finAcked = true - } - if conn.undoMarker != 0 && newUnacked >= conn.undoLimit && conn.frtoState == goFRTOInactive { - conn.undoMarker = 0 - } - if acked > 0 { - conn.duplicateAckCount = 0 - if conn.inRecovery { - if newUnacked >= conn.recoveryPoint { - conn.inRecovery = false - conn.congestionWindow = conn.slowStartThreshold - conn.pipe = 0 - } else { - conn.congestionWindow = max(conn.slowStartThreshold, uint32(conn.effectiveMSS)) - if conn.highestSacked <= newUnacked { - conn.highestRetransmit = newUnacked - e.retransmitFrom(conn, newUnacked) - } - e.retransmitHoles(conn) - } - } else { - conn.growCongestionWindow(acked) - if conn.frtoState == goFRTOInactive { - e.continueRetransmit(conn, newUnacked) - } - } - if conn.retransmitAttempts > 0 && conn.finSent && !conn.finAcked && !conn.hasOutstandingData() { - e.retransmitFrom(conn, newUnacked) - } - conn.retransmitAttempts = 0 - conn.probeAttempts = 0 - } - e.advanceFinState(conn) - e.rearmRetransmit(conn) -} - -func (c *GoConn) growCongestionWindow(acked uint64) { - mss := uint32(c.effectiveMSS) - counted := uint32(min(acked, uint64(2*mss))) - if c.congestionWindow < c.slowStartThreshold { - c.congestionWindow += counted - c.ackedBytes = 0 - } else { - c.ackedBytes += counted - if c.ackedBytes >= c.congestionWindow { - c.ackedBytes -= c.congestionWindow - c.congestionWindow += mss - } - } - c.congestionWindow = min(c.congestionWindow, goMaxCongestionSize) -} - -func (e *goEngine) sampleRoundTrip(conn *GoConn, acked uint64, timestampEcho uint32, hasTimestamp bool) { - var sample int32 - sampled := false - walked := false - slept := false - for index := range conn.scoreboard.entries { - descriptor := &conn.scoreboard.entries[index] - if descriptor.endOffset > acked { - break - } - walked = true - if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { - slept = true - } else if descriptor.flags&goDescriptorRetransmitted == 0 { - sample = descriptor.sentAt - sampled = true - } - } - now := e.now() - var measured int32 - switch { - case sampled: - units := int64(conn.stamp(now) - sample) - if units < 0 { - return - } - measured = int32(min(units*goTimeUnit/1000, goRTOMaxMicros)) - case hasTimestamp && walked && !slept && timestampEcho != 0: - elapsed := goTimestampAt(now) - timestampEcho - if int32(elapsed) < 0 { - return - } - measured = int32(min(int64(elapsed)< unacked { - return - } - sent := conn.sentTail.Load() - flight := sent - unacked - if flight == 0 { - return - } - conn.armCongestionUndo() - conn.slowStartThreshold = max(uint32(flight/2), 2*uint32(conn.effectiveMSS)) - conn.congestionWindow = conn.slowStartThreshold - conn.inRecovery = true - conn.recoveryPoint = sent - conn.highestRetransmit = unacked - conn.pipe = flight - conn.retransmitPoint = 0 - conn.duplicateAckCount = 0 - e.retransmitFrom(conn, unacked) - e.retransmitHoles(conn) -} - -func (c *GoConn) armCongestionUndo() { - if c.undoMarker != 0 { - return - } - c.undoMarker = c.sendUnacked.Load() - c.undoLimit = c.sentTail.Load() - c.undoCongestionWindow = c.congestionWindow - c.undoSlowStartThreshold = c.slowStartThreshold - c.undoRetransmits = 0 -} - -func (e *goEngine) applyDuplicateSack(conn *GoConn, block goRawSackBlock) { - if conn.undoMarker == 0 || block.start < int64(conn.undoMarker) { - return - } - conn.undoRetransmits-- - if conn.undoRetransmits > 0 { - return - } - conn.undoCongestionReduction() -} - -func (c *GoConn) undoCongestionReduction() { - c.undoMarker = 0 - c.slowStartThreshold = c.undoSlowStartThreshold - c.congestionWindow = max(c.congestionWindow, c.undoCongestionWindow) - c.inRecovery = false - c.duplicateAckCount = 0 - c.retransmitPoint = 0 - c.pipe = 0 -} - func goParseTimestampOption(options []byte) (uint32, uint32, bool) { for index := 0; index < len(options); { switch options[index] { @@ -719,18 +460,17 @@ func goIsDuplicateSack(blocks []goRawSackBlock, ackOffset int64) bool { if first.end <= ackOffset { return true } - for _, block := range blocks[1:] { - if block.start <= first.start && first.end <= block.end { - return true - } + if len(blocks) > 1 { + second := blocks[1] + return second.start <= first.start && first.end <= second.end } return false } -func goSelectSackBlocks(raw []goRawSackBlock, blocks *[goMaxSackBlocks]goSackBlock) int { +func goSelectSackBlocks(raw []goRawSackBlock, unacked uint64, sent uint64, blocks *[goMaxSackBlocks]goSackBlock) int { count := 0 for _, block := range raw { - if block.start < 0 { + if block.start <= int64(unacked) || block.end <= block.start || block.end > int64(sent) { continue } blocks[count] = goSackBlock{start: uint64(block.start), end: uint64(block.end)} @@ -739,173 +479,6 @@ func goSelectSackBlocks(raw []goRawSackBlock, blocks *[goMaxSackBlocks]goSackBlo return count } -func (e *goEngine) applySackBlocks(conn *GoConn, blocks []goSackBlock) bool { - unacked := conn.sendUnacked.Load() - updated := false - sackedBytes := uint64(0) - sackedRuns := 0 - previousSacked := false - start := unacked - scoreboard := &conn.scoreboard - for index := 0; index < len(scoreboard.entries); index++ { - descriptor := &scoreboard.entries[index] - end := descriptor.endOffset - if descriptor.flags&goDescriptorSacked == 0 { - for _, block := range blocks { - if block.end <= start || block.start >= end { - continue - } - if block.start > start { - scoreboard.split(index, block.start) - index++ - descriptor = &scoreboard.entries[index] - start = block.start - } - if block.end < end { - scoreboard.split(index, block.end) - descriptor = &scoreboard.entries[index] - end = block.end - } - descriptor.flags |= goDescriptorSacked - updated = true - break - } - } - if descriptor.flags&goDescriptorSacked == 0 { - previousSacked = false - } else { - sackedBytes += end - start - if !previousSacked { - sackedRuns++ - } - previousSacked = true - } - start = end - } - for _, block := range blocks { - if block.end > conn.highestSacked { - conn.highestSacked = block.end - } - } - if conn.inRecovery { - if updated { - e.retransmitHoles(conn) - } - return updated - } - if sackedBytes > uint64(goDupThresh-1)*uint64(conn.effectiveMSS) || sackedRuns >= goDupThresh { - e.enterRecovery(conn) - } - return updated -} - -func (c *GoConn) lostEdge(unacked uint64) uint64 { - threshold := uint64(goDupThresh-1) * uint64(c.effectiveMSS) - sackedBytes := uint64(0) - sackedRuns := 0 - previousSacked := false - entries := c.scoreboard.entries - for index := len(entries); index > 0; { - index-- - descriptor := &entries[index] - if descriptor.flags&goDescriptorSacked == 0 { - previousSacked = false - if sackedBytes > threshold || sackedRuns >= goDupThresh { - return descriptor.endOffset - } - continue - } - sackedBytes += descriptor.endOffset - c.scoreboard.startOf(index, unacked) - if !previousSacked { - sackedRuns++ - } - previousSacked = true - } - return 0 -} - -func (c *GoConn) estimatePipe(unacked uint64, lostEdge uint64) uint64 { - pipe := uint64(0) - start := unacked - for index := range c.scoreboard.entries { - descriptor := &c.scoreboard.entries[index] - end := descriptor.endOffset - if descriptor.flags&goDescriptorSacked == 0 { - if start >= lostEdge { - pipe += end - start - } - if start < c.highestRetransmit { - pipe += min(end, c.highestRetransmit) - start - } - } - start = end - } - sent := c.sentTail.Load() - if sent > start { - pipe += sent - start - } - return pipe -} - -func (e *goEngine) retransmitHoles(conn *GoConn) { - unacked := conn.sendUnacked.Load() - lostEdge := conn.lostEdge(unacked) - conn.pipe = conn.estimatePipe(unacked, lostEdge) - if lostEdge <= unacked { - return - } - window := uint64(conn.congestionWindow) - mss := uint64(conn.effectiveMSS) - sent := 0 - start := unacked - for index := 0; index < len(conn.scoreboard.entries); index++ { - descriptor := &conn.scoreboard.entries[index] - end := descriptor.endOffset - if start >= lostEdge { - return - } - if descriptor.flags&goDescriptorSacked != 0 || end <= conn.highestRetransmit { - start = end - continue - } - for offset := max(start, conn.highestRetransmit); offset < end; sent++ { - if sent == goRecoveryBurst { - conn.pipe = max(conn.pipe, window) - return - } - if conn.pipe+mss > window { - return - } - length := int(min(end-offset, mss)) - length = e.transmitRetransmit(conn, offset, length) - if length == 0 { - return - } - offset += uint64(length) - conn.pipe += uint64(length) - } - start = end - } -} - -func (c *GoConn) publishPermit(ackOffset uint64) { - unacked := c.sendUnacked.Load() - limit := unacked + uint64(c.congestionWindow) - switch { - case c.frtoState == goFRTOAwaitSecondAck: - limit = c.frtoSendLimit - case c.inRecovery: - limit = c.sentTail.Load() - if uint64(c.congestionWindow) > c.pipe { - limit += uint64(c.congestionWindow) - c.pipe - } - } - permit := max(min(limit, ackOffset+c.peerWindow), unacked) - c.sendPermit.Store(permit) - c.wakeTransmitter() - c.wakeWriter() -} - func (c *GoConn) releaseTransmitted(unacked uint64) { edge := min(unacked, c.transmittedTail.Load()) if edge <= c.sendReleased.Load() { @@ -993,6 +566,12 @@ func (e *goEngine) deliverSegment(conn *GoConn, segOffset int64, payload []byte, e.acceptFin(conn) return } + if fin && finOffset > receiveNext && finOffset < edge { + if conn.oooRanges == nil { + conn.oooRanges = new(goRangeSet) + } + conn.oooRanges.insert(uint64(finOffset), uint64(finOffset), true) + } e.markAck(conn, true) return } @@ -1103,6 +682,7 @@ func (e *goEngine) acceptFin(conn *GoConn) { return } conn.finReceived = true + conn.oooRanges = nil conn.receiveNext++ conn.receiveNextAck.Store(conn.receiveNext) conn.receiveShutdown.Store(true) @@ -1299,6 +879,7 @@ func (e *goEngine) writeConnControl(conn *GoConn, segment *goSegment) bool { frame, _ := conn.buildFrame(e.controlScratch[:], e.controlSegments[:0], segment, &conn.transmitStore, false) e.controlSegments = frame err := e.platformIO.writeFrame(frame, ForwardFrameMeta{}) + clear(frame) if err != nil { e.stack.logger.Trace(E.Cause(err, "go: write control frame")) return false @@ -1393,20 +974,25 @@ func (e *goEngine) handleDroppedFrames(conn *GoConn) { if conn.connState.Load() >= goConnStateAborted { return } - conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) - unacked := conn.sendUnacked.Load() - start := unacked + e.drainDescriptors(conn) + lost := false for index := range conn.scoreboard.entries { descriptor := &conn.scoreboard.entries[index] - end := descriptor.endOffset - if descriptor.flags&goDescriptorDropped != 0 && descriptor.flags&goDescriptorSacked == 0 { - if e.transmitRetransmit(conn, start, int(end-start)) == 0 { - break - } + if descriptor.flags&goDescriptorDropped != 0 && descriptor.flags&(goDescriptorSacked|goDescriptorLost) == 0 { + e.markLost(conn, descriptor) + lost = true + } + } + e.refreshPacketsOut(conn) + if lost { + e.enterCWR(conn) + if conn.congestionState < goCongestionRecovery { + e.enterRecovery(conn) } - start = end + e.xmitRetransmitQueue(conn) } e.rearmRetransmit(conn) + conn.publishPermit(conn.sendUnacked.Load()) } func (e *goEngine) handleTransmitBlocked(conn *GoConn) { @@ -1461,7 +1047,11 @@ func (e *goEngine) rearmRetransmit(conn *GoConn) { conn.retransmitDeadline = 0 conn.retransmitArmed.Store(false) if !conn.hasOutstanding() { - e.rearmTimer(conn) + if conn.persistNeeded() { + e.updatePersist(conn) + } else { + e.rearmTimer(conn) + } return } conn.retransmitArmed.Store(true) @@ -1475,25 +1065,6 @@ func (e *goEngine) rearmRetransmit(conn *GoConn) { e.rearmTimer(conn) } -func (e *goEngine) armProbe(conn *GoConn) { - if conn.probeAttempts >= goProbeAttempts || !conn.hasOutstanding() || conn.frtoState != goFRTOInactive || conn.inRecovery { - conn.probeDeadline = 0 - return - } - timeout := int32(goProbeNoSampleMicros) - if conn.smoothedRoundTrip > 0 { - timeout = max(2*conn.smoothedRoundTrip, goProbeFloorMicros) - } - if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS) { - timeout += goProbeDelayedAck - } - if timeout >= conn.retransmitTimeout { - conn.probeDeadline = 0 - return - } - conn.probeDeadline = e.now() + int64(timeout)*int64(time.Microsecond) -} - func (c *GoConn) persistNeeded() bool { return c.peerWindow == 0 && (c.bufferedTail.Load() > c.sendUnacked.Load() || (c.finSent && !c.finAcked)) } @@ -1504,7 +1075,7 @@ func (e *goEngine) updatePersist(conn *GoConn) { conn.probeDeadline = 0 if conn.persistDeadline == 0 { conn.persistAttempts = 0 - conn.persistDeadline = e.now() + int64(goPersistInitial) + conn.persistDeadline = e.now() + int64(conn.retransmitTimeout)*int64(time.Microsecond) } e.rearmTimer(conn) return @@ -1523,7 +1094,7 @@ func (e *goEngine) updatePersist(conn *GoConn) { func (e *goEngine) rearmTimer(conn *GoConn) { next := int64(0) - for _, deadline := range [5]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline} { + for _, deadline := range [8]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline, conn.idleDeadline, conn.pacingDeadline, conn.reorderDeadline} { if deadline == 0 { continue } @@ -1552,6 +1123,18 @@ func (e *goEngine) fireConnTimer(conn *GoConn, now int64) { e.expireHandshake(conn, now) return } + if conn.pacingDeadline != 0 && conn.pacingDeadline <= now { + e.expirePacing(conn) + return + } + if conn.idleDeadline != 0 && conn.idleDeadline <= now { + e.expireIdleRestart(conn, now) + return + } + if conn.reorderDeadline != 0 && conn.reorderDeadline <= now { + e.expireReorder(conn) + return + } if conn.persistDeadline != 0 && conn.persistDeadline <= now { e.expirePersist(conn, now) return @@ -1592,165 +1175,16 @@ func (e *goEngine) expirePersist(conn *GoConn, now int64) { segment := goSegment{offset: conn.sendUnacked.Load() - 1, flags: header.TCPFlagAck} e.writeConnControl(conn, &segment) conn.persistAttempts = min(conn.persistAttempts+1, 8) - backoff := min(int64(goPersistInitial)< unacked: - offset := unacked - if sent-unacked > uint64(conn.effectiveMSS) { - offset = sent - uint64(conn.effectiveMSS) - } - e.transmitRetransmit(conn, offset, int(sent-offset)) - case conn.finSent && !conn.finAcked: - segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} - e.writeConnControl(conn, &segment) - } - conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) - e.armProbe(conn) - e.rearmTimer(conn) -} - -func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { - if !conn.hasOutstanding() { - conn.retransmitDeadline = 0 - conn.retransmitArmed.Store(false) - e.rearmTimer(conn) - return - } - if conn.persistNeeded() { - e.updatePersist(conn) - return - } - conn.probeDeadline = 0 - conn.probeAttempts = 0 - conn.retransmitAttempts++ - if conn.retransmitAttempts >= goRetransmitLimit && conn.persistDeadline == 0 { - conn.retransmitDeadline = 0 - e.sendReset(conn) - e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: retransmit limit"), goDeathImmediate) - return - } - conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) - unacked := conn.sendUnacked.Load() - sent := conn.sentTail.Load() - recovering := conn.inRecovery || (conn.retransmitPoint > unacked && conn.frtoState != goFRTOAwaitFirstAck) - flight := sent - unacked - if flight > 0 { - conn.armCongestionUndo() - conn.slowStartThreshold = max(uint32(flight/2), 2*uint32(conn.effectiveMSS)) - } - conn.congestionWindow = uint32(conn.effectiveMSS) - conn.inRecovery = false - conn.duplicateAckCount = 0 - conn.highestRetransmit = unacked - conn.pipe = 0 - conn.retransmitPoint = sent - conn.retransmitHighWater = unacked - conn.highestSacked = unacked - for index := range conn.scoreboard.entries { - conn.scoreboard.entries[index].flags &^= goDescriptorSacked - } - if conn.sackPermitted && !recovering && flight > 0 { - conn.frtoState = goFRTOAwaitFirstAck - } else { - conn.frtoState = goFRTOInactive - } - e.continueRetransmit(conn, unacked) - if !conn.hasOutstandingData() { - e.retransmitFrom(conn, unacked) - } - conn.retransmitTimeout = min(conn.retransmitTimeout*2, goRTOMaxMicros) - conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + backoff := min(int64(conn.retransmitTimeout)< previousUnacked - switch conn.frtoState { - case goFRTOAwaitFirstAck: - if !advanced { - return - } - conn.frtoRecoveryPoint = conn.sentTail.Load() - conn.frtoSendLimit = conn.frtoRecoveryPoint + 2*uint64(conn.effectiveMSS) - windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) - if ackOffset < conn.retransmitHighWater || ackOffset >= conn.frtoRecoveryPoint || !conn.hasDataWaiting() || windowEnd < conn.frtoSendLimit { - e.abandonFRTO(conn, 2) - return - } - conn.frtoState = goFRTOAwaitSecondAck - case goFRTOAwaitSecondAck: - if !advanced && !newSack { - return - } - if ackOffset > conn.frtoRecoveryPoint || conn.highestSacked > conn.frtoRecoveryPoint { - e.abandonFRTO(conn, 3) - return - } - conn.frtoState = goFRTOInactive - conn.retransmitHighWater = 0 - conn.undoCongestionReduction() - } -} - -func (e *goEngine) abandonFRTO(conn *GoConn, segments uint32) { - conn.frtoState = goFRTOInactive - conn.congestionWindow = min(conn.congestionWindow, segments*uint32(conn.effectiveMSS)) - e.continueRetransmit(conn, conn.sendUnacked.Load()) -} - -func (e *goEngine) continueRetransmit(conn *GoConn, unacked uint64) { - if conn.retransmitPoint <= unacked { - conn.retransmitPoint = 0 - return - } - offset := max(unacked, conn.retransmitHighWater) - limit := min(conn.retransmitPoint, unacked+uint64(conn.congestionWindow)) - entries := conn.scoreboard.entries - index := 0 - for sent := 0; offset < limit && sent < goRecoveryBurst; { - for index < len(entries) && entries[index].endOffset <= offset { - index++ - } - end := limit - if index < len(entries) { - descriptor := &entries[index] - if descriptor.flags&goDescriptorSacked != 0 { - offset = min(descriptor.endOffset, limit) - continue - } - end = min(end, descriptor.endOffset) - } - length := int(min(end-offset, uint64(conn.effectiveMSS))) - length = e.transmitRetransmit(conn, offset, length) - if length == 0 { - break - } - offset += uint64(length) - sent++ - } - conn.retransmitHighWater = offset } func (e *goEngine) retransmitFrom(conn *GoConn, offset uint64) { sent := conn.sentTail.Load() offset = max(offset, 1) if offset < sent { - e.transmitRetransmit(conn, offset, int(min(sent-offset, uint64(conn.effectiveMSS)))) + e.transmitRetransmit(conn, offset, int(min(sent-offset, uint64(conn.effectiveMSS))), false) return } if conn.finSent && !conn.finAcked { @@ -1759,40 +1193,6 @@ func (e *goEngine) retransmitFrom(conn *GoConn, offset uint64) { } } -func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int) int { - windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) - limit := min(conn.sentTail.Load(), windowEnd) - if conn.peerWindow == 0 || offset >= limit { - return 0 - } - length = int(min(uint64(length), uint64(conn.effectiveMSS), limit-offset)) - segment := goSegment{offset: offset, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} - if !e.writeConnControl(conn, &segment) { - return 0 - } - end := offset + uint64(length) - if end > conn.highestRetransmit { - conn.highestRetransmit = end - } - if conn.undoMarker != 0 && offset >= conn.undoMarker { - conn.undoRetransmits++ - } - e.markRetransmitted(conn, offset, end) - return length -} - -func (e *goEngine) markRetransmitted(conn *GoConn, start uint64, end uint64) { - previous := conn.sendUnacked.Load() - for index := range conn.scoreboard.entries { - descriptor := &conn.scoreboard.entries[index] - if previous < end && descriptor.endOffset > start { - descriptor.flags |= goDescriptorRetransmitted - descriptor.flags &^= goDescriptorDropped - } - previous = descriptor.endOffset - } -} - func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { if conn.connState.Load() >= goConnStateDead { return @@ -1819,6 +1219,9 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { conn.probeDeadline = 0 conn.persistDeadline = 0 conn.handshakeDeadline = 0 + conn.idleDeadline = 0 + conn.pacingDeadline = 0 + conn.reorderDeadline = 0 conn.finPending = false switch class { case goDeathFinLinger: @@ -1876,6 +1279,7 @@ func (e *goEngine) reapDying() { next := conn.dyingNext conn.dyingNext = nil if conn.reclaimable() { + e.platformIO.flush() conn.releaseResources() conn.onDyingList = false } else { @@ -1892,7 +1296,7 @@ func (e *goEngine) reapDying() { } func (c *GoConn) reclaimable() bool { - if c.spliced.Load() || c.writerActive.Load() != 0 || c.transmitterActive.Load() != 0 || c.postedTarget.Load() != nil { + if c.spliced.Load() || c.writerActive.Load() != 0 || c.flushActive.Load() != 0 || c.transmitterActive.Load() != 0 || c.postedTarget.Load() != nil { return false } if c.receiveDrainable.Load() && !c.userClosed.Load() && c.receiveAvailable.Load() > c.consumedTail.Load() { @@ -1912,6 +1316,7 @@ func (c *GoConn) releaseResources() { c.oooRanges = nil c.descriptors.release() c.scoreboard.reset() + c.releaseCongestionControl() if c.receiveTarget.buffer != nil { c.receiveTarget.buffer.Release() c.receiveTarget.buffer = nil @@ -1992,6 +1397,9 @@ func (e *goEngine) expireReclaimTick(now int64) { func (e *goEngine) reclaim() { e.slabPool.trim() e.descriptorPool.trim() + for _, conn := range e.flows { + conn.scoreboard.trim() + } for index := range e.reassemblyEntries { entry := &e.reassemblyEntries[index] if !entry.active { diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go index 530a2c98..f2a9264a 100644 --- a/stack_go_tcp_output.go +++ b/stack_go_tcp_output.go @@ -5,6 +5,7 @@ import ( "net" "net/netip" "os" + "time" "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" @@ -155,11 +156,7 @@ func (c *GoConn) FrontHeadroom() int { } func (c *GoConn) WriterMTU() int { - mss := int(c.effectiveMSS) - if c.engine.platformIO.transmitSegmentOffload() { - return max(goGSOMaxPayload/mss*mss, mss) - } - return max(0xffff/mss*mss, mss) + return int(c.gsoMaxSize()) } func (c *GoConn) publishBuffered(tail uint64) { @@ -177,11 +174,14 @@ func (c *GoConn) transmitInline() { c.transmitSignal.notify() return } - _, blocked := c.transmitLoop(false, 0) + _, result := c.transmitLoop(false, 0) c.transmitOwner.Store(0) - if blocked { + switch result { + case goTransmitBlocked: c.wakeTransmitterGoroutine() return + case goTransmitPaced: + return } if c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 { return @@ -198,11 +198,14 @@ func (c *GoConn) transmitOnEngine() bool { c.transmitSignal.notify() return true } - written, blocked := c.transmitLoop(false, budget) + written, result := c.transmitLoop(false, budget) c.transmitOwner.Store(0) c.engine.inlineTransmitBudget = budget - written - if blocked { + switch result { + case goTransmitBlocked: return false + case goTransmitPaced: + return true } return c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 } @@ -248,9 +251,28 @@ func (c *GoConn) nextTransmitLength() int { } permit := c.sendPermit.Load() if permit <= sent { + if c.permitWindowBound.Load() { + c.windowLimitedSince.Store(true) + } + return 0 + } + packets := int32(c.sendPacketPermit.Load() - c.dataSegmentsOut.Load()) + if packets <= 0 { + c.windowLimitedSince.Store(true) return 0 } - return c.frameLength(int(min(pending, permit-sent))) + length := int(min(pending, permit-sent, uint64(packets)*uint64(c.effectiveMSS))) + if pending < uint64(c.effectiveMSS) && c.writerActive.Load() != 0 && sent > c.sendUnacked.Load() { + return 0 + } + // Windows delays the ACK of a lone segment by 40 ms and its receive window auto-tuning never + // grows while every window is drained by one segment, so a window narrower than two segments is + // always covered by two frames. + allowance := permit - c.sendUnacked.Load() + if allowance < 2*uint64(c.effectiveMSS) && packets > 1 { + length = min(length, int((allowance+1)/2)) + } + return c.frameLength(length) } func (c *GoConn) writeBudget(pending int) int { @@ -334,7 +356,15 @@ func (c *GoConn) parkTransmitBlocked() error { return nil } -func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, bool) { +type goTransmitResult uint8 + +const ( + goTransmitDone goTransmitResult = iota + goTransmitBlocked + goTransmitPaced +) + +func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResult) { if transmitter { defer c.engine.platformIO.flush() } @@ -342,35 +372,69 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, bool) { written := 0 if c.blockedValid { if !transmitter { - return written, true + return written, goTransmitBlocked } if !c.retryBlocked() { - return written, false + return written, goTransmitDone } sentAny = true } + paced := c.congestion.pacing + result := goTransmitDone for c.connState.Load() < goConnStateAborted { if budget > 0 && written >= budget { break } length := c.nextTransmitLength() if length == 0 { + sent := c.sentTail.Load() + pending := c.bufferedTail.Load() - sent + c.checkAppLimited(pending, c.sendPermit.Load(), sent) + if pending > 0 && !c.permitWindowBound.Load() && sent == c.sendUnacked.Load() { + c.armRetransmitIfIdle() + } break } + var now int64 + if paced { + now = c.engine.readClock() + if stamp := c.pacingStamp.Load(); c.dataSegmentsOut.Load() >= goPacingUnpacedSegments && stamp > now+goWheelTick { + c.requestPacing(stamp - goWheelTick) + result = goTransmitPaced + break + } + } written += length sent := c.sentTail.Load() segment := goSegment{offset: sent, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} if length > int(c.effectiveMSS) { segment.gsoSize = c.effectiveMSS } - var flags uint8 - if c.engine.engineState.Load() == goEngineParked { - flags |= goDescriptorNoSample + var flags uint16 + unacked := c.sendUnacked.Load() + flightEmpty := sent == unacked + if flightEmpty { + flags |= goDescriptorTxStart } - c.pushDescriptors(sent, length, c.stamp(c.engine.now()), flags) + if !paced { + now = c.engine.readClock() + } + c.pushDescriptors(sent, length, c.stamp(now), flags, flightEmpty) c.sentTail.Store(sent + uint64(length)) + creditBase := c.packetCreditBase.Load() + flight := uint64(c.dataSegmentsOut.Load() - creditBase) + if flight > c.peakFlight.Load() { + c.peakFlight.Store(flight) + } sentAny = true + if int32(c.sendPacketPermit.Load()-c.dataSegmentsOut.Load()) <= 0 || + sent+uint64(length) >= c.sendPermit.Load() && c.permitWindowBound.Load() { + c.windowLimitedSince.Store(true) + } err := c.transmitFrame(&segment) + if paced && err != errGoTransmitBlocked { + c.advancePacing(now, length) + } switch err { case nil: c.transmittedTail.Store(sent + uint64(length)) @@ -380,24 +444,24 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, bool) { c.blockedValid = true if !transmitter { c.finishTransmit(sentAny) - return written, true + return written, goTransmitBlocked } if !c.retryBlocked() { - return written, false + return written, goTransmitDone } case errGoFrameDropped: c.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) - c.engine.postMessage(&c.droppedMessage) c.transmittedTail.Store(sent + uint64(length)) + c.engine.postMessage(&c.droppedMessage) default: if c.connState.Load() < goConnStateAborted { c.requestAbort(E.Cause(err, "go: write tun")) } - return written, false + return written, goTransmitDone } } c.finishTransmit(sentAny) - return written, false + return written, result } func (c *GoConn) retryBlocked() bool { @@ -415,16 +479,20 @@ func (c *GoConn) retryBlocked() bool { continue case errGoFrameDropped: c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorDropped) - c.engine.postMessage(&c.droppedMessage) default: if c.connState.Load() < goConnStateAborted { c.requestAbort(E.Cause(err, "go: write tun")) } return false } + if c.congestion.pacing { + c.advancePacing(c.engine.readClock(), c.blockedSegment.length) + } c.blockedValid = false - c.blockedFrame = nil c.transmittedTail.Store(end) + if err == errGoFrameDropped { + c.engine.postMessage(&c.droppedMessage) + } return true } } @@ -445,49 +513,94 @@ func (c *GoConn) finishTransmit(sentAny bool) { } } -func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint8) { - c.descriptors.push(offset+uint64(length), int32(length), goDescriptorAmend|flags) +func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint16) { + c.descriptors.push(goSentDescriptor{endOffset: offset + uint64(length), sentAt: int32(length), flags: goDescriptorAmend | flags}) } -func (c *GoConn) pushDescriptors(offset uint64, length int, sentAt int32, flags uint8) { +func (c *GoConn) pushDescriptors(offset uint64, length int, sentAt int32, flags uint16, flightEmpty bool) { + delivered, deliveredStamp, firstSentStamp, appLimited := c.snapshotDelivery(sentAt, flightEmpty) + flags |= goDescriptorRated + if appLimited { + flags |= goDescriptorAppLimited + } step := int(c.effectiveMSS) for length > 0 { span := min(length, step) offset += uint64(span) length -= span - c.descriptors.push(offset, sentAt, flags) + c.dataSegmentsOut.Add(1) + c.descriptors.push(goSentDescriptor{ + endOffset: offset, + sentAt: sentAt, + deliveredStamp: deliveredStamp, + firstSentStamp: firstSentStamp, + delivered: delivered, + flags: flags, + }) + flags &^= goDescriptorTxStart } } func (c *GoConn) frameLength(pending int) int { limit := int(c.effectiveMSS) if c.engine.platformIO.transmitSegmentOffload() { - limit = max(goGSOMaxPayload/limit*limit, limit) + limit = int(c.gsoMaxSize()) + if frameLimit := int(c.frameLimit.Load()); frameLimit > 0 { + limit = min(limit, max(frameLimit, int(c.effectiveMSS))) + } } return min(pending, limit) } -func (c *GoConn) transmitFrame(segment *goSegment) error { - if c.blockedValid && c.blockedFrame != nil && segment.offset == c.blockedSegment.offset && segment.length == c.blockedSegment.length { - return c.engine.platformIO.writeData(c.blockedFrame, c.blockedMeta) +func (c *GoConn) advancePacing(now int64, length int) { + rate := c.pacingRate.Load() + if rate == 0 || c.dataSegmentsOut.Load() < goPacingUnpacedSegments { + c.pacingStamp.Store(now) + return + } + for { + prior := c.pacingStamp.Load() + stamp := max(prior, now) + lengthNanos := int64(uint64(length) * uint64(time.Second) / rate) + credit := stamp - prior + lengthNanos -= min(lengthNanos/2, credit) + if c.pacingStamp.CompareAndSwap(prior, stamp+lengthNanos) { + return + } } - offload := c.engine.platformIO.transmitChecksumOffload() - buffer, payload, contiguous := c.transmitStore.leadingRun(segment.offset, segment.length) +} + +func (c *GoConn) requestPacing(deadline int64) { + c.pacingRequest.Store(deadline) + c.engine.postMessage(&c.pacingMessage) +} + +func (c *GoConn) transmitFrame(segment *goSegment) error { var ( frame [][]byte meta ForwardFrameMeta ) - if contiguous && buffer.Start() >= c.FrontHeadroom() { - var packet []byte - packet, meta = c.buildContiguousFrame(buffer, payload, segment, offload) - c.transmitSegments = append(c.transmitSegments[:0], packet) - frame = c.transmitSegments + if c.blockedValid && c.blockedFrame != nil && segment.offset == c.blockedSegment.offset && segment.length == c.blockedSegment.length { + frame, meta = c.blockedFrame, c.blockedMeta } else { - frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload) - c.transmitSegments = frame + offload := c.engine.platformIO.transmitChecksumOffload() + buffer, payload, contiguous := c.transmitStore.leadingRun(segment.offset, segment.length) + if contiguous && buffer.Start() >= c.FrontHeadroom() { + var packet []byte + packet, meta = c.buildContiguousFrame(buffer, payload, segment, offload) + c.transmitSegments = append(c.transmitSegments[:0], packet) + frame = c.transmitSegments + } else { + frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload) + c.transmitSegments = frame + } } err := c.engine.platformIO.writeData(frame, meta) - if err == errGoTransmitBlocked { + switch err { + case nil, errGoFrameDropped: + c.blockedFrame = nil + clear(c.transmitSegments[:cap(c.transmitSegments)]) + case errGoTransmitBlocked: c.blockedFrame = frame c.blockedMeta = meta } @@ -554,7 +667,7 @@ func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *go }) options := packet[layout.ipHeaderLength+header.TCPMinimumSize:] if layout.timestampLength > 0 { - goEncodeTimestampOption(options, goTimestampAt(c.engine.now()), c.tsRecent.Load()) + goEncodeTimestampOption(options, goTimestampAt(c.engine.readClock()), c.tsRecent.Load()) } if len(layout.sackBlocks) > 0 { goEncodeSackOption(options[layout.timestampLength:], layout.sackBlocks, c.clientISN) @@ -570,7 +683,6 @@ func goNetworkHeaderLength(ipVersion uint8) int { } func goEncodeNetworkHeader(packet []byte, ipVersion uint8, source netip.Addr, destination netip.Addr, tcpLength int, ident uint16) uint16 { - var network header.Network if ipVersion == 4 { ipHdr := header.IPv4(packet) ipHdr.Encode(&header.IPv4Fields{ @@ -582,19 +694,17 @@ func goEncodeNetworkHeader(packet []byte, ipVersion uint8, source netip.Addr, de DstAddr: destination, }) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - network = ipHdr - } else { - ipHdr := header.IPv6(packet) - ipHdr.Encode(&header.IPv6Fields{ - PayloadLength: uint16(tcpLength), - TransportProtocol: header.TCPProtocolNumber, - HopLimit: synthesizedTTL, - SrcAddr: source, - DstAddr: destination, - }) - network = ipHdr - } - return header.PseudoHeaderChecksum(header.TCPProtocolNumber, network.SourceAddressSlice(), network.DestinationAddressSlice(), uint16(tcpLength)) + return header.PseudoHeaderChecksum(header.TCPProtocolNumber, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(tcpLength)) + } + ipHdr := header.IPv6(packet) + ipHdr.Encode(&header.IPv6Fields{ + PayloadLength: uint16(tcpLength), + TransportProtocol: header.TCPProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: source, + DstAddr: destination, + }) + return header.PseudoHeaderChecksum(header.TCPProtocolNumber, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(tcpLength)) } func (c *GoConn) offloadMeta(tcpHdr header.TCP, pseudoSum uint16, layout *goFrameLayout, segment *goSegment) ForwardFrameMeta { @@ -696,7 +806,7 @@ func (c *GoConn) buildSynAck(synOptions header.TCPSynOptions, localMSS uint16) { optionsLength += header.EncodeSACKPermittedOption(optionsStorage[optionsLength:]) } if c.timestampsEnabled { - optionsLength += header.EncodeTSOption(goTimestampAt(c.engine.now()), c.tsRecent.Load(), optionsStorage[optionsLength:]) + optionsLength += header.EncodeTSOption(goTimestampAt(c.engine.readClock()), c.tsRecent.Load(), optionsStorage[optionsLength:]) } optionsLength += header.AddTCPOptionPadding(optionsStorage, optionsLength) tcpHeaderLength := header.TCPMinimumSize + optionsLength @@ -719,17 +829,14 @@ func (c *GoConn) buildSynAck(synOptions header.TCPSynOptions, localMSS uint16) { } func (c *GoConn) writeSynAck() error { - var frame [1][]byte - frame[0] = c.synAckImage[:c.synAckLength] - return goIgnoreDropped(c.engine.platformIO.writeFrame(frame[:], ForwardFrameMeta{})) + return goIgnoreDropped(c.engine.platformIO.writePacket(c.synAckImage[:c.synAckLength], ForwardFrameMeta{})) } func (c *GoConn) writeReset() error { - var scratch [header.IPv6MinimumSize + header.TCPMinimumSize + goTimestampOptionLength]byte var frame [1][]byte segment := goSegment{offset: 0, flags: header.TCPFlagRst | header.TCPFlagAck} - segments, _ := c.buildFrame(scratch[:], frame[:0], &segment, &c.transmitStore, false) - return goIgnoreDropped(c.engine.platformIO.writeFrame(segments, ForwardFrameMeta{})) + segments, _ := c.buildFrame(c.transmitScratch[:], frame[:0], &segment, &c.transmitStore, false) + return goIgnoreDropped(c.engine.platformIO.writePacket(segments[0], ForwardFrameMeta{})) } var errGoTransmitBlocked = E.New("go: transmit blocked") diff --git a/stack_go_tcp_rate.go b/stack_go_tcp_rate.go new file mode 100644 index 00000000..0ec8aa38 --- /dev/null +++ b/stack_go_tcp_rate.go @@ -0,0 +1,120 @@ +package tun + +import "time" + +// Mirrors the delivery rate estimator of net/ipv4/tcp_input.c (formerly tcp_rate.c). + +func goStampMicros(units int32) int64 { + return int64(units) * goTimeUnit / int64(time.Microsecond) +} + +func goStampAfter(stamp1 int32, stamp2 int32, offset1 uint64, offset2 uint64) bool { + delta := stamp1 - stamp2 + return delta > 0 || (delta == 0 && offset1 > offset2) +} + +func goPackDelivery(delivered uint32, stamp int32) uint64 { + return uint64(delivered)<<32 | uint64(uint32(stamp)) +} + +func goUnpackDelivery(state uint64) (uint32, int32) { + return uint32(state >> 32), int32(uint32(state)) +} + +func (c *GoConn) snapshotDelivery(sentAt int32, flightEmpty bool) (uint32, int32, int32, bool) { + if flightEmpty { + for { + state := c.deliveryState.Load() + delivered, _ := goUnpackDelivery(state) + if c.deliveryState.CompareAndSwap(state, goPackDelivery(delivered, sentAt)) { + break + } + } + c.firstSentStamp.Store(sentAt) + } + delivered, deliveredStamp := goUnpackDelivery(c.deliveryState.Load()) + return delivered, deliveredStamp, c.firstSentStamp.Load(), c.appLimited.Load() != 0 +} + +func (e *goEngine) rateDescriptorDelivered(conn *GoConn, descriptor *goSentDescriptor, sample *goRateSample) { + if descriptor.flags&goDescriptorRated == 0 || descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + return + } + if !sample.valid || goStampAfter(descriptor.sentAt, conn.firstSentStamp.Load(), descriptor.endOffset, sample.lastEndOffset) { + sample.valid = true + sample.priorDelivered = descriptor.delivered + sample.priorStamp = descriptor.deliveredStamp + sample.appLimited = descriptor.flags&goDescriptorAppLimited != 0 + sample.retransmitted = descriptor.flags&goDescriptorRetransmitted != 0 + sample.lastEndOffset = descriptor.endOffset + conn.firstSentStamp.Store(descriptor.sentAt) + sample.sendIntervalMicros = goStampMicros(descriptor.sentAt - descriptor.firstSentStamp) + } + if descriptor.flags&goDescriptorSacked != 0 { + descriptor.flags &^= goDescriptorRated + } +} + +func (e *goEngine) rateGenerate(conn *GoConn, sample *goRateSample, delivered uint32, lost uint32, now int64) { + appLimited := conn.appLimited.Load() + if appLimited != 0 && int32(conn.delivered-appLimited) > 0 { + conn.appLimited.CompareAndSwap(appLimited, 0) + } + stamp := conn.stamp(now) + if delivered > 0 { + conn.deliveredTime = now + conn.deliveredStamp = stamp + conn.publishDelivery() + } + sample.ackedSacked = delivered + sample.losses = lost + if !sample.valid || conn.sackReneging { + sample.delivered = -1 + sample.intervalMicros = -1 + return + } + sample.delivered = int64(conn.delivered - sample.priorDelivered) + sendMicros := sample.sendIntervalMicros + receiveMicros := goStampMicros(stamp - sample.priorStamp) + sample.intervalMicros = max(sendMicros, receiveMicros) + sample.receiveIntervalMicros = receiveMicros + if sample.intervalMicros < int64(conn.minRoundTripMicros()) { + sample.intervalMicros = -1 + return + } + if !sample.appLimited || uint64(sample.delivered)*uint64(conn.rateIntervalMicros) >= uint64(conn.rateDelivered)*uint64(sample.intervalMicros) { + conn.rateDelivered = uint32(sample.delivered) + conn.rateIntervalMicros = uint32(sample.intervalMicros) + conn.rateAppLimited = sample.appLimited + } +} + +func (c *GoConn) countDelivered(packets uint32) { + c.delivered += packets + c.deliveredTime = c.engine.now() + c.deliveredStamp = c.stamp(c.deliveredTime) + c.publishDelivery() +} + +func (c *GoConn) publishDelivery() { + for { + state := c.deliveryState.Load() + _, stamp := goUnpackDelivery(state) + if c.deliveredStamp-stamp > 0 { + stamp = c.deliveredStamp + } + if c.deliveryState.CompareAndSwap(state, goPackDelivery(c.delivered, stamp)) { + return + } + } +} + +func (c *GoConn) checkAppLimited(pending uint64, permit uint64, sent uint64) { + segments := c.dataSegmentsOut.Load() + if pending >= uint64(c.effectiveMSS) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerActive.Load() != 0 || c.writerParked.Load() { + return + } + inFlight := segments - c.packetCreditBase.Load() + delivered, _ := goUnpackDelivery(c.deliveryState.Load()) + c.appLimited.Store(max(delivered+inFlight, 1)) +} diff --git a/stack_go_udp.go b/stack_go_udp.go index 4896d969..7550cfbf 100644 --- a/stack_go_udp.go +++ b/stack_go_udp.go @@ -22,6 +22,8 @@ const ( goPacketBatchSize = 32 goUDPGSOType = 5 goUDPChecksumOffset = 6 + // UDP_MAX_SEGMENTS in include/linux/udp.h bounds the segments of one USO packet. + goGSOMaxSegments = 128 ) type goUDPFrame struct { @@ -31,6 +33,12 @@ type goUDPFrame struct { meta ForwardFrameMeta } +var goUDPFramePool = sync.Pool{ + New: func() any { + return new([goPacketBatchSize]goUDPFrame) + }, +} + type goUDPUserData struct { engine *goEngine owner *goEngine @@ -50,30 +58,40 @@ func (e *goEngine) processUDPSegments(frame *goFrame) { return } count := (len(packet) - headerLength + int(meta.gsoSize) - 1) / int(meta.gsoSize) - if count == 0 { + if count == 0 || count > goGSOMaxSegments || headerLength+int(meta.gsoSize) > e.stack.mtu { return } options := *e.packetReadOptions.Load() - buffers := newGoReadBuffers(headerLength+int(meta.gsoSize), count, options) - defer buf.ReleaseMulti(buffers) - segments := make([][]byte, count) - sizes := make([]int, count) - for index, buffer := range buffers { - segments[index] = buffer.FreeBytes() + if e.gsoReadOptions != options { + buf.ReleaseMulti(e.gsoBuffers[:]) + clear(e.gsoBuffers[:]) + e.gsoReadOptions = options + } + for index := range count { + buffer := e.gsoBuffers[index] + if buffer == nil { + buffer = options.NewBufferSize(e.stack.mtu) + e.gsoBuffers[index] = buffer + } + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + e.gsoSegments[index] = buffer.FreeBytes() } count, err := GSOSplit(packet, GSOOptions{ GSOType: GSOUDPL4, HdrLen: uint16(headerLength), GSOSize: meta.gsoSize, CsumStart: meta.checksumStart, CsumOffset: meta.checksumOffset, NeedsCsum: meta.needsChecksum, - }, segments, sizes, 0) + }, e.gsoSegments[:count], e.gsoSizes[:count], 0) if err != nil { e.stack.logger.Trace(E.Cause(err, "go: split UDP segments")) return } for index := range count { - buffer := buffers[index] - buffer.Truncate(sizes[index]) + buffer := e.gsoBuffers[index] + buffer.Truncate(e.gsoSizes[index]) options.PostReturn(buffer) - e.processFrame(&goFrame{buffer: buffer}) + e.gsoFrame = goFrame{buffer: buffer} + e.processFrame(&e.gsoFrame) } e.flushPacketUploads() } @@ -345,8 +363,12 @@ func (w *GoPacketConn) WritePacketBatch(buffers []*buf.Buffer, destinations []M. if len(buffers) == 0 || len(buffers) != len(destinations) { return os.ErrInvalid } - var frames [goPacketBatchSize]goUDPFrame + frames := goUDPFramePool.Get().(*[goPacketBatchSize]goUDPFrame) count := 0 + defer func() { + clear(frames[:count]) + goUDPFramePool.Put(frames) + }() for index, buffer := range buffers { if !destinations[index].IsIP() { return os.ErrInvalid @@ -404,7 +426,7 @@ func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) err } packet := buffer.Bytes() if !fragmented { - return goIgnoreDropped(w.platformIO.writeFrame([][]byte{packet}, meta)) + return goIgnoreDropped(w.platformIO.writePacket(packet, meta)) } var ( fragments [][]byte @@ -424,7 +446,10 @@ func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) err } var writeErr error for _, fragment := range fragments { - writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeFrame([][]byte{fragment}, ForwardFrameMeta{}))) + err = goIgnoreDropped(w.platformIO.writePacket(fragment, ForwardFrameMeta{})) + if err != nil { + writeErr = E.Errors(writeErr, err) + } } return writeErr } @@ -450,7 +475,7 @@ func (w *GoPacketConn) HandshakeFailure(err error) error { if !ok { return nil } - return goIgnoreDropped(w.platformIO.writeFrame([][]byte{reply}, ForwardFrameMeta{})) + return goIgnoreDropped(w.platformIO.writePacket(reply, ForwardFrameMeta{})) } func (w *GoPacketConn) closeSplice(err error) { diff --git a/tun_windows.go b/tun_windows.go index 29ad40e9..82cd29e6 100644 --- a/tun_windows.go +++ b/tun_windows.go @@ -381,10 +381,9 @@ retry: if t.close.Load() == 1 { return 0, os.ErrClosed } - var packet []byte - packet, err = t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: n = copy(p, packet) t.session.ReleaseReceivePacket(packet) t.rate.update(uint64(n)) @@ -401,7 +400,7 @@ retry: case windows.ERROR_INVALID_DATA: return 0, errors.New("send ring corrupt") } - return 0, fmt.Errorf("read failed: %w", err) + return 0, fmt.Errorf("read failed: %w", errno) } } @@ -434,9 +433,9 @@ retry: t.running.Done() return nil, nil, os.ErrClosed } - packet, err := t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: packetSize := len(packet) t.rate.update(uint64(packetSize)) return packet, func() { @@ -458,7 +457,7 @@ retry: return nil, nil, errors.New("send ring corrupt") } t.running.Done() - return nil, nil, fmt.Errorf("read failed: %w", err) + return nil, nil, fmt.Errorf("read failed: %w", errno) } } @@ -475,9 +474,9 @@ retry: if t.close.Load() == 1 { return os.ErrClosed } - packet, err := t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: packetSize := len(packet) block(packet) t.session.ReleaseReceivePacket(packet) @@ -495,7 +494,7 @@ retry: case windows.ERROR_INVALID_DATA: return errors.New("send ring corrupt") } - return fmt.Errorf("read failed: %w", err) + return fmt.Errorf("read failed: %w", errno) } } @@ -506,19 +505,19 @@ func (t *NativeTun) Write(p []byte) (n int, err error) { return 0, os.ErrClosed } t.rate.update(uint64(len(p))) - packet, err := t.session.AllocateSendPacket(len(p)) + packet, errno := t.session.AllocateSendPacket(len(p)) copy(packet, p) - if err == nil { + if errno == 0 { t.session.SendPacket(packet) return len(p), nil } - switch err { + switch errno { case windows.ERROR_HANDLE_EOF: return 0, os.ErrClosed case windows.ERROR_BUFFER_OVERFLOW: return 0, nil // Dropping when ring is full. } - return 0, fmt.Errorf("write failed: %w", err) + return 0, fmt.Errorf("write failed: %w", errno) } func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { @@ -532,8 +531,8 @@ func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { packetSize += len(packetElement) } t.rate.update(uint64(packetSize)) - packet, err := t.session.AllocateSendPacket(packetSize) - if err == nil { + packet, errno := t.session.AllocateSendPacket(packetSize) + if errno == 0 { var index int for _, packetElement := range packetElementList { index += copy(packet[index:], packetElement) @@ -541,32 +540,19 @@ func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { t.session.SendPacket(packet) return } - switch err { + switch errno { case windows.ERROR_HANDLE_EOF: return 0, os.ErrClosed case windows.ERROR_BUFFER_OVERFLOW: return 0, nil // Dropping when ring is full. } - return 0, fmt.Errorf("write failed: %w", err) + return 0, fmt.Errorf("write failed: %w", errno) } func (t *NativeTun) readWaitHandle() windows.Handle { return t.readWait } -func (t *NativeTun) resizeSessionRing(capacity uint32) error { - t.session.End() - t.session = wintun.Session{} - t.readWait = 0 - session, err := t.adapter.StartSession(capacity) - if err != nil { - return E.Cause(err, "restart wintun session") - } - t.session = session - t.readWait = session.ReadWaitEvent() - return nil -} - func (t *NativeTun) receiveInto(buffer []byte) (int, error) { t.running.Add(1) defer t.running.Done() @@ -574,9 +560,9 @@ func (t *NativeTun) receiveInto(buffer []byte) (int, error) { if t.close.Load() == 1 { return 0, os.ErrClosed } - packet, err := t.session.ReceivePacket() - if err != nil { - switch err { + packet, errno := t.session.ReceivePacket() + if errno != 0 { + switch errno { case windows.ERROR_NO_MORE_ITEMS: return 0, nil case windows.ERROR_HANDLE_EOF: @@ -584,7 +570,7 @@ func (t *NativeTun) receiveInto(buffer []byte) (int, error) { case windows.ERROR_INVALID_DATA: return 0, E.New("wintun: receive ring corrupt") } - return 0, E.Cause(err, "wintun: receive packet") + return 0, E.Cause(errno, "wintun: receive packet") } if len(packet) > len(buffer) { t.session.ReleaseReceivePacket(packet) @@ -606,12 +592,12 @@ func (t *NativeTun) transmitGather(segments [][]byte) error { for _, segment := range segments { packetSize += len(segment) } - packet, err := t.session.AllocateSendPacket(packetSize) - if err != nil { - if err == windows.ERROR_HANDLE_EOF { + packet, errno := t.session.AllocateSendPacket(packetSize) + if errno != 0 { + if errno == windows.ERROR_HANDLE_EOF { return os.ErrClosed } - return err + return errno } var index int for _, segment := range segments { From 7a7eec32b3c22187f9ab3bf0feec9b9e9458a5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 15 Sep 2026 20:31:58 +0800 Subject: [PATCH 09/25] Mark Android bypass routes as static to keep them out of NetworkStack --- redirect_route_bypass_android.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/redirect_route_bypass_android.go b/redirect_route_bypass_android.go index 67868673..5dd79b03 100644 --- a/redirect_route_bypass_android.go +++ b/redirect_route_bypass_android.go @@ -270,6 +270,9 @@ func buildBypassRoutes(family int, set *netipx.IPSet, sources []netlink.Route, t route := source route.Dst = prefixToIPNet(prefix) route.Table = tableIndex + // NetworkStack IpClientLinkObserver imports every kernel and ra + // route on the interface into LinkProperties regardless of table. + route.Protocol = unix.RTPROT_STATIC routes = append(routes, route) } var remainingBuilder netipx.IPSetBuilder From 9d17d934245b94ac8b6cbe92a33c9c363aa27a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 15 Sep 2026 18:47:00 +0800 Subject: [PATCH 10/25] Rework go conn concurrency --- stack_go_conn.go | 449 +++++++++++------------- stack_go_engine.go | 18 +- stack_go_integration_cc_test.go | 2 +- stack_go_integration_flow_linux_test.go | 26 +- stack_go_integration_tcp_test.go | 6 +- stack_go_io_darwin.go | 4 +- stack_go_io_linux.go | 2 +- stack_go_io_other.go | 2 +- stack_go_io_windows.go | 2 +- stack_go_ring.go | 2 +- stack_go_splice.go | 114 +++--- stack_go_tcp_ack.go | 20 +- stack_go_tcp_input.go | 217 +++++++----- stack_go_tcp_output.go | 376 +++++++++++++------- stack_go_tcp_rate.go | 8 +- stack_go_udp.go | 8 +- 16 files changed, 681 insertions(+), 575 deletions(-) diff --git a/stack_go_conn.go b/stack_go_conn.go index 107884ec..a7ee305a 100644 --- a/stack_go_conn.go +++ b/stack_go_conn.go @@ -18,11 +18,9 @@ import ( ) const ( - goConnStateJudged uint32 = iota - goConnStateEngaged - goConnStateEstablished - goConnStateAborted - goConnStateDead + goPhaseJudged uint8 = iota + goPhaseEngaged + goPhaseEstablished ) const ( @@ -36,12 +34,6 @@ const ( goTCPClosed ) -const ( - goCloseModeNone uint32 = iota - goCloseModeGraceful - goCloseModeAbort -) - const ( goDeathImmediate uint8 = iota goDeathFinLinger @@ -52,6 +44,7 @@ const ( goTimeUnit = 4096 goTimestampShift = 20 goLocalWindowShift = 7 + goPermitWindowBit = 1 << 63 ) var errGoReset = E.Cause(syscall.ECONNRESET, "go: connection reset by peer") @@ -63,9 +56,10 @@ type goPostedReceive struct { filled int } -type goConnError struct { - err error -} +var ( + goTargetTaken goPostedReceive + goTargetFilled goPostedReceive +) type GoConn struct { engine *goEngine @@ -77,6 +71,7 @@ type GoConn struct { receiveCapacity uint64 receiveCapacityMax uint64 publishedEdge uint64 + sentEdge uint64 receiveRoundTripMark uint64 receiveRoundTripStamp int64 receiveSpaceConsumed uint64 @@ -182,7 +177,6 @@ type GoConn struct { discardReceive bool finSent bool finAcked bool - finReceived bool ackForced bool ackDirty bool onBlockedList bool @@ -194,99 +188,94 @@ type GoConn struct { ackNext *GoConn blockedNext *GoConn dyingNext *GoConn + splice *goSpliceStream synAckImage [96]byte _ [64]byte - sendPermit atomic.Uint64 - sendPacketPermit atomic.Uint32 - packetCreditBase atomic.Uint32 - sendUnacked atomic.Uint64 - sendReleased atomic.Uint64 - receiveNextAck atomic.Uint64 - receiveEdge atomic.Uint64 - sentEdge atomic.Uint64 - receiveAvailable atomic.Uint64 - receiveCapacityPublished atomic.Uint64 - windowUpdateThreshold atomic.Uint64 - deliveryState atomic.Uint64 - pacingRate atomic.Uint64 - peakFlight atomic.Uint64 - pacingStamp atomic.Int64 - pacingRequest atomic.Int64 - firstSentStamp atomic.Int32 - appLimited atomic.Uint32 - frameLimit atomic.Uint32 - tsRecent atomic.Uint32 - permitWindowBound atomic.Bool - windowLimitedSince atomic.Bool - ident atomic.Uint32 - connState atomic.Uint32 - receiveShutdown atomic.Bool - everEstablished atomic.Bool - receiveDrainable atomic.Bool - receiveReleased atomic.Bool - _ [64]byte - - sentTail atomic.Uint64 - transmittedTail atomic.Uint64 - bufferedTail atomic.Uint64 - writerParked atomic.Bool - retransmitArmed atomic.Bool - writerActive atomic.Int32 - flushActive atomic.Int32 - writerNeeds atomic.Int32 - transmitterActive atomic.Int32 - transmitOwner atomic.Int32 - dataSegmentsOut atomic.Uint32 - _ [64]byte - - consumedTail atomic.Uint64 - readerParked atomic.Bool - readerActive atomic.Int32 - postedTarget atomic.Pointer[goPostedReceive] - targetDone atomic.Bool - _ [64]byte - - transmitStore goTransmitStore - slabHolder goSlabHolder - transmitScratch [goHeaderScratchSize]byte - transmitSegments [][]byte - descriptors goDescriptorRing - blockedSegment goSegment - blockedFrame [][]byte - blockedMeta ForwardFrameMeta - blockedValid bool - + sendPermit atomic.Uint64 + sendPacketPermit atomic.Uint32 + packetCreditBase atomic.Uint32 + sendUnacked atomic.Uint64 + sendReleased atomic.Uint64 + receiveNextAck atomic.Uint64 + receiveEdge atomic.Uint64 + receiveAvailable atomic.Uint64 + windowUpdateAt atomic.Uint64 + deliveryState atomic.Uint64 + pacingRate atomic.Uint64 + pacingStamp atomic.Int64 + firstSentStamp atomic.Int32 + appLimited atomic.Uint32 + frameLimit atomic.Uint32 + tsRecent atomic.Uint32 + _ [64]byte + + transmitAccess sync.Mutex + sentTail atomic.Uint64 + transmittedTail atomic.Uint64 + dataSentEdge atomic.Uint64 + peakFlight atomic.Uint64 + pacingRequest atomic.Int64 + dataSegmentsOut atomic.Uint32 + ident atomic.Uint32 + windowLimitedSince atomic.Bool + transmitStore goTransmitStore + descriptors goDescriptorRing + transmitScratch [goHeaderScratchSize]byte + transmitSegments [][]byte + blockedSegment goSegment + blockedFrame [][]byte + blockedMeta ForwardFrameMeta + blockedValid bool + _ [64]byte + + writeAccess sync.Mutex + bufferedTail atomic.Uint64 + writerWaiting atomic.Int32 + writeDeadline pipe.Deadline + writeSignal goSignal + _ [64]byte + + readAccess sync.Mutex + consumedTail atomic.Uint64 + readerParked atomic.Bool + postedTarget atomic.Pointer[goPostedReceive] receiveTarget goPostedReceive readWaitOptions N.ReadWaitOptions readDeadline pipe.Deadline - writeDeadline pipe.Deadline - - writeAccess sync.Mutex - readAccess sync.Mutex - readSignal goSignal - writeSignal goSignal - transmitSignal goSignal - establishedSignal chan struct{} - closeSignal chan struct{} - connErr atomic.Pointer[goConnError] - closeMode atomic.Uint32 - writeShut atomic.Bool - readShut atomic.Bool - userClosed atomic.Bool - finRequested atomic.Bool - engageMessage goMessage - closeMessage goMessage - readShutMessage goMessage - retransmitMessage goMessage - windowMessage goMessage - blockedMessage goMessage - droppedMessage goMessage - pacingMessage goMessage - spliceMessage goMessage - splicePending atomic.Pointer[goSpliceStream] - spliced atomic.Bool - splice *goSpliceStream + readSignal goSignal + _ [64]byte + + access sync.Mutex + phase uint8 + dead bool + finReceived bool + drainable bool + released bool + spliced bool + userClosed bool + writeShut bool + readShut bool + finRequested bool + abort bool + writing bool + transmitterRunning bool + err error + splicePending *goSpliceStream + establishedSignal chan struct{} + closeSignal chan struct{} + transmitSignal goSignal + engageMessage goMessage + closeMessage goMessage + readShutMessage goMessage + retransmitMessage goMessage + windowMessage goMessage + transmitMessage goMessage + blockedMessage goMessage + droppedMessage goMessage + pacingMessage goMessage + spliceMessage goMessage + slabHolder goSlabHolder } func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, destination M.Socksaddr) { @@ -312,6 +301,7 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, d c.readShutMessage = goMessage{kind: goMessageConnReadShut, conn: c} c.retransmitMessage = goMessage{kind: goMessageConnRetransmit, conn: c} c.windowMessage = goMessage{kind: goMessageConnWindow, conn: c} + c.transmitMessage = goMessage{kind: goMessageConnTransmit, conn: c} c.blockedMessage = goMessage{kind: goMessageConnBlocked, conn: c} c.droppedMessage = goMessage{kind: goMessageConnDropped, conn: c} c.pacingMessage = goMessage{kind: goMessageConnPacing, conn: c} @@ -337,81 +327,39 @@ func (c *GoConn) stamp(now int64) int32 { return int32((now - c.epoch) / goTimeUnit) } -func (c *GoConn) storeError(err error) { - c.connErr.CompareAndSwap(nil, &goConnError{err: err}) -} - -func (c *GoConn) loadError() error { - value := c.connErr.Load() - if value == nil { - return nil - } - return value.err +func (c *GoConn) closed() bool { + c.access.Lock() + dead := c.dead + c.access.Unlock() + return dead } func (c *GoConn) closeError() error { - err := c.loadError() + c.access.Lock() + err := c.err + c.access.Unlock() if err != nil { return err } return net.ErrClosed } -func (c *GoConn) enterWriter() bool { - c.writerActive.Add(1) - if c.connState.Load() >= goConnStateAborted { - c.writerActive.Add(-1) - c.engine.reapAfterExit() - return false - } - return true -} - -func (c *GoConn) exitWriter() { - c.flushActive.Add(1) - c.writerActive.Add(-1) - if c.connState.Load() < goConnStateAborted { - sent := c.sentTail.Load() - if c.bufferedTail.Load() > sent { - c.transmitInline() - } else { - c.checkAppLimited(0, c.sendPermit.Load(), sent) - } - } - c.flushActive.Add(-1) - c.engine.reapAfterExit() - if c.finRequested.Load() { - c.engine.postMessage(&c.closeMessage) - } -} - -func (c *GoConn) enterReader() bool { - c.readerActive.Add(1) - state := c.connState.Load() - if state < goConnStateAborted { - return true - } - if state == goConnStateDead && c.receiveDrainable.Load() && !c.receiveReleased.Load() { - return true +func (c *GoConn) fail(err error) { + c.access.Lock() + if c.err == nil { + c.err = err } - c.exitReader() - return false -} - -func (c *GoConn) exitReader() { - c.readerActive.Add(-1) - c.engine.reapAfterExit() -} - -func (c *GoConn) requestAbort(err error) { - c.storeError(err) - c.closeMode.Store(goCloseModeAbort) + c.abort = true + c.access.Unlock() c.engine.postMessage(&c.closeMessage) } func (c *GoConn) awaitHandshake(ctx context.Context, deadlineSignal <-chan struct{}) error { - state := c.connState.Load() - if state >= goConnStateEstablished { + c.access.Lock() + phase := c.phase + dead := c.dead + c.access.Unlock() + if phase == goPhaseEstablished || dead { return c.handshakeResult() } select { @@ -419,12 +367,7 @@ func (c *GoConn) awaitHandshake(ctx context.Context, deadlineSignal <-chan struc return os.ErrDeadlineExceeded default: } - if c.connState.CompareAndSwap(goConnStateJudged, goConnStateEngaged) { - err := c.writeSynAck() - if err != nil { - c.requestAbort(E.Cause(err, "go: send SYN-ACK")) - return err - } + if phase == goPhaseJudged { c.engine.postMessage(&c.engageMessage) } select { @@ -433,42 +376,42 @@ func (c *GoConn) awaitHandshake(ctx context.Context, deadlineSignal <-chan struc case <-deadlineSignal: return os.ErrDeadlineExceeded case <-ctx.Done(): - c.requestAbort(ctx.Err()) + c.fail(ctx.Err()) return E.Cause(ctx.Err(), "go: handshake") case <-c.engine.stack.ctx.Done(): cause := context.Cause(c.engine.stack.ctx) - c.requestAbort(cause) + c.fail(cause) return E.Cause(cause, "go: handshake") } } func (c *GoConn) handshakeResult() error { - err := c.loadError() - if err != nil { - return err - } - if c.everEstablished.Load() { - return nil + c.access.Lock() + defer c.access.Unlock() + if c.err != nil { + return c.err } - if c.connState.Load() >= goConnStateAborted { + if c.dead && c.phase != goPhaseEstablished { return net.ErrClosed } return nil } func (c *GoConn) HandshakeFailure(err error) error { - if c.connState.CompareAndSwap(goConnStateJudged, goConnStateAborted) { - c.storeError(err) - writeErr := c.writeReset() - c.closeMode.Store(goCloseModeAbort) - c.engine.postMessage(&c.closeMessage) - return writeErr - } - if c.connState.Load() >= goConnStateAborted { + c.access.Lock() + if c.dead { + c.access.Unlock() return os.ErrInvalid } - c.userClosed.Store(true) - c.requestAbort(err) + c.userClosed = true + c.writeShut = true + c.readShut = true + c.abort = true + if c.err == nil { + c.err = err + } + c.access.Unlock() + c.engine.postMessage(&c.closeMessage) return nil } @@ -477,11 +420,15 @@ func (c *GoConn) HandshakeSuccess() error { } func (c *GoConn) NeedHandshakeForRead() bool { - return c.connState.Load() < goConnStateEstablished + c.access.Lock() + defer c.access.Unlock() + return c.phase != goPhaseEstablished } func (c *GoConn) NeedHandshakeForWrite() bool { - return c.connState.Load() < goConnStateEstablished + c.access.Lock() + defer c.access.Unlock() + return c.phase != goPhaseEstablished } func (c *GoConn) InitializeReadWaiter(options N.ReadWaitOptions) bool { @@ -495,11 +442,16 @@ func (c *GoConn) WaitReadBuffer() (*buf.Buffer, error) { return nil, err } c.readAccess.Lock() - defer c.readAccess.Unlock() - if !c.enterReader() { + buffer, err := c.waitReadBufferLocked() + c.readAccess.Unlock() + c.engine.reapAfterExit() + return buffer, err +} + +func (c *GoConn) waitReadBufferLocked() (*buf.Buffer, error) { + if !c.readable() { return nil, c.receiveResult() } - defer c.exitReader() for { select { case <-c.readDeadline.Wait(): @@ -547,11 +499,16 @@ func (c *GoConn) Read(p []byte) (int, error) { return 0, nil } c.readAccess.Lock() - defer c.readAccess.Unlock() - if !c.enterReader() { + n, err := c.readLocked(p) + c.readAccess.Unlock() + c.engine.reapAfterExit() + return n, err +} + +func (c *GoConn) readLocked(p []byte) (int, error) { + if !c.readable() { return 0, c.receiveResult() } - defer c.exitReader() for { select { case <-c.readDeadline.Wait(): @@ -584,18 +541,22 @@ func (c *GoConn) Read(p []byte) (int, error) { } } +func (c *GoConn) readable() bool { + c.access.Lock() + defer c.access.Unlock() + return !c.dead || (c.drainable && !c.released) +} + func (c *GoConn) receiveResult() error { - err := c.loadError() - if err != nil { - return err + c.access.Lock() + defer c.access.Unlock() + if c.err != nil { + return c.err } - if c.userClosed.Load() { + if c.userClosed { return net.ErrClosed } - if c.readShut.Load() || c.receiveShutdown.Load() { - return io.EOF - } - if c.connState.Load() >= goConnStateAborted { + if c.readShut || c.finReceived || c.dead { return io.EOF } return nil @@ -613,28 +574,11 @@ func (c *GoConn) copyReceived(target []byte, consumed uint64, available uint64) } func (c *GoConn) advanceConsumed(n int) { - if c.windowUpdateDue(c.consumedTail.Add(uint64(n))) { + if c.consumedTail.Add(uint64(n)) >= c.windowUpdateAt.Load() { c.engine.postMessage(&c.windowMessage) } } -func (c *GoConn) windowUpdateDue(consumed uint64) bool { - edge := c.sentEdge.Load() - capacity := c.receiveCapacityPublished.Load() - free := consumed + capacity - if free <= edge || free-edge < c.windowUpdateThreshold.Load() { - return false - } - available := c.receiveAvailable.Load() - if available < edge { - window := edge - available - if 2*window > capacity || free-available < 2*window { - return false - } - } - return true -} - func (c *GoConn) parkReader() (bool, error) { target := &c.receiveTarget target.filled = 0 @@ -643,7 +587,6 @@ func (c *GoConn) parkReader() (bool, error) { } else { target.target = target.direct } - c.targetDone.Store(false) c.readSignal.drain() c.postedTarget.Store(target) c.readerParked.Store(true) @@ -679,40 +622,35 @@ func (c *GoConn) claimTarget(target *goPostedReceive) bool { target.target = nil return false } - for !c.targetDone.Load() { + for c.postedTarget.Load() != &goTargetFilled { <-c.readSignal } + c.postedTarget.Store(nil) target.target = nil return true } func (c *GoConn) Close() error { - if c.connState.CompareAndSwap(goConnStateJudged, goConnStateAborted) { - c.storeError(net.ErrClosed) - writeErr := c.writeReset() - c.closeMode.Store(goCloseModeAbort) - c.engine.postMessage(&c.closeMessage) - return writeErr - } - if !c.userClosed.CompareAndSwap(false, true) { - return nil - } - c.writeShut.Store(true) - c.readShut.Store(true) - if c.connState.Load() == goConnStateEngaged || c.splicePending.Load() != nil || c.spliced.Load() { - c.requestAbort(net.ErrClosed) + c.access.Lock() + if c.userClosed { + c.access.Unlock() return nil } - if c.connState.Load() >= goConnStateAborted { - c.engine.postMessage(&c.closeMessage) - c.wakeUser() - return nil - } - if c.receiveAvailable.Load() > c.consumedTail.Load() { - c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeAbort) + c.userClosed = true + c.writeShut = true + c.readShut = true + graceful := !c.dead && !c.abort && c.phase == goPhaseEstablished && !c.spliced && c.splicePending == nil && + c.receiveAvailable.Load() <= c.consumedTail.Load() + if graceful { + c.finRequested = true } else { - c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeGraceful) - c.finRequested.Store(true) + c.abort = true + if c.err == nil { + c.err = net.ErrClosed + } + } + c.access.Unlock() + if graceful { c.engine.postMessage(&c.readShutMessage) } c.engine.postMessage(&c.closeMessage) @@ -721,22 +659,27 @@ func (c *GoConn) Close() error { } func (c *GoConn) CloseRead() error { - if c.connState.Load() >= goConnStateAborted { + c.access.Lock() + if c.dead { + c.access.Unlock() return nil } - c.readShut.Store(true) + c.readShut = true + c.access.Unlock() c.engine.postMessage(&c.readShutMessage) c.readSignal.notify() return nil } func (c *GoConn) CloseWrite() error { - if c.connState.Load() >= goConnStateAborted { + c.access.Lock() + if c.dead { + c.access.Unlock() return nil } - c.writeShut.Store(true) - c.closeMode.CompareAndSwap(goCloseModeNone, goCloseModeGraceful) - c.finRequested.Store(true) + c.writeShut = true + c.finRequested = true + c.access.Unlock() c.engine.postMessage(&c.closeMessage) c.writeSignal.notify() return nil diff --git a/stack_go_engine.go b/stack_go_engine.go index 6d8caf72..da4e63dd 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -33,6 +33,7 @@ const ( goMessageConnReadShut goMessageConnRetransmit goMessageConnWindow + goMessageConnTransmit goMessageConnBlocked goMessageConnDropped goMessageConnPacing @@ -447,6 +448,8 @@ func (e *goEngine) handleMessage(message *goMessage) { e.handleRetransmitArm(message.conn) case goMessageConnWindow: e.handleWindowUpdate(message.conn) + case goMessageConnTransmit: + e.handleTransmitRequest(message.conn) case goMessageConnBlocked: e.handleTransmitBlocked(message.conn) case goMessageConnDropped: @@ -500,7 +503,16 @@ func (e *goEngine) releaseInjected() { } func (e *goEngine) handleEngage(conn *GoConn) { - if conn.connState.Load() != goConnStateEngaged { + if conn.dead || conn.phase != goPhaseJudged { + return + } + conn.access.Lock() + conn.phase = goPhaseEngaged + conn.access.Unlock() + err := conn.writeSynAck() + if err != nil { + e.sendReset(conn) + e.detachConn(conn, E.Cause(err, "go: send SYN-ACK"), goDeathAbortLinger) return } conn.handshakeAttempts = 0 @@ -517,7 +529,9 @@ func (e *goEngine) shutdown() { unreset := e.closeAllFlows() for conn := e.dyingList; conn != nil; conn = conn.dyingNext { e.spliceDetach(conn, net.ErrClosed) - conn.receiveDrainable.Store(false) + conn.access.Lock() + conn.drainable = false + conn.access.Unlock() } deadline := e.now() + int64(goShutdownBudget) for len(unreset) > 0 { diff --git a/stack_go_integration_cc_test.go b/stack_go_integration_cc_test.go index 25112143..bd414138 100644 --- a/stack_go_integration_cc_test.go +++ b/stack_go_integration_cc_test.go @@ -101,7 +101,7 @@ func TestGoKernelTCPLifecycle(t *testing.T) { } } traffic.await(test, conn, "graceful close", func(flow kernelTCPFlow) bool { - return flow.fin != 0 && flow.acked >= flow.fin && flow.peerFIN && conn.connState.Load() >= goConnStateAborted + return flow.fin != 0 && flow.acked >= flow.fin && flow.peerFIN && conn.closed() }) }) } diff --git a/stack_go_integration_flow_linux_test.go b/stack_go_integration_flow_linux_test.go index fd19169c..3fb71899 100644 --- a/stack_go_integration_flow_linux_test.go +++ b/stack_go_integration_flow_linux_test.go @@ -106,13 +106,13 @@ func TestGoKernelReceiveWindowUpdateLoss(t *testing.T) { }() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { - if server.sentEdge.Load()-server.receiveAvailable.Load() < 1<= 1<= 1< server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() > server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { break } time.Sleep(time.Millisecond) } - if server.sendPermit.Load() != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + if server.sendPermit.Load()&^goPermitWindowBit != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { test.Fatal("window did not shrink below outstanding data") } completed := make(chan kernelIOResult, 1) @@ -318,10 +318,10 @@ func TestGoKernelWindowLimitedRecovery(t *testing.T) { completed <- kernelIOResult{n: n + initial, err: writeErr} }() deadline = time.Now().Add(time.Second) - for !server.writerParked.Load() && time.Now().Before(deadline) { + for server.writerWaiting.Load() == 0 && time.Now().Before(deadline) { time.Sleep(time.Millisecond) } - if !server.writerParked.Load() { + if server.writerWaiting.Load() == 0 { test.Fatal("write did not block at the closed window") } server.SetWriteDeadline(time.Now()) @@ -403,14 +403,14 @@ func TestGoKernelZeroWindow(t *testing.T) { } deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { - if server.sendPermit.Load() == server.sendUnacked.Load() && server.sentTail.Load() == server.bufferedTail.Load() { + if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() == server.bufferedTail.Load() { break } time.Sleep(time.Millisecond) } unacked := server.sendUnacked.Load() - if server.sendPermit.Load() != unacked || server.sentTail.Load() != uint64(len(payload)+1) || unacked <= 1 || unacked >= server.sentTail.Load() { - test.Fatalf("window did not close with only outstanding data: unacked=%d sent=%d buffered=%d permit=%d", unacked, server.sentTail.Load(), server.bufferedTail.Load(), server.sendPermit.Load()) + if server.sendPermit.Load()&^goPermitWindowBit != unacked || server.sentTail.Load() != uint64(len(payload)+1) || unacked <= 1 || unacked >= server.sentTail.Load() { + test.Fatalf("window did not close with only outstanding data: unacked=%d sent=%d buffered=%d permit=%d", unacked, server.sentTail.Load(), server.bufferedTail.Load(), server.sendPermit.Load()&^goPermitWindowBit) } windowUpdate := func(event kernelTCPEvent) bool { return !event.outgoing && event.flags == header.TCPFlagAck && event.window > 0 diff --git a/stack_go_integration_tcp_test.go b/stack_go_integration_tcp_test.go index cdc425af..3516a081 100644 --- a/stack_go_integration_tcp_test.go +++ b/stack_go_integration_tcp_test.go @@ -266,7 +266,7 @@ func (h *kernelTCPHarness) inspectLocked(platform *kernelTCPIO) { continue } flow.processed = conn.sendUnacked.Load() - flow.closed = conn.connState.Load() >= goConnStateDead + flow.closed = conn.closed() if flow.closed || flow.failure != "" { continue } @@ -307,7 +307,7 @@ func (h *kernelTCPHarness) await(t *testing.T, conn *GoConn, description string, case <-h.changed: case <-tick.C: case <-timer.C: - t.Fatalf("%s timed out: sent=%d ACK=%d processed=%d FIN=%d peerFIN=%v connection=%d; recent TCP events: %+v", description, flow.sent, flow.acked, flow.processed, flow.fin, flow.peerFIN, conn.connState.Load(), flow.events[max(0, len(flow.events)-16):]) + t.Fatalf("%s timed out: sent=%d ACK=%d processed=%d FIN=%d peerFIN=%v closed=%v; recent TCP events: %+v", description, flow.sent, flow.acked, flow.processed, flow.fin, flow.peerFIN, conn.closed(), flow.events[max(0, len(flow.events)-16):]) } } } @@ -462,7 +462,7 @@ func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, pac } p.harness.access.Lock() defer p.harness.access.Unlock() - if p.harness.stopped || flow.conn.connState.Load() >= goConnStateDead { + if p.harness.stopped || flow.conn.closed() { return } event.at = time.Since(p.harness.epoch) diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go index b15d09e8..1a459b16 100644 --- a/stack_go_io_darwin.go +++ b/stack_go_io_darwin.go @@ -18,8 +18,8 @@ import ( ) const ( - goWakeIdent = 0 - goEngineInlineTransmit = true + goWakeIdent = 0 + goEngineTransmits = true goDarwinTransmitBatch = 64 goDarwinBatchHeader = 128 diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go index 746f0b1e..494731f8 100644 --- a/stack_go_io_linux.go +++ b/stack_go_io_linux.go @@ -18,7 +18,7 @@ import ( "golang.org/x/sys/unix" ) -const goEngineInlineTransmit = false +const goEngineTransmits = false var goEmptyVirtioHeader [virtioNetHdrLen]byte diff --git a/stack_go_io_other.go b/stack_go_io_other.go index 146fffc3..2f3f1363 100644 --- a/stack_go_io_other.go +++ b/stack_go_io_other.go @@ -9,7 +9,7 @@ import ( E "github.com/sagernet/sing/common/exceptions" ) -const goEngineInlineTransmit = false +const goEngineTransmits = false func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { return nil, E.New("go: unsupported platform") diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index b4e953c2..6544d365 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -14,7 +14,7 @@ import ( "golang.org/x/sys/windows" ) -const goEngineInlineTransmit = false +const goEngineTransmits = false const ( goCompletionKeyTun uintptr = iota + 1 diff --git a/stack_go_ring.go b/stack_go_ring.go index 2df22056..522ff990 100644 --- a/stack_go_ring.go +++ b/stack_go_ring.go @@ -501,7 +501,7 @@ func (p *goDescriptorPool) trim() { } // release requires exclusive access to both sides of the ring. For a live -// connection the engine holds transmitOwner and checks that all sent data is ACKed. +// connection the engine holds transmitAccess and checks that all sent data is ACKed. func (r *goDescriptorRing) release() { for index := range r.blocks { if block := r.blocks[index].Swap(nil); block != nil { diff --git a/stack_go_splice.go b/stack_go_splice.go index 990ceae6..54a208e9 100644 --- a/stack_go_splice.go +++ b/stack_go_splice.go @@ -130,7 +130,10 @@ type goSpliceStream struct { } func (c *GoConn) Splice(owner SpliceSocket, options SpliceOptions) bool { - if c.connState.Load() != goConnStateEstablished { + c.access.Lock() + eligible := !c.dead && c.phase == goPhaseEstablished && !c.spliced && c.splicePending == nil + c.access.Unlock() + if !eligible { return false } splice := &goSpliceStream{conn: c, owner: owner, options: options} @@ -146,7 +149,13 @@ func (c *GoConn) Splice(owner SpliceSocket, options SpliceOptions) bool { return false } splice.socket = socket - if !c.splicePending.CompareAndSwap(nil, splice) { + c.access.Lock() + installed := !c.dead && !c.spliced && c.splicePending == nil + if installed { + c.splicePending = splice + } + c.access.Unlock() + if !installed { socket.close() owner.Detach() return false @@ -156,22 +165,23 @@ func (c *GoConn) Splice(owner SpliceSocket, options SpliceOptions) bool { } func (s *goSpliceStream) Close() error { - s.conn.requestAbort(net.ErrClosed) + s.conn.fail(net.ErrClosed) return nil } func (e *goEngine) handleSpliceEngage(conn *GoConn) { - splice := conn.splicePending.Load() + conn.access.Lock() + splice := conn.splicePending + conn.splicePending = nil + conn.access.Unlock() if splice == nil { return } if conn.splice != nil { - conn.splicePending.Store(nil) e.spliceRelease(splice, net.ErrClosed) return } - if conn.connState.Load() >= goConnStateAborted { - conn.splicePending.Store(nil) + if conn.dead { e.spliceRelease(splice, conn.closeError()) return } @@ -184,16 +194,16 @@ func (e *goEngine) handleSpliceEngage(conn *GoConn) { err = e.platformIO.registerSocket(&splice.socket, splice.token, goInterestRead) } if err != nil { - conn.splicePending.Store(nil) e.releaseSpliceToken(splice.token) e.spliceRelease(splice, err) - e.spliceAbort(conn, err) + e.abortConn(conn, err) return } splice.interest = goInterestRead conn.splice = splice - conn.spliced.Store(true) - conn.splicePending.Store(nil) + conn.access.Lock() + conn.spliced = true + conn.access.Unlock() e.spliceFlushUpload(conn) } @@ -207,7 +217,11 @@ func (e *goEngine) spliceRelease(splice *goSpliceStream, err error) { } func (e *goEngine) spliceDetach(conn *GoConn, err error) { - pending := conn.splicePending.Swap(nil) + conn.access.Lock() + pending := conn.splicePending + conn.splicePending = nil + conn.spliced = false + conn.access.Unlock() if pending != nil { e.spliceRelease(pending, err) } @@ -219,7 +233,6 @@ func (e *goEngine) spliceDetach(conn *GoConn, err error) { e.spliceUnlinkDirty(conn) } conn.splice = nil - conn.spliced.Store(false) e.platformIO.unregisterSocket(&splice.socket) e.releaseSpliceToken(splice.token) e.spliceRelease(splice, err) @@ -242,10 +255,14 @@ func (e *goEngine) spliceUnlinkDirty(conn *GoConn) { } } -func (e *goEngine) spliceAbort(conn *GoConn, err error) { - if conn.connState.Load() >= goConnStateDead { - conn.storeError(err) - conn.receiveDrainable.Store(false) +func (e *goEngine) abortConn(conn *GoConn, err error) { + if conn.dead { + conn.access.Lock() + if conn.err == nil { + conn.err = err + } + conn.drainable = false + conn.access.Unlock() e.spliceDetach(conn, err) return } @@ -261,7 +278,7 @@ func (e *goEngine) spliceSetInterest(conn *GoConn, interest uint8) { splice.interest = interest err := e.platformIO.updateSocket(&splice.socket, interest) if err != nil { - e.spliceAbort(conn, err) + e.abortConn(conn, err) } } @@ -325,13 +342,13 @@ func (e *goEngine) spliceFlushUpload(conn *GoConn) { if consumed < available { splice.blocked = true } - if conn.windowUpdateDue(consumed) { + if consumed >= conn.windowUpdateAt.Load() { e.handleWindowUpdate(conn) } case goSocketWouldBlock(errno): splice.blocked = true default: - e.spliceAbort(conn, E.Cause(errno, "go: write peer")) + e.abortConn(conn, E.Cause(errno, "go: write peer")) return } } @@ -414,7 +431,7 @@ func (e *goEngine) spliceReadPeer(conn *GoConn) { e.spliceMaybeFinish(conn) return default: - e.spliceAbort(conn, E.Cause(errno, "go: read peer")) + e.abortConn(conn, E.Cause(errno, "go: read peer")) return } } @@ -423,12 +440,19 @@ func (e *goEngine) spliceReadPeer(conn *GoConn) { } func (e *goEngine) spliceTransmit(conn *GoConn) bool { - if conn.blockedValid || !conn.transmitOwner.CompareAndSwap(0, 1) { + if !conn.transmitAccess.TryLock() { + return true + } + if conn.blockedValid { + conn.transmitAccess.Unlock() return true } - _, result := conn.transmitLoop(false, 0) - conn.transmitOwner.Store(0) - if conn.connState.Load() >= goConnStateAborted { + _, result, armed := conn.transmitLoop(false, 0) + conn.transmitAccess.Unlock() + if conn.retransmitDeadline == 0 && (armed || conn.hasOutstanding()) { + e.rearmRetransmit(conn) + } + if conn.dead { return false } if result == goTransmitBlocked { @@ -449,40 +473,6 @@ func (e *goEngine) spliceResume(conn *GoConn) { e.spliceSetInterest(conn, splice.interest|goInterestRead) } -func (e *goEngine) spliceRetryBlocked(conn *GoConn) { - segment := conn.blockedSegment - end := segment.offset + uint64(segment.length) - err := conn.transmitFrame(&segment) - switch err { - case nil: - case errGoTransmitBlocked: - conn.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) - e.handleTransmitBlocked(conn) - return - case errGoFrameDropped: - conn.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) - conn.blockedValid = false - conn.blockedFrame = nil - conn.transmittedTail.Store(end) - e.handleDroppedFrames(conn) - return - default: - e.spliceAbort(conn, E.Cause(err, "go: write tun")) - return - } - if conn.congestion.pacing { - conn.advancePacing(e.now(), segment.length) - } - conn.blockedValid = false - conn.blockedFrame = nil - conn.transmittedTail.Store(end) - e.spliceResume(conn) - if conn.splice != nil && conn.splice.downloadClosed { - e.maybeSendFin(conn) - e.spliceMaybeFinish(conn) - } -} - func (e *goEngine) spliceMaybeFinish(conn *GoConn) { splice := conn.splice if splice == nil || !splice.uploadClosed || !splice.downloadClosed || !conn.finSent { @@ -706,7 +696,11 @@ func (e *goEngine) packetSpliceClose(w *GoPacketConn, err error) { func (e *goEngine) releasePending(message *goMessage) { switch message.kind { case goMessageConnSplice: - pending := message.conn.splicePending.Swap(nil) + conn := message.conn + conn.access.Lock() + pending := conn.splicePending + conn.splicePending = nil + conn.access.Unlock() if pending != nil { e.spliceRelease(pending, net.ErrClosed) } diff --git a/stack_go_tcp_ack.go b/stack_go_tcp_ack.go index dffa5e4e..6752261f 100644 --- a/stack_go_tcp_ack.go +++ b/stack_go_tcp_ack.go @@ -114,7 +114,7 @@ func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, ackOffset = int64(priorUnacked) } if conn.state == goTCPSynReceived { - if conn.connState.Load() != goConnStateEngaged { + if conn.phase != goPhaseEngaged { return false } if segmentAck != int64(conn.sentTail.Load()) { @@ -200,7 +200,7 @@ func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, } if ackFlags&goAckUnackedAdvanced != 0 { e.advanceFinState(conn) - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return true } if len(conn.scoreboard.entries) == 0 && conn.sendUnacked.Load() == conn.sentTail.Load() { @@ -999,8 +999,10 @@ func (c *GoConn) publishPermit(ackOffset uint64) { limit += uint64(c.congestionWindow-inFlight) * uint64(c.effectiveMSS) } windowEnd := ackOffset + c.peerWindow - c.permitWindowBound.Store(limit <= windowEnd) permit := max(min(limit, windowEnd), unacked) + if limit <= windowEnd { + permit |= goPermitWindowBit + } c.sendPacketPermit.Store(c.packetCreditBase.Load() + c.congestionWindow + c.flight.leftOut() - c.flight.retransmitOut) c.sendPermit.Store(permit) c.wakeTransmitter() @@ -1038,7 +1040,7 @@ func (e *goEngine) updateFrameLimit(conn *GoConn) { } func (e *goEngine) handlePacingRequest(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return } conn.pacingDeadline = conn.pacingRequest.Load() @@ -1102,8 +1104,11 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { unacked := conn.sendUnacked.Load() pending := conn.bufferedTail.Load() - sent windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + conn.access.Lock() + writing := conn.writing + conn.access.Unlock() switch { - case pending > 0 && committed == sent && windowEnd > sent && conn.writerActive.Load() == 0: + case pending > 0 && committed == sent && windowEnd > sent && !writing: length := min(pending, uint64(conn.effectiveMSS), windowEnd-sent) conn.probeRetransmitted = false conn.probeHighSeq = sent + length @@ -1111,8 +1116,8 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { if packets := conn.dataSegmentsOut.Load(); int32(conn.sendPacketPermit.Load()-packets) <= 0 { conn.sendPacketPermit.Store(packets + 1) } - if permit < sent+length { - conn.sendPermit.Store(sent + length) + if permit&^goPermitWindowBit < sent+length { + conn.sendPermit.Store(sent + length | permit&goPermitWindowBit) } conn.wakeTransmitter() case committed > unacked: @@ -1153,7 +1158,6 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { if !conn.hasOutstanding() { conn.retransmitDeadline = 0 - conn.retransmitArmed.Store(false) e.rearmTimer(conn) return } diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index 7228949d..46a10cc4 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -3,6 +3,7 @@ package tun import ( "encoding/binary" "hash/maphash" + "math" "net" "syscall" "time" @@ -108,11 +109,11 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { e.handleDuplicateSyn(existing) return } - if existing.connState.Load() < goConnStateDead && existing.state != goTCPSynReceived { + if !existing.dead && existing.state != goTCPSynReceived { e.markAck(existing, true) return } - if existing.connState.Load() < goConnStateDead { + if !existing.dead { e.detachConn(existing, E.New("go: connection replaced by new SYN"), goDeathImmediate) } e.removeFlow(existing) @@ -162,9 +163,8 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { conn.receiveAvailable.Store(1) conn.receiveNextAck.Store(1) conn.receiveEdge.Store(conn.publishedEdge) - conn.sentEdge.Store(conn.publishedEdge) - conn.receiveCapacityPublished.Store(conn.receiveCapacity) - conn.windowUpdateThreshold.Store(min(uint64(conn.effectiveMSS), conn.receiveCapacity/2)) + conn.sentEdge = conn.publishedEdge + conn.dataSentEdge.Store(conn.publishedEdge) conn.receiveSpaceConsumed = 1 conn.receiveSpaceStamp = now conn.sentTail.Store(1) @@ -174,6 +174,7 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { conn.sendReleased.Store(1) conn.sendPermit.Store(1 + min(uint64(conn.congestionWindow)*uint64(conn.effectiveMSS), conn.peerWindow)) conn.sendPacketPermit.Store(conn.congestionWindow) + conn.refreshWindowUpdateAt() conn.buildSynAck(synOptions, localMSS) conn.keyed = true e.flows[key] = conn @@ -191,7 +192,7 @@ func (e *goEngine) localMSS(ipVersion uint8) uint16 { func (e *goEngine) evictFlow() bool { visited := 0 for _, conn := range e.flows { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { e.removeFlow(conn) return true } @@ -216,13 +217,15 @@ func (e *goEngine) evictFlow() bool { func (e *goEngine) handleDuplicateSyn(conn *GoConn) { switch { - case conn.connState.Load() == goConnStateEngaged: + case conn.dead: + if conn.deathClass == goDeathAbortLinger { + e.sendReset(conn) + } + case conn.phase == goPhaseEngaged: err := conn.writeSynAck() if err != nil { e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) } - case conn.connState.Load() >= goConnStateAborted && conn.deathClass == goDeathAbortLinger: - e.sendReset(conn) } } @@ -232,7 +235,7 @@ func (e *goEngine) inputTCP(conn *GoConn, parsed *forwardPacket) { if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { return } - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { e.inputTombstone(conn, parsed) return } @@ -288,7 +291,7 @@ func (e *goEngine) inputTCP(conn *GoConn, parsed *forwardPacket) { if !e.processAck(conn, tcpHdr, segOffset, len(payload), flags, timestampEcho, hasTimestamp) { return } - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return } if len(payload) > 0 || flags&header.TCPFlagFin != 0 { @@ -377,8 +380,9 @@ func (c *GoConn) segmentAcceptable(segOffset int64, segmentLength int64) bool { func (e *goEngine) establishConn(conn *GoConn) { conn.state = goTCPEstablished conn.handshakeDeadline = 0 - conn.everEstablished.Store(true) - conn.connState.CompareAndSwap(goConnStateEngaged, goConnStateEstablished) + conn.access.Lock() + conn.phase = goPhaseEstablished + conn.access.Unlock() close(conn.establishedSignal) e.wokeHandlerThisBurst = true } @@ -492,15 +496,13 @@ func (c *GoConn) releaseTransmitted(unacked uint64) { } func (c *GoConn) releaseIdleDescriptors() { - if len(c.scoreboard.entries) != 0 || !c.transmitOwner.CompareAndSwap(0, 1) { + if len(c.scoreboard.entries) != 0 || !c.transmitAccess.TryLock() { return } if c.sendUnacked.Load() == c.sentTail.Load() && !c.blockedValid { c.descriptors.release() } - c.transmitOwner.Store(0) - // A writer can publish data while we own the transmitter. Resume it if its - // attempt to transmit raced with this cleanup. + c.transmitAccess.Unlock() if c.nextTransmitLength() != 0 { c.wakeTransmitter() } @@ -525,7 +527,12 @@ func (c *GoConn) wakeTransmitter() { if c.nextTransmitLength() == 0 { return } - if goEngineInlineTransmit && c.transmitOnEngine() { + if goEngineTransmits { + if c.engine.inlineTransmitBudget <= 0 { + c.engine.postMessage(&c.transmitMessage) + } else { + c.engine.transmitOnEngine(c, c.engine.inlineTransmitBudget) + } return } c.wakeTransmitterGoroutine() @@ -533,10 +540,8 @@ func (c *GoConn) wakeTransmitter() { } func (c *GoConn) wakeWriter() { - if !c.writerParked.Load() { - return - } - if !c.writeReady(int(max(c.writerNeeds.Load(), 1))) { + required := c.writerWaiting.Load() + if required == 0 || !c.writeReady(int(required)) { return } c.signalWriter() @@ -624,8 +629,8 @@ func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { direct := 0 taken := false if conn.receiveAvailable.Load() == conn.consumedTail.Load() { - target := conn.postedTarget.Swap(nil) - if target != nil { + target := conn.postedTarget.Load() + if target != nil && target != &goTargetFilled && conn.postedTarget.CompareAndSwap(target, &goTargetTaken) { direct = copy(target.target, data) target.filled = direct taken = true @@ -639,7 +644,7 @@ func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { conn.receiveAvailable.Store(conn.receiveNext) conn.publishReceiveWindow() if taken { - conn.targetDone.Store(true) + conn.postedTarget.Store(&goTargetFilled) } if conn.readerParked.Load() || taken { conn.signalReader() @@ -681,11 +686,12 @@ func (e *goEngine) acceptFin(conn *GoConn) { e.markAck(conn, true) return } + conn.access.Lock() conn.finReceived = true + conn.access.Unlock() conn.oooRanges = nil conn.receiveNext++ conn.receiveNextAck.Store(conn.receiveNext) - conn.receiveShutdown.Store(true) conn.publishReceiveWindow() if conn.readerParked.Load() { conn.signalReader() @@ -738,14 +744,35 @@ func (c *GoConn) publishReceiveWindow() { base = c.receiveNext } edge := base + c.receiveCapacity - if edge <= c.publishedEdge { - return + if edge > c.publishedEdge && (base >= c.receiveNext || edge-c.advertisedEdge() >= c.windowUpdateThreshold()) { + c.publishedEdge = edge + c.receiveEdge.Store(edge) } - if base < c.receiveNext && edge-c.sentEdge.Load() < c.windowUpdateThreshold.Load() { - return + c.refreshWindowUpdateAt() +} + +func (c *GoConn) advertisedEdge() uint64 { + return max(c.sentEdge, c.dataSentEdge.Load()) +} + +func (c *GoConn) windowUpdateThreshold() uint64 { + return min(uint64(c.effectiveMSS), c.receiveCapacity/2) +} + +func (c *GoConn) refreshWindowUpdateAt() { + edge := c.advertisedEdge() + capacity := int64(c.receiveCapacity) + at := int64(edge) + int64(c.windowUpdateThreshold()) - capacity + available := c.receiveAvailable.Load() + if available < edge { + window := int64(edge - available) + if 2*window > capacity { + c.windowUpdateAt.Store(math.MaxUint64) + return + } + at = max(at, int64(available)+2*window-capacity) } - c.publishedEdge = edge - c.receiveEdge.Store(edge) + c.windowUpdateAt.Store(uint64(max(at, 0))) } func (e *goEngine) measureReceiveRoundTrip(conn *GoConn) { @@ -785,8 +812,6 @@ func (e *goEngine) moderateReceiveCapacity(conn *GoConn) { return } conn.receiveCapacity = capacity - conn.receiveCapacityPublished.Store(capacity) - conn.windowUpdateThreshold.Store(min(uint64(conn.effectiveMSS), capacity/2)) } func (e *goEngine) receiveCapacityLimit(conn *GoConn) uint64 { @@ -862,6 +887,7 @@ func (e *goEngine) sendAck(conn *GoConn) bool { if !e.writeConnControl(conn, &segment) { return false } + conn.refreshWindowUpdateAt() conn.dsackStart = 0 conn.dsackEnd = 0 conn.ackPending = 0 @@ -876,7 +902,7 @@ func (e *goEngine) sendReset(conn *GoConn) bool { } func (e *goEngine) writeConnControl(conn *GoConn, segment *goSegment) bool { - frame, _ := conn.buildFrame(e.controlScratch[:], e.controlSegments[:0], segment, &conn.transmitStore, false) + frame, _ := conn.buildFrame(e.controlScratch[:], e.controlSegments[:0], segment, &conn.transmitStore, false, true) e.controlSegments = frame err := e.platformIO.writeFrame(frame, ForwardFrameMeta{}) clear(frame) @@ -888,13 +914,13 @@ func (e *goEngine) writeConnControl(conn *GoConn, segment *goSegment) bool { } func (e *goEngine) maybeSendFin(conn *GoConn) { - if !conn.finPending || conn.finSent || conn.state == goTCPSynReceived { + if !conn.finPending || conn.finSent || conn.state == goTCPSynReceived || conn.dead { return } - if conn.connState.Load() >= goConnStateAborted { - return - } - if conn.writerActive.Load() != 0 { + conn.access.Lock() + writing := conn.writing + conn.access.Unlock() + if writing { return } buffered := conn.bufferedTail.Load() @@ -915,18 +941,21 @@ func (e *goEngine) maybeSendFin(conn *GoConn) { } func (e *goEngine) handleCloseRequest(conn *GoConn) { - if conn.connState.Load() >= goConnStateDead { - if conn.userClosed.Load() || conn.closeMode.Load() == goCloseModeAbort { - conn.receiveDrainable.Store(false) - e.spliceDetach(conn, conn.loadError()) + conn.access.Lock() + abort := conn.abort || conn.splicePending != nil + discard := conn.userClosed || conn.abort + if conn.dead && discard { + conn.drainable = false + } + err := conn.err + conn.access.Unlock() + if conn.dead { + if discard { + e.spliceDetach(conn, err) } return } - if conn.connState.Load() == goConnStateAborted { - e.detachConn(conn, nil, goDeathAbortLinger) - return - } - if conn.closeMode.Load() == goCloseModeAbort || conn.splice != nil || conn.splicePending.Load() != nil { + if abort || conn.splice != nil { e.sendReset(conn) e.detachConn(conn, net.ErrClosed, goDeathAbortLinger) return @@ -936,7 +965,7 @@ func (e *goEngine) handleCloseRequest(conn *GoConn) { } func (e *goEngine) handleReadShut(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return } conn.discardReceive = true @@ -953,25 +982,25 @@ func (e *goEngine) handleReadShut(conn *GoConn) { } func (e *goEngine) handleWindowUpdate(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return } conn.receiveChain.releaseBelow(conn.consumedTail.Load()) conn.publishReceiveWindow() - if conn.publishedEdge > conn.sentEdge.Load() || conn.ackPending > 0 { + if conn.publishedEdge > conn.advertisedEdge() || conn.ackPending > 0 { e.markAck(conn, true) } } func (e *goEngine) handleRetransmitArm(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead || conn.retransmitDeadline != 0 { return } e.rearmRetransmit(conn) } func (e *goEngine) handleDroppedFrames(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { return } e.drainDescriptors(conn) @@ -996,7 +1025,7 @@ func (e *goEngine) handleDroppedFrames(conn *GoConn) { } func (e *goEngine) handleTransmitBlocked(conn *GoConn) { - if conn.connState.Load() >= goConnStateAborted { + if conn.dead { conn.transmitSignal.notify() return } @@ -1021,8 +1050,8 @@ func (e *goEngine) releaseBlockedWriters() { next := conn.blockedNext conn.blockedNext = nil conn.onBlockedList = false - if conn.splice != nil { - e.spliceRetryBlocked(conn) + if conn.splice != nil || goEngineTransmits { + e.retryBlockedOnEngine(conn) } else { conn.transmitSignal.notify() e.wokeHandlerThisBurst = true @@ -1045,16 +1074,12 @@ func (c *GoConn) hasOutstanding() bool { func (e *goEngine) rearmRetransmit(conn *GoConn) { if !conn.hasOutstanding() { conn.retransmitDeadline = 0 - conn.retransmitArmed.Store(false) - if !conn.hasOutstanding() { - if conn.persistNeeded() { - e.updatePersist(conn) - } else { - e.rearmTimer(conn) - } - return + if conn.persistNeeded() { + e.updatePersist(conn) + } else { + e.rearmTimer(conn) } - conn.retransmitArmed.Store(true) + return } if conn.persistNeeded() { e.updatePersist(conn) @@ -1194,24 +1219,26 @@ func (e *goEngine) retransmitFrom(conn *GoConn, offset uint64) { } func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { - if conn.connState.Load() >= goConnStateDead { + if conn.dead { return } - if err != nil { - conn.storeError(err) - } - if class == goDeathFinLinger && conn.receiveAvailable.Load() > conn.consumedTail.Load() { - conn.receiveDrainable.Store(true) + conn.access.Lock() + if err != nil && conn.err == nil { + conn.err = err } - conn.connState.Store(goConnStateDead) + err = conn.err + conn.drainable = class == goDeathFinLinger && conn.receiveAvailable.Load() > conn.consumedTail.Load() + conn.dead = true + established := conn.phase == goPhaseEstablished + conn.access.Unlock() conn.deathClass = class conn.state = goTCPClosed if class == goDeathFinLinger && conn.splice != nil && !conn.splice.uploadClosed { e.spliceMarkDirty(conn) } else { - e.spliceDetach(conn, conn.loadError()) + e.spliceDetach(conn, err) } - if !conn.everEstablished.Load() { + if !established { close(conn.establishedSignal) } close(conn.closeSignal) @@ -1249,7 +1276,7 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { } func (e *goEngine) removeFlow(conn *GoConn) { - if conn.connState.Load() < goConnStateDead { + if !conn.dead { e.detachConn(conn, net.ErrClosed, goDeathImmediate) return } @@ -1278,9 +1305,7 @@ func (e *goEngine) reapDying() { for conn := e.dyingList; conn != nil; { next := conn.dyingNext conn.dyingNext = nil - if conn.reclaimable() { - e.platformIO.flush() - conn.releaseResources() + if conn.reclaim() { conn.onDyingList = false } else { if now-conn.dyingSince > int64(goDyingLeakTimeout) { @@ -1295,22 +1320,26 @@ func (e *goEngine) reapDying() { e.dyingList = kept } -func (c *GoConn) reclaimable() bool { - if c.spliced.Load() || c.writerActive.Load() != 0 || c.flushActive.Load() != 0 || c.transmitterActive.Load() != 0 || c.postedTarget.Load() != nil { +func (c *GoConn) reclaim() bool { + c.access.Lock() + blocked := c.spliced || c.transmitterRunning || (c.drainable && !c.userClosed && c.receiveAvailable.Load() > c.consumedTail.Load()) + c.access.Unlock() + if blocked || !c.readAccess.TryLock() { return false } - if c.receiveDrainable.Load() && !c.userClosed.Load() && c.receiveAvailable.Load() > c.consumedTail.Load() { + defer c.readAccess.Unlock() + if !c.writeAccess.TryLock() { return false } - c.receiveReleased.Store(true) - if c.readerActive.Load() != 0 { - c.receiveReleased.Store(false) + defer c.writeAccess.Unlock() + if !c.transmitAccess.TryLock() { return false } - return true -} - -func (c *GoConn) releaseResources() { + defer c.transmitAccess.Unlock() + c.access.Lock() + c.released = true + c.access.Unlock() + c.engine.platformIO.flush() c.receiveChain.releaseAll() c.transmitStore.releaseAll() c.oooRanges = nil @@ -1321,12 +1350,13 @@ func (c *GoConn) releaseResources() { c.receiveTarget.buffer.Release() c.receiveTarget.buffer = nil } + return true } func (e *goEngine) closeAllFlows() []*GoConn { var unreset []*GoConn for _, conn := range e.flows { - if conn.connState.Load() < goConnStateDead && !e.sendReset(conn) { + if !conn.dead && !e.sendReset(conn) { unreset = append(unreset, conn) } e.detachConn(conn, net.ErrClosed, goDeathImmediate) @@ -1337,7 +1367,7 @@ func (e *goEngine) closeAllFlows() []*GoConn { func (e *goEngine) expireSweepTick(now int64) { for _, conn := range e.flows { - if conn.connState.Load() >= goConnStateDead { + if conn.dead { continue } sent := conn.sentTail.Load() @@ -1361,7 +1391,10 @@ func (e *goEngine) expireSweepTick(now int64) { } func (e *goEngine) keepalive(conn *GoConn, now int64) { - if conn.state == goTCPFinWait2 && conn.userClosed.Load() && now-conn.finWait2Since > int64(goFinWait2Timeout) { + conn.access.Lock() + userClosed := conn.userClosed + conn.access.Unlock() + if conn.state == goTCPFinWait2 && userClosed && now-conn.finWait2Since > int64(goFinWait2Timeout) { e.sendReset(conn) e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: orphaned FIN_WAIT_2"), goDeathImmediate) return diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go index f2a9264a..216483ee 100644 --- a/stack_go_tcp_output.go +++ b/stack_go_tcp_output.go @@ -43,21 +43,77 @@ func (c *GoConn) Write(p []byte) (int, error) { return 0, err } c.writeAccess.Lock() - defer c.writeAccess.Unlock() - if !c.enterWriter() { - return 0, net.ErrClosed + var n int + if c.enterWriter() { + n, err = c.writeLocked(p) + c.exitWriter() + } else { + err = net.ErrClosed + } + c.writeAccess.Unlock() + c.engine.reapAfterExit() + return n, err +} + +func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { + err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) + if err != nil { + buffer.Release() + return err + } + c.writeAccess.Lock() + if c.enterWriter() { + err = c.writeBufferLocked(buffer) + c.exitWriter() + } else { + buffer.Release() + err = net.ErrClosed + } + c.writeAccess.Unlock() + c.engine.reapAfterExit() + return err +} + +func (c *GoConn) enterWriter() bool { + c.access.Lock() + defer c.access.Unlock() + if c.dead || c.spliced || c.splicePending != nil { + return false + } + c.writing = true + return true +} + +func (c *GoConn) exitWriter() { + c.access.Lock() + c.writing = false + dead := c.dead + finRequested := c.finRequested + c.access.Unlock() + if !dead { + sent := c.sentTail.Load() + if c.bufferedTail.Load() > sent { + c.transmitPublished() + } else { + c.checkAppLimited(0, c.sendPermit.Load()&^goPermitWindowBit, sent) + } + } + if finRequested { + c.engine.postMessage(&c.closeMessage) } - defer c.exitWriter() - return c.writeLocked(p) } func (c *GoConn) writeLocked(p []byte) (int, error) { total := 0 for { - if c.connState.Load() >= goConnStateAborted { + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + c.access.Unlock() + if dead { return total, c.closeError() } - if c.writeShut.Load() { + if writeShut { return total, net.ErrClosed } select { @@ -88,31 +144,23 @@ func (c *GoConn) writeLocked(p []byte) (int, error) { } } -func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { - err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) - if err != nil { - buffer.Release() - return err - } - c.writeAccess.Lock() - defer c.writeAccess.Unlock() - if !c.enterWriter() { - buffer.Release() - return net.ErrClosed - } - defer c.exitWriter() +func (c *GoConn) writeBufferLocked(buffer *buf.Buffer) error { length := buffer.Len() if length > goTransmitCapacityMax { defer buffer.Release() - _, err = c.writeLocked(buffer.Bytes()) + _, err := c.writeLocked(buffer.Bytes()) return err } for { - if c.connState.Load() >= goConnStateAborted { + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + c.access.Unlock() + if dead { buffer.Release() return c.closeError() } - if c.writeShut.Load() { + if writeShut { buffer.Release() return net.ErrClosed } @@ -130,10 +178,10 @@ func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { if budget < length { if budget > 0 && c.storeEmpty() { defer buffer.Release() - _, err = c.writeLocked(buffer.Bytes()) + _, err := c.writeLocked(buffer.Bytes()) return err } - err = c.parkWriter(length) + err := c.parkWriter(length) if err != nil { buffer.Release() return err @@ -143,7 +191,7 @@ func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { buffered := c.bufferedTail.Load() if !c.transmitStore.adopt(buffered, buffer) { defer buffer.Release() - _, err = c.writeLocked(buffer.Bytes()) + _, err := c.writeLocked(buffer.Bytes()) return err } c.publishBuffered(buffered + uint64(length)) @@ -161,21 +209,22 @@ func (c *GoConn) WriterMTU() int { func (c *GoConn) publishBuffered(tail uint64) { c.bufferedTail.Store(tail) - if c.transmitterActive.Load() != 0 { - c.transmitSignal.notify() - return - } - c.transmitInline() + c.transmitPublished() } -func (c *GoConn) transmitInline() { +func (c *GoConn) transmitPublished() { + if goEngineTransmits { + c.engine.postMessage(&c.transmitMessage) + return + } for { - if !c.transmitOwner.CompareAndSwap(0, 1) { + if !c.transmitAccess.TryLock() { c.transmitSignal.notify() return } - _, result := c.transmitLoop(false, 0) - c.transmitOwner.Store(0) + _, result, armed := c.transmitLoop(false, 0) + c.transmitAccess.Unlock() + c.afterTransmit(armed) switch result { case goTransmitBlocked: c.wakeTransmitterGoroutine() @@ -183,35 +232,113 @@ func (c *GoConn) transmitInline() { case goTransmitPaced: return } - if c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 { + if c.closed() || c.nextTransmitLength() == 0 { return } } } -func (c *GoConn) transmitOnEngine() bool { - budget := c.engine.inlineTransmitBudget - if budget <= 0 { - return false +func (c *GoConn) afterTransmit(armed bool) { + if armed { + c.engine.postMessage(&c.retransmitMessage) } - if !c.transmitOwner.CompareAndSwap(0, 1) { - c.transmitSignal.notify() - return true + c.access.Lock() + finRequested := c.finRequested + c.access.Unlock() + if finRequested && c.sentTail.Load() == c.bufferedTail.Load() { + c.engine.postMessage(&c.closeMessage) + } +} + +func (e *goEngine) transmitOnEngine(conn *GoConn, budget int) { + if !conn.transmitAccess.TryLock() { + conn.transmitSignal.notify() + return + } + written, result, armed := conn.transmitLoop(false, budget) + conn.transmitAccess.Unlock() + if budget > 0 { + e.inlineTransmitBudget = budget - written + } + if conn.retransmitDeadline == 0 && (armed || conn.hasOutstanding()) { + e.rearmRetransmit(conn) } - written, result := c.transmitLoop(false, budget) - c.transmitOwner.Store(0) - c.engine.inlineTransmitBudget = budget - written + e.maybeSendFin(conn) switch result { case goTransmitBlocked: - return false - case goTransmitPaced: - return true + e.handleTransmitBlocked(conn) + case goTransmitDone: + if !conn.dead && conn.nextTransmitLength() != 0 { + e.postMessage(&conn.transmitMessage) + } + } +} + +func (e *goEngine) handleTransmitRequest(conn *GoConn) { + if conn.dead { + return + } + if conn.splice != nil { + e.spliceResume(conn) + return + } + e.transmitOnEngine(conn, 0) +} + +func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { + if !conn.transmitAccess.TryLock() { + conn.transmitSignal.notify() + return + } + segment := conn.blockedSegment + end := segment.offset + uint64(segment.length) + err := conn.transmitFrame(&segment) + switch err { + case nil: + case errGoTransmitBlocked: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) + conn.transmitAccess.Unlock() + e.handleTransmitBlocked(conn) + return + case errGoFrameDropped: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmittedTail.Store(end) + conn.transmitAccess.Unlock() + e.handleDroppedFrames(conn) + return + default: + conn.transmitAccess.Unlock() + e.abortConn(conn, E.Cause(err, "go: write tun")) + return + } + if conn.congestion.pacing { + conn.advancePacing(e.now(), segment.length) + } + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmittedTail.Store(end) + conn.transmitAccess.Unlock() + if conn.splice != nil { + e.spliceResume(conn) + if conn.splice != nil && conn.splice.downloadClosed { + e.maybeSendFin(conn) + e.spliceMaybeFinish(conn) + } + return } - return c.connState.Load() >= goConnStateAborted || c.nextTransmitLength() == 0 + e.transmitOnEngine(conn, 0) } func (c *GoConn) wakeTransmitterGoroutine() { - if c.transmitterActive.CompareAndSwap(0, 1) { + c.access.Lock() + spawn := !c.transmitterRunning && !c.dead + if spawn { + c.transmitterRunning = true + } + c.access.Unlock() + if spawn { go c.runTransmitter() return } @@ -219,21 +346,24 @@ func (c *GoConn) wakeTransmitterGoroutine() { } func (c *GoConn) hasDataWaiting() bool { - return c.bufferedTail.Load() > c.sentTail.Load() || c.writerParked.Load() + return c.bufferedTail.Load() > c.sentTail.Load() || c.writerWaiting.Load() != 0 } func (c *GoConn) runTransmitter() { defer func() { - c.transmitterActive.Store(0) + c.access.Lock() + c.transmitterRunning = false + c.access.Unlock() c.engine.reapAfterExit() }() for { - if c.connState.Load() >= goConnStateAborted { + if c.closed() { return } - if c.transmitOwner.CompareAndSwap(0, 1) { - c.transmitLoop(true, 0) - c.transmitOwner.Store(0) + if c.transmitAccess.TryLock() { + _, _, armed := c.transmitLoop(true, 0) + c.transmitAccess.Unlock() + c.afterTransmit(armed) } select { case <-c.transmitSignal: @@ -250,20 +380,26 @@ func (c *GoConn) nextTransmitLength() int { return 0 } permit := c.sendPermit.Load() - if permit <= sent { - if c.permitWindowBound.Load() { + if permit&^goPermitWindowBit <= sent { + if permit&goPermitWindowBit != 0 { c.windowLimitedSince.Store(true) } return 0 } + permit &^= goPermitWindowBit packets := int32(c.sendPacketPermit.Load() - c.dataSegmentsOut.Load()) if packets <= 0 { c.windowLimitedSince.Store(true) return 0 } length := int(min(pending, permit-sent, uint64(packets)*uint64(c.effectiveMSS))) - if pending < uint64(c.effectiveMSS) && c.writerActive.Load() != 0 && sent > c.sendUnacked.Load() { - return 0 + if pending < uint64(c.effectiveMSS) && sent > c.sendUnacked.Load() { + c.access.Lock() + writing := c.writing + c.access.Unlock() + if writing { + return 0 + } } // Windows delays the ACK of a lone segment by 40 ms and its receive window auto-tuning never // grows while every window is drained by one segment, so a window narrower than two segments is @@ -289,7 +425,7 @@ func (c *GoConn) writeBudget(pending int) int { } } unacked := c.sendUnacked.Load() - permit := max(c.sendPermit.Load(), unacked) + permit := max(c.sendPermit.Load()&^goPermitWindowBit, unacked) limit := min(permit+max(permit-unacked, goTransmitReadAhead), released+goTransmitCapacityMax) budget := int64(limit) - int64(buffered) if pending > 0 { @@ -311,33 +447,36 @@ func (c *GoConn) writeReady(required int) bool { } func (c *GoConn) parkWriter(required int) error { - c.writerNeeds.Store(int32(required)) - c.writerParked.Store(true) + c.writerWaiting.Store(int32(required)) c.writeSignal.drain() if c.writeReady(required) { - c.writerParked.Store(false) + c.writerWaiting.Store(0) return nil } - if c.connState.Load() >= goConnStateAborted { - c.writerParked.Store(false) + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + c.access.Unlock() + if dead { + c.writerWaiting.Store(0) return c.closeError() } - if c.writeShut.Load() { - c.writerParked.Store(false) + if writeShut { + c.writerWaiting.Store(0) return net.ErrClosed } deadlineSignal := c.writeDeadline.Wait() select { case <-c.writeSignal: case <-deadlineSignal: - c.writerParked.Store(false) + c.writerWaiting.Store(0) return os.ErrDeadlineExceeded case <-c.closeSignal: - c.writerParked.Store(false) + c.writerWaiting.Store(0) return c.closeError() } - c.writerParked.Store(false) - if c.connState.Load() >= goConnStateAborted { + c.writerWaiting.Store(0) + if c.closed() { return c.closeError() } return nil @@ -350,7 +489,7 @@ func (c *GoConn) parkTransmitBlocked() error { case <-c.closeSignal: return c.closeError() } - if c.connState.Load() >= goConnStateAborted { + if c.closed() { return c.closeError() } return nil @@ -364,24 +503,23 @@ const ( goTransmitPaced ) -func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResult) { +func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResult, bool) { if transmitter { defer c.engine.platformIO.flush() } - sentAny := false written := 0 + armed := false if c.blockedValid { if !transmitter { - return written, goTransmitBlocked + return written, goTransmitBlocked, armed } if !c.retryBlocked() { - return written, goTransmitDone + return written, goTransmitDone, armed } - sentAny = true } paced := c.congestion.pacing result := goTransmitDone - for c.connState.Load() < goConnStateAborted { + for !c.closed() { if budget > 0 && written >= budget { break } @@ -389,9 +527,10 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu if length == 0 { sent := c.sentTail.Load() pending := c.bufferedTail.Load() - sent - c.checkAppLimited(pending, c.sendPermit.Load(), sent) - if pending > 0 && !c.permitWindowBound.Load() && sent == c.sendUnacked.Load() { - c.armRetransmitIfIdle() + permit := c.sendPermit.Load() + c.checkAppLimited(pending, permit&^goPermitWindowBit, sent) + if pending > 0 && permit&goPermitWindowBit == 0 && sent == c.sendUnacked.Load() { + armed = true } break } @@ -415,6 +554,7 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu flightEmpty := sent == unacked if flightEmpty { flags |= goDescriptorTxStart + armed = true } if !paced { now = c.engine.readClock() @@ -426,9 +566,9 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu if flight > c.peakFlight.Load() { c.peakFlight.Store(flight) } - sentAny = true + permit := c.sendPermit.Load() if int32(c.sendPacketPermit.Load()-c.dataSegmentsOut.Load()) <= 0 || - sent+uint64(length) >= c.sendPermit.Load() && c.permitWindowBound.Load() { + sent+uint64(length) >= permit&^goPermitWindowBit && permit&goPermitWindowBit != 0 { c.windowLimitedSince.Store(true) } err := c.transmitFrame(&segment) @@ -443,25 +583,23 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu c.blockedSegment = segment c.blockedValid = true if !transmitter { - c.finishTransmit(sentAny) - return written, goTransmitBlocked + return written, goTransmitBlocked, armed } if !c.retryBlocked() { - return written, goTransmitDone + return written, goTransmitDone, armed } case errGoFrameDropped: c.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) c.transmittedTail.Store(sent + uint64(length)) c.engine.postMessage(&c.droppedMessage) default: - if c.connState.Load() < goConnStateAborted { - c.requestAbort(E.Cause(err, "go: write tun")) + if !c.closed() { + c.fail(E.Cause(err, "go: write tun")) } - return written, goTransmitDone + return written, goTransmitDone, armed } } - c.finishTransmit(sentAny) - return written, result + return written, result, armed } func (c *GoConn) retryBlocked() bool { @@ -480,8 +618,8 @@ func (c *GoConn) retryBlocked() bool { case errGoFrameDropped: c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorDropped) default: - if c.connState.Load() < goConnStateAborted { - c.requestAbort(E.Cause(err, "go: write tun")) + if !c.closed() { + c.fail(E.Cause(err, "go: write tun")) } return false } @@ -497,22 +635,6 @@ func (c *GoConn) retryBlocked() bool { } } -func (c *GoConn) finishTransmit(sentAny bool) { - if !sentAny { - return - } - if c.spliced.Load() { - if c.retransmitDeadline == 0 { - c.engine.rearmRetransmit(c) - } - return - } - c.armRetransmitIfIdle() - if c.finRequested.Load() && c.sentTail.Load() == c.bufferedTail.Load() { - c.engine.postMessage(&c.closeMessage) - } -} - func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint16) { c.descriptors.push(goSentDescriptor{endOffset: offset + uint64(length), sentAt: int32(length), flags: goDescriptorAmend | flags}) } @@ -591,7 +713,7 @@ func (c *GoConn) transmitFrame(segment *goSegment) error { c.transmitSegments = append(c.transmitSegments[:0], packet) frame = c.transmitSegments } else { - frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload) + frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload, false) c.transmitSegments = frame } } @@ -607,12 +729,6 @@ func (c *GoConn) transmitFrame(segment *goSegment) error { return err } -func (c *GoConn) armRetransmitIfIdle() { - if c.retransmitArmed.CompareAndSwap(false, true) { - c.engine.postMessage(&c.retransmitMessage) - } -} - type goFrameLayout struct { ipHeaderLength int tcpHeaderLength int @@ -642,7 +758,7 @@ func (c *GoConn) frameLayout(segment *goSegment) goFrameLayout { return layout } -func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *goSegment) uint16 { +func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *goSegment, control bool) uint16 { tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) receiveAck := c.receiveNextAck.Load() edge := c.receiveEdge.Load() @@ -650,11 +766,10 @@ func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *go if edge > receiveAck { window = uint16(min((edge-receiveAck)>>c.localWindowShift, 0xffff)) } - for { - sent := c.sentEdge.Load() - if edge <= sent || c.sentEdge.CompareAndSwap(sent, edge) { - break - } + if control { + c.sentEdge = max(c.sentEdge, edge) + } else if edge > c.dataSentEdge.Load() { + c.dataSentEdge.Store(edge) } tcpHdr.Encode(&header.TCPFields{ SrcPort: c.destination.Port, @@ -721,10 +836,10 @@ func (c *GoConn) offloadMeta(tcpHdr header.TCP, pseudoSum uint16, layout *goFram return meta } -func (c *GoConn) buildFrame(scratch []byte, segments [][]byte, segment *goSegment, store *goTransmitStore, offload bool) ([][]byte, ForwardFrameMeta) { +func (c *GoConn) buildFrame(scratch []byte, segments [][]byte, segment *goSegment, store *goTransmitStore, offload bool, control bool) ([][]byte, ForwardFrameMeta) { layout := c.frameLayout(segment) packet := scratch[:layout.headerLength()] - pseudoSum := c.encodeHeaders(packet, &layout, segment) + pseudoSum := c.encodeHeaders(packet, &layout, segment, control) segments = append(segments, packet) if segment.length > 0 { segments = store.appendRuns(segments, segment.offset, segment.length) @@ -745,7 +860,7 @@ func (c *GoConn) buildContiguousFrame(buffer *buf.Buffer, payload []byte, segmen headerLength := layout.headerLength() packet := buffer.ExtendHeader(headerLength) packet = packet[:headerLength+len(payload)] - pseudoSum := c.encodeHeaders(packet[:headerLength], &layout, segment) + pseudoSum := c.encodeHeaders(packet[:headerLength], &layout, segment, false) tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) if offload { return packet, c.offloadMeta(tcpHdr, pseudoSum, &layout, segment) @@ -832,13 +947,6 @@ func (c *GoConn) writeSynAck() error { return goIgnoreDropped(c.engine.platformIO.writePacket(c.synAckImage[:c.synAckLength], ForwardFrameMeta{})) } -func (c *GoConn) writeReset() error { - var frame [1][]byte - segment := goSegment{offset: 0, flags: header.TCPFlagRst | header.TCPFlagAck} - segments, _ := c.buildFrame(c.transmitScratch[:], frame[:0], &segment, &c.transmitStore, false) - return goIgnoreDropped(c.engine.platformIO.writePacket(segments[0], ForwardFrameMeta{})) -} - var errGoTransmitBlocked = E.New("go: transmit blocked") func goGSOType(ipVersion uint8) uint8 { diff --git a/stack_go_tcp_rate.go b/stack_go_tcp_rate.go index 0ec8aa38..20262f62 100644 --- a/stack_go_tcp_rate.go +++ b/stack_go_tcp_rate.go @@ -111,7 +111,13 @@ func (c *GoConn) publishDelivery() { func (c *GoConn) checkAppLimited(pending uint64, permit uint64, sent uint64) { segments := c.dataSegmentsOut.Load() - if pending >= uint64(c.effectiveMSS) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerActive.Load() != 0 || c.writerParked.Load() { + if pending >= uint64(c.effectiveMSS) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerWaiting.Load() != 0 { + return + } + c.access.Lock() + writing := c.writing + c.access.Unlock() + if writing { return } inFlight := segments - c.packetCreditBase.Load() diff --git a/stack_go_udp.go b/stack_go_udp.go index 7550cfbf..1b7a38be 100644 --- a/stack_go_udp.go +++ b/stack_go_udp.go @@ -228,6 +228,10 @@ func (s *Go) prepareUDPConnection(source M.Socksaddr, destination M.Socksaddr, u return true, s.ctx, writer, writer.handleSessionClose } +type goPacketCloseError struct { + err error +} + type GoPacketConn struct { engine *goEngine key udpNatSessionKey @@ -235,7 +239,7 @@ type GoPacketConn struct { splice *goSplicePacket splicePending atomic.Pointer[goSplicePacket] spliceActive atomic.Bool - spliceCloseError atomic.Pointer[goConnError] + spliceCloseError atomic.Pointer[goPacketCloseError] spliceMessage goMessage closeMessage goMessage mtu int @@ -479,7 +483,7 @@ func (w *GoPacketConn) HandshakeFailure(err error) error { } func (w *GoPacketConn) closeSplice(err error) { - w.spliceCloseError.CompareAndSwap(nil, &goConnError{err: err}) + w.spliceCloseError.CompareAndSwap(nil, &goPacketCloseError{err: err}) w.engine.postMessage(&w.closeMessage) } From 89a7074f3de4feb443fa322f10ca545f54fe85ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 15 Sep 2026 23:21:45 +0800 Subject: [PATCH 11/25] Add userspace API and improve qdisc for stack --- flow_dispatch.go | 2 - flow_dns.go | 3 - stack_go.go | 248 ++++++--- stack_go_conn.go | 84 ++- stack_go_conn_options.go | 84 +++ stack_go_dial.go | 339 +++++++++++++ stack_go_engine.go | 211 ++++++-- stack_go_integration_cc_test.go | 10 +- stack_go_integration_flow_linux_test.go | 23 +- stack_go_integration_listen_test.go | 182 +++++++ stack_go_integration_memory_darwin_test.go | 15 + stack_go_integration_memory_linux_test.go | 29 ++ stack_go_integration_memory_test.go | 565 +++++++++++++++++++++ stack_go_integration_options_test.go | 458 +++++++++++++++++ stack_go_integration_packet_flow_test.go | 10 +- stack_go_integration_packet_test.go | 50 +- stack_go_integration_tcp_test.go | 24 +- stack_go_integration_test.go | 226 ++++++++- stack_go_io.go | 11 +- stack_go_io_darwin.go | 56 +- stack_go_io_linux.go | 59 ++- stack_go_io_memory.go | 304 +++++++++++ stack_go_io_windows.go | 104 +++- stack_go_listen.go | 240 +++++++++ stack_go_ring.go | 4 - stack_go_socket_unix.go | 15 +- stack_go_splice.go | 57 ++- stack_go_tcp_ack.go | 75 +-- stack_go_tcp_bbr.go | 36 +- stack_go_tcp_cong.go | 48 +- stack_go_tcp_cubic.go | 10 +- stack_go_tcp_input.go | 165 ++++-- stack_go_tcp_output.go | 182 ++++--- stack_go_tcp_rate.go | 6 +- stack_go_udp.go | 78 ++- stack_go_udp_socket.go | 462 +++++++++++++++++ stack_go_udp_test.go | 183 +++++++ stack_gvisor.go | 11 +- stack_gvisor_filter.go | 4 - stack_gvisor_tcp_conn.go | 5 +- stack_system.go | 11 +- tun_linux_gvisor.go | 2 - tun_memory.go | 290 +++++++++++ tun_windows_gvisor.go | 2 - udp_egress.go | 6 +- udp_nat.go | 127 +++-- 46 files changed, 4541 insertions(+), 575 deletions(-) create mode 100644 stack_go_conn_options.go create mode 100644 stack_go_dial.go create mode 100644 stack_go_integration_listen_test.go create mode 100644 stack_go_integration_memory_darwin_test.go create mode 100644 stack_go_integration_memory_linux_test.go create mode 100644 stack_go_integration_memory_test.go create mode 100644 stack_go_integration_options_test.go create mode 100644 stack_go_io_memory.go create mode 100644 stack_go_listen.go create mode 100644 stack_go_udp_socket.go create mode 100644 stack_go_udp_test.go create mode 100644 tun_memory.go diff --git a/flow_dispatch.go b/flow_dispatch.go index a17d3b7c..cf321fbc 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -845,8 +845,6 @@ func isICMPProtocol(protocol uint8) bool { return protocol == uint8(header.ICMPv4ProtocolNumber) || protocol == uint8(header.ICMPv6ProtocolNumber) } -var _ Return = (*forwardReturn)(nil) - type forwardReturn struct { dispatcher *ForwardDispatcher closed atomic.Bool diff --git a/flow_dns.go b/flow_dns.go index 9c525073..125e9586 100644 --- a/flow_dns.go +++ b/flow_dns.go @@ -9,7 +9,6 @@ import ( "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" M "github.com/sagernet/sing/common/metadata" - N "github.com/sagernet/sing/common/network" ) func (d *ForwardDispatcher) hijackDNSPacket(packet *forwardPacket) { @@ -20,8 +19,6 @@ func (d *ForwardDispatcher) hijackDNSPacket(packet *forwardPacket) { d.handler.NewDNSPacket(header.UDP(packet.transport).Payload(), M.SocksaddrFromNetIP(packet.source), M.SocksaddrFromNetIP(packet.destination), writer) } -var _ N.PacketWriter = (*dnsResponseWriter)(nil) - type dnsResponseWriter struct { writeback ForwardWriteback source netip.AddrPort diff --git a/stack_go.go b/stack_go.go index 7dea18a5..982f7d8c 100644 --- a/stack_go.go +++ b/stack_go.go @@ -9,32 +9,35 @@ import ( "sync/atomic" "time" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/logger" ) type Go struct { - ctx context.Context - tun Tun - mtu int - handler Handler - logger logger.Logger - broadcastAddr netip.Addr - inet4LoopbackAddress []netip.Addr - inet6LoopbackAddress []netip.Addr - udpTimeout time.Duration - icmpTimeout time.Duration - udpNATOptions UDPNatOptions - memoryPressure func() MemoryPressure - udpNats []*UDPNat - udpIdentification atomic.Uint32 - dispatcher *ForwardDispatcher - directory goFlowDirectory - congestion *goCongestionOps - queueFactory func(stack *Go) ([]goPlatformIO, error) - access sync.Mutex - engines []*goEngine - closed atomic.Bool + ctx context.Context + tun Tun + mtu int + handler Handler + logger logger.Logger + broadcastAddr netip.Addr + inet4LoopbackAddress []netip.Addr + inet6LoopbackAddress []netip.Addr + udpTimeout time.Duration + icmpTimeout time.Duration + udpNATOptions UDPNatOptions + memoryPressure func() MemoryPressure + udpNats []*UDPNat + fragmentIdentification atomic.Uint32 + dispatcher *ForwardDispatcher + directory goFlowDirectory + congestion *goCongestionOps + queueFactory func(stack *Go) ([]goPlatformIO, error) + access sync.Mutex + engines []*goEngine + closed atomic.Bool + validateChecksum bool } func NewGo(options StackOptions) (*Go, error) { @@ -42,7 +45,7 @@ func NewGo(options StackOptions) (*Go, error) { if err != nil { return nil, err } - return &Go{ + stack := &Go{ congestion: congestion, queueFactory: newGoPlatformQueues, ctx: options.Context, @@ -56,23 +59,32 @@ func NewGo(options StackOptions) (*Go, error) { udpTimeout: options.UDPTimeout, icmpTimeout: options.ICMPTimeout, udpNATOptions: UDPNatOptions{ - Timeout: options.UDPTimeout, - Mapping: options.UDPMapping, - Filtering: options.UDPFiltering, - MaxSize: options.UDPNATMax, - InterfaceFinder: options.InterfaceFinder, - ExcludeInterface: []string{options.TunOptions.Name}, + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, }, memoryPressure: options.MemoryPressure, - }, nil + directory: goFlowDirectory{ + flows: make(map[flowKey]*GoConn), + udpSockets: make(map[netip.AddrPort]*GoUDPConn), + listeners: make(map[netip.AddrPort]*GoListener), + }, + } + memoryTun, isMemoryTun := options.Tun.(*MemoryTun) + if isMemoryTun { + stack.validateChecksum = true + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goMemoryIO{stack: current, tun: memoryTun, handed: make([]*buf.Buffer, 0, goReadBatch)}}, nil + } + } + return stack, nil } func (s *Go) Start() error { s.access.Lock() defer s.access.Unlock() - if s.closed.Load() { - return E.New("stack is closed") - } queues, err := s.queueFactory(s) if err != nil { return err @@ -88,51 +100,53 @@ func (s *Go) Start() error { } return E.Errors(err, goCloseQueues(queues[:index])) } - udpNATOptions := s.udpNATOptions - udpNATOptions.Handler = s.handler - udpNATOptions.Prepare = s.prepareUDPConnection - maxSize := udpNATOptions.MaxSize - if maxSize == 0 { - if runtime.GOOS == "ios" { - maxSize = 4096 - } else { - maxSize = 16384 - } - } natCount := uint32(1) - if udpNATOptions.Mapping == NATMappingAddressAndPortDependent { - natCount = min(uint32(len(queues)), maxSize) - } - udpNats := make([]*UDPNat, natCount) - for index := range udpNats { - udpNATOptions.Shared = index+int(natCount) < len(queues) - udpNATOptions.MaxSize = maxSize / natCount - if uint32(index) < maxSize%natCount { - udpNATOptions.MaxSize++ + var udpNats []*UDPNat + if s.handler != nil { + udpNATOptions := s.udpNATOptions + udpNATOptions.Handler = s.handler + udpNATOptions.Prepare = s.prepareUDPConnection + maxSize := udpNATOptions.MaxSize + if maxSize == 0 { + if runtime.GOOS == "ios" { + maxSize = 4096 + } else { + maxSize = 16384 + } } - udpNat := NewUDPNat(udpNATOptions) - err = udpNat.Start() - if err != nil { - udpNat.Close() - for _, started := range udpNats[:index] { - started.Close() + if udpNATOptions.Mapping == NATMappingAddressAndPortDependent { + natCount = min(uint32(len(queues)), maxSize) + } + udpNats = make([]*UDPNat, natCount) + for index := range udpNats { + udpNATOptions.Shared = index+int(natCount) < len(queues) + udpNATOptions.MaxSize = maxSize / natCount + if uint32(index) < maxSize%natCount { + udpNATOptions.MaxSize++ + } + udpNat := NewUDPNat(udpNATOptions) + err = udpNat.Start() + if err != nil { + udpNat.Close() + for _, started := range udpNats[:index] { + started.Close() + } + return E.Errors(err, goCloseQueues(queues)) } - return E.Errors(err, goCloseQueues(queues)) + udpNats[index] = udpNat } - udpNats[index] = udpNat + s.dispatcher = NewForwardDispatcher(s.handler, &goWriteback{platformIO: queues[0]}, s.logger, s.udpTimeout, s.icmpTimeout) } s.udpNats = udpNats - s.dispatcher = NewForwardDispatcher(s.handler, &goWriteback{platformIO: queues[0]}, s.logger, s.udpTimeout, s.icmpTimeout) - if len(queues) > 1 { - s.directory.flows = make(map[flowKey]*GoConn) - if natCount > 1 { - s.directory.udpFlows = make(map[udpNatSessionKey]*GoPacketConn) - } + if natCount > 1 { + s.directory.udpFlows = make(map[udpNatSessionKey]*GoPacketConn) } s.engines = make([]*goEngine, len(queues)) for index, queue := range queues { s.engines[index] = newGoEngine(s, queue, len(queues)) - s.engines[index].udpNat = udpNats[index%int(natCount)] + if udpNats != nil { + s.engines[index].udpNat = udpNats[index%int(natCount)] + } } for _, engine := range s.engines { go engine.run() @@ -148,26 +162,104 @@ func goCloseQueues(queues []goPlatformIO) error { return err } +func (s *Go) HasEndpoint(protocol uint8, local, remote netip.AddrPort) bool { + if s.closed.Load() || !local.IsValid() || !remote.IsValid() { + return false + } + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + remote = netip.AddrPortFrom(remote.Addr().Unmap(), remote.Port()) + if local.Addr().Is4() != remote.Addr().Is4() { + return false + } + s.directory.access.RLock() + defer s.directory.access.RUnlock() + switch protocol { + case uint8(header.TCPProtocolNumber): + return s.directory.flows[flowKey{protocol: protocol, source: remote, destination: local}] != nil || s.directory.listeners[local] != nil + case uint8(header.UDPProtocolNumber): + socket := s.directory.udpSockets[local] + return socket != nil && (!socket.connected || socket.remote == remote) + default: + return false + } +} + type goFlowDirectory struct { - access sync.RWMutex - flows map[flowKey]*GoConn - udpFlows map[udpNatSessionKey]*GoPacketConn + access sync.RWMutex + flows map[flowKey]*GoConn + udpFlows map[udpNatSessionKey]*GoPacketConn + udpSockets map[netip.AddrPort]*GoUDPConn + listeners map[netip.AddrPort]*GoListener } -func (d *goFlowDirectory) insert(key flowKey, conn *GoConn) { - if d.flows == nil { - return +func (d *goFlowDirectory) insertListener(listener *GoListener) { + d.access.Lock() + if len(listener.engine.stack.engines) > 1 { + listener.engine.tcpListeners[listener.local] = listener + } + d.listeners[listener.local] = listener + d.access.Unlock() +} + +func (d *goFlowDirectory) removeListener(listener *GoListener) { + d.access.Lock() + if len(listener.engine.stack.engines) > 1 && listener.engine.tcpListeners[listener.local] == listener { + delete(listener.engine.tcpListeners, listener.local) + } + if d.listeners[listener.local] == listener { + delete(d.listeners, listener.local) + } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookupListener(local netip.AddrPort) *GoListener { + d.access.RLock() + listener := d.listeners[local] + d.access.RUnlock() + return listener +} + +func (d *goFlowDirectory) insertUDPSocket(socket *GoUDPConn) { + d.access.Lock() + if len(socket.engine.stack.engines) > 1 { + socket.engine.udpSockets[socket.local] = socket + } + d.udpSockets[socket.local] = socket + d.access.Unlock() +} + +func (d *goFlowDirectory) removeUDPSocket(socket *GoUDPConn) { + d.access.Lock() + if len(socket.engine.stack.engines) > 1 && socket.engine.udpSockets[socket.local] == socket { + delete(socket.engine.udpSockets, socket.local) + } + if d.udpSockets[socket.local] == socket { + delete(d.udpSockets, socket.local) } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookupUDPSocket(local netip.AddrPort) *GoUDPConn { + d.access.RLock() + socket := d.udpSockets[local] + d.access.RUnlock() + return socket +} + +func (d *goFlowDirectory) insert(key flowKey, conn *GoConn) { d.access.Lock() + if len(conn.engine.stack.engines) > 1 { + conn.engine.flows[key] = conn + } d.flows[key] = conn d.access.Unlock() } func (d *goFlowDirectory) remove(key flowKey, conn *GoConn) { - if d.flows == nil { - return - } d.access.Lock() + if len(conn.engine.stack.engines) > 1 && conn.engine.flows[key] == conn { + delete(conn.engine.flows, key) + } if d.flows[key] == conn { delete(d.flows, key) } @@ -175,9 +267,6 @@ func (d *goFlowDirectory) remove(key flowKey, conn *GoConn) { } func (d *goFlowDirectory) lookup(key flowKey) *GoConn { - if d.flows == nil { - return nil - } d.access.RLock() conn := d.flows[key] d.access.RUnlock() @@ -193,6 +282,7 @@ func (s *Go) ResetNetwork() { for _, udpNat := range s.udpNats { udpNat.Purge() } + s.dispatcher.ResetNetwork() for _, engine := range s.engines { engine.postMessage(&engine.resetMessage) } diff --git a/stack_go_conn.go b/stack_go_conn.go index a7ee305a..d6f29143 100644 --- a/stack_go_conn.go +++ b/stack_go_conn.go @@ -25,6 +25,7 @@ const ( const ( goTCPSynReceived uint8 = iota + goTCPSynSent goTCPEstablished goTCPFinWait1 goTCPFinWait2 @@ -64,12 +65,11 @@ var ( type GoConn struct { engine *goEngine key flowKey - source M.Socksaddr - destination M.Socksaddr + peer M.Socksaddr + local M.Socksaddr epoch int64 receiveNext uint64 receiveCapacity uint64 - receiveCapacityMax uint64 publishedEdge uint64 sentEdge uint64 receiveRoundTripMark uint64 @@ -104,6 +104,7 @@ type GoConn struct { persistDeadline int64 lingerDeadline int64 handshakeDeadline int64 + handshakeStamp int64 idleDeadline int64 pacingDeadline int64 reorderDeadline int64 @@ -167,10 +168,11 @@ type GoConn struct { peerWindowShift uint8 localWindowShift uint8 peerMSS uint16 - effectiveMSS uint16 + effectiveMSS atomic.Uint32 lastPeerWindow uint16 - synAckLength uint8 + handshakeLength uint8 keyed bool + socket bool finPending bool sackPermitted bool timestampsEnabled bool @@ -189,7 +191,8 @@ type GoConn struct { blockedNext *GoConn dyingNext *GoConn splice *goSpliceStream - synAckImage [96]byte + listener *GoListener + handshakeImage [96]byte _ [64]byte sendPermit atomic.Uint64 @@ -204,6 +207,9 @@ type GoConn struct { deliveryState atomic.Uint64 pacingRate atomic.Uint64 pacingStamp atomic.Int64 + queuedBytes atomic.Int64 + departedTail atomic.Uint64 + queueThrottled atomic.Bool firstSentStamp atomic.Int32 appLimited atomic.Uint32 frameLimit atomic.Uint32 @@ -248,6 +254,14 @@ type GoConn struct { access sync.Mutex phase uint8 + receiveCapacityMax uint64 + transmitCapacity uint64 + keepaliveIdle time.Duration + keepaliveInterval time.Duration + keepaliveCount uint8 + keepaliveEnabled bool + nagle bool + linger int dead bool finReceived bool drainable bool @@ -263,8 +277,10 @@ type GoConn struct { err error splicePending *goSpliceStream establishedSignal chan struct{} + finAckedSignal chan struct{} closeSignal chan struct{} transmitSignal goSignal + dialMessage goMessage engageMessage goMessage closeMessage goMessage readShutMessage goMessage @@ -274,16 +290,17 @@ type GoConn struct { blockedMessage goMessage droppedMessage goMessage pacingMessage goMessage + throttleMessage goMessage spliceMessage goMessage slabHolder goSlabHolder } -func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, destination M.Socksaddr) { +func (c *GoConn) initialize(engine *goEngine, key flowKey, peer M.Socksaddr, local M.Socksaddr) { c.engine = engine c.key = key - c.source = source - c.destination = destination - c.epoch = engine.now() + c.peer = peer + c.local = local + c.epoch = engine.coarseTime.Load() c.lastSendStamp = -1 c.readDeadline = pipe.MakeDeadline() c.writeDeadline = pipe.MakeDeadline() @@ -291,11 +308,19 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, d c.writeSignal = make(goSignal, 1) c.transmitSignal = make(goSignal, 1) c.establishedSignal = make(chan struct{}) + c.finAckedSignal = make(chan struct{}) c.closeSignal = make(chan struct{}) + c.receiveCapacityMax = goReceiveCapacityMax + c.transmitCapacity = goTransmitCapacityMax + c.keepaliveIdle = tcpEstablishedTimeout + c.keepaliveInterval = goKeepaliveInterval + c.keepaliveCount = goKeepaliveCount + c.keepaliveEnabled = true + c.linger = -1 c.receiveChain = goSlabChain{maxSlots: goReceiveCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} c.transmitStore.chain = goSlabChain{maxSlots: goTransmitCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} c.descriptors.pool = &engine.descriptorPool - c.timerNode.expire = c.expireTimer + c.timerNode.expire = func(now int64) { engine.fireConnTimer(c, now) } c.engageMessage = goMessage{kind: goMessageConnEngage, conn: c} c.closeMessage = goMessage{kind: goMessageConnClose, conn: c} c.readShutMessage = goMessage{kind: goMessageConnReadShut, conn: c} @@ -305,9 +330,21 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, source M.Socksaddr, d c.blockedMessage = goMessage{kind: goMessageConnBlocked, conn: c} c.droppedMessage = goMessage{kind: goMessageConnDropped, conn: c} c.pacingMessage = goMessage{kind: goMessageConnPacing, conn: c} + c.throttleMessage = goMessage{kind: goMessageConnThrottle, conn: c} c.spliceMessage = goMessage{kind: goMessageConnSplice, conn: c} } +func (c *GoConn) handshaking() bool { + return c.state == goTCPSynReceived || c.state == goTCPSynSent +} + +func (c *GoConn) receiveWindowBound() uint64 { + c.access.Lock() + capacityMax := c.receiveCapacityMax + c.access.Unlock() + return min(capacityMax, uint64(0xffff)< 0 { + timer := time.NewTimer(time.Duration(linger) * time.Second) + defer timer.Stop() + select { + case <-c.finAckedSignal: + case <-c.closeSignal: + case <-timer.C: + } + } return nil } @@ -686,11 +733,17 @@ func (c *GoConn) CloseWrite() error { } func (c *GoConn) LocalAddr() net.Addr { - return c.source.TCPAddr() + if c.socket { + return c.local.TCPAddr() + } + return c.peer.TCPAddr() } func (c *GoConn) RemoteAddr() net.Addr { - return c.destination.TCPAddr() + if c.socket { + return c.peer.TCPAddr() + } + return c.local.TCPAddr() } func (c *GoConn) SetDeadline(t time.Time) error { @@ -728,7 +781,6 @@ func (s goSignal) drain() { } var ( - _ net.Conn = (*GoConn)(nil) _ N.ReadWaiter = (*GoConn)(nil) _ N.ExtendedWriter = (*GoConn)(nil) _ N.FrontHeadroom = (*GoConn)(nil) diff --git a/stack_go_conn_options.go b/stack_go_conn_options.go new file mode 100644 index 00000000..6a2be215 --- /dev/null +++ b/stack_go_conn_options.go @@ -0,0 +1,84 @@ +package tun + +import ( + "net" + "time" +) + +const ( + goKeepaliveDefaultIdle = 5 * time.Minute + goKeepaliveDefaultInterval = 75 * time.Second + goKeepaliveDefaultCount = 9 + goKeepaliveMaxCount = 127 +) + +func (c *GoConn) SetKeepAlive(keepalive bool) error { + c.access.Lock() + c.keepaliveEnabled = keepalive + c.access.Unlock() + return nil +} + +func (c *GoConn) SetKeepAlivePeriod(d time.Duration) error { + c.access.Lock() + c.setKeepaliveIdle(d) + c.access.Unlock() + return nil +} + +func (c *GoConn) SetKeepAliveConfig(config net.KeepAliveConfig) error { + c.access.Lock() + c.keepaliveEnabled = config.Enable + c.setKeepaliveIdle(config.Idle) + switch { + case config.Interval == 0: + c.keepaliveInterval = goKeepaliveDefaultInterval + case config.Interval > 0: + c.keepaliveInterval = config.Interval + } + switch { + case config.Count == 0: + c.keepaliveCount = goKeepaliveDefaultCount + case config.Count > 0: + c.keepaliveCount = uint8(min(config.Count, goKeepaliveMaxCount)) + } + c.access.Unlock() + return nil +} + +func (c *GoConn) setKeepaliveIdle(idle time.Duration) { + switch { + case idle == 0: + c.keepaliveIdle = goKeepaliveDefaultIdle + case idle > 0: + c.keepaliveIdle = idle + } +} + +func (c *GoConn) SetNoDelay(noDelay bool) error { + c.access.Lock() + c.nagle = !noDelay + c.access.Unlock() + return nil +} + +func (c *GoConn) SetLinger(sec int) error { + c.access.Lock() + c.linger = sec + c.access.Unlock() + return nil +} + +func (c *GoConn) SetReadBuffer(size int) error { + c.access.Lock() + c.receiveCapacityMax = uint64(min(max(size, goSlabSize), goReceiveCapacityMax)) + c.access.Unlock() + return nil +} + +func (c *GoConn) SetWriteBuffer(size int) error { + c.access.Lock() + c.transmitCapacity = uint64(min(max(size, goSlabSize), goTransmitCapacityMax)) + c.access.Unlock() + return nil +} diff --git a/stack_go_dial.go b/stack_go_dial.go new file mode 100644 index 00000000..a6777163 --- /dev/null +++ b/stack_go_dial.go @@ -0,0 +1,339 @@ +package tun + +import ( + "context" + "hash/maphash" + "math/rand/v2" + "net" + "net/netip" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" +) + +const ( + goSynRetransmit = time.Second + goSynAttempts = 6 + goEphemeralPortBase = 32768 + goEphemeralPortSpan = 60999 - goEphemeralPortBase + 1 + goEphemeralAttempts = 1024 +) + +var errGoInvalidAddress = E.New("go: invalid address") + +func (s *Go) DialTCP(ctx context.Context, source netip.Addr, destination netip.AddrPort) (*GoConn, error) { + source = source.Unmap() + destination = netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !source.IsValid() || !destination.IsValid() || destination.Port() == 0 || source.Is4() != destination.Addr().Is4() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(destination) + if err != nil { + return nil, err + } + conn := new(GoConn) + conn.initialize(engine, flowKey{}, M.SocksaddrFromNetIP(destination), M.SocksaddrFrom(source, 0)) + conn.socket = true + conn.ipVersion = goIPVersion(source) + conn.state = goTCPSynSent + conn.phase = goPhaseEngaged + conn.localWindowShift = goLocalWindowShift + conn.timestampsEnabled = true + conn.sackPermitted = true + conn.receiveCapacity = min(goReceiveCapacityBase, conn.receiveWindowBound()) + conn.dialMessage = goMessage{kind: goMessageConnDial, conn: conn} + engine.postMessage(&conn.dialMessage) + err = conn.awaitHandshake(ctx, nil) + if err != nil { + return nil, err + } + return conn, nil +} + +func goIPVersion(addr netip.Addr) uint8 { + if addr.Is4() { + return 4 + } + return 6 +} + +func (s *Go) selectEngine(destination netip.AddrPort) (*goEngine, error) { + s.access.Lock() + defer s.access.Unlock() + if s.closed.Load() { + return nil, net.ErrClosed + } + return s.engines[maphash.Comparable(s.engines[0].sequenceSeed, destination)%uint64(len(s.engines))], nil +} + +func goAllocatePort(available func(port uint16) bool) (uint16, bool) { + offset := rand.IntN(goEphemeralPortSpan) + for attempt := range goEphemeralAttempts { + port := uint16(goEphemeralPortBase + (offset+attempt)%goEphemeralPortSpan) + if available(port) { + return port, true + } + } + return 0, false +} + +func (e *goEngine) handleDial(conn *GoConn) { + if conn.dead { + return + } + if len(e.flows) >= e.flowCapacity && !e.evictFlow() { + e.detachConn(conn, E.Cause(syscall.ENOBUFS, "go: flow table full"), goDeathImmediate) + return + } + key := flowKey{protocol: uint8(header.TCPProtocolNumber), source: conn.peer.AddrPort()} + port, allocated := goAllocatePort(func(port uint16) bool { + key.destination = netip.AddrPortFrom(conn.local.Addr, port) + return e.flows[key] == nil && (len(e.stack.engines) == 1 || e.stack.directory.lookup(key) == nil) && e.lookupListener(key.destination) == nil + }) + if !allocated { + e.detachConn(conn, E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted"), goDeathImmediate) + return + } + now := e.coarseTime.Load() + conn.key = key + conn.local.Port = port + conn.sendISN = e.initialSequence(key, now) + conn.lastActivity = now + conn.buildHandshake(e.localMSS(conn.ipVersion), true, true, header.TCPFlagSyn, 0) + conn.keyed = true + e.stack.directory.insert(key, conn) + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.detachConn(conn, E.Cause(err, "go: send SYN"), goDeathImmediate) + return + } + conn.handshakeAttempts = 0 + conn.handshakeStamp = now + conn.handshakeDeadline = now + int64(goSynRetransmit) + e.rearmTimer(conn) +} + +func (e *goEngine) retransmitSyn(conn *GoConn, now int64) { + conn.handshakeAttempts++ + if conn.handshakeAttempts >= goSynAttempts { + conn.handshakeDeadline = 0 + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: connect timeout"), goDeathImmediate) + return + } + conn.buildHandshake(e.localMSS(conn.ipVersion), true, true, header.TCPFlagSyn, 0) + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN")) + } + conn.handshakeDeadline = now + int64(goSynRetransmit)<= 0 { + conn.peerWindowShift = uint8(synOptions.WS) + } else { + conn.localWindowShift = 0 + } + if synOptions.TS { + conn.tsRecent.Store(synOptions.TSVal) + effectiveMSS -= goTimestampOptionLength + } else { + conn.timestampsEnabled = false + } + conn.effectiveMSS.Store(uint32(effectiveMSS)) + e.initializeSession(conn, now, tcpHdr.WindowSize()) + if conn.handshakeAttempts == 0 { + roundTrip := max((now-conn.handshakeStamp)/int64(time.Microsecond), 1) + e.ackUpdateRoundTrip(conn, 0, roundTrip, -1, roundTrip, &e.rateSample, 0, false, now) + } + e.establishConn(conn) + e.rearmTimer(conn) + e.sendAck(conn) +} + +func (e *goEngine) inputICMPError(parsed *forwardPacket) { + inner, ok := parsed.icmpErrorInner() + if !ok { + return + } + embedded, ok := parseEmbedded(inner) + if !ok { + return + } + mtu, tooBig := goICMPPacketTooBig(parsed) + if tooBig { + e.inputPacketTooBig(&embedded, mtu) + return + } + err := goICMPError(parsed) + if err == nil { + return + } + switch embedded.protocol { + case uint8(header.TCPProtocolNumber): + conn := e.flows[embedded.flowKey().reversed()] + if conn == nil || conn.dead || conn.state != goTCPSynSent { + return + } + if len(embedded.payload) < header.TCPAckNumOffset || header.TCP(embedded.payload).SequenceNumber() != conn.sendISN { + return + } + e.detachConn(conn, E.Cause(err, "go: connect"), goDeathImmediate) + case uint8(header.UDPProtocolNumber): + socket := e.udpSockets[embedded.source] + if socket == nil || !socket.connected || socket.remote != embedded.destination { + return + } + socket.deliverError(err) + } +} + +func goICMPError(parsed *forwardPacket) error { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + icmpHdr := header.ICMPv4(parsed.transport) + switch icmpHdr.Type() { + case header.ICMPv4DstUnreachable: + switch icmpHdr.Code() { + case header.ICMPv4NetUnreachable, header.ICMPv4NetUnreachableForTos: + return syscall.ENETUNREACH + case header.ICMPv4HostUnreachable, header.ICMPv4HostUnreachableForTos: + return syscall.EHOSTUNREACH + case header.ICMPv4ProtoUnreachable: + return syscall.EPROTO + case header.ICMPv4PortUnreachable: + return syscall.ECONNREFUSED + case header.ICMPv4NetProhibited, header.ICMPv4HostProhibited: + return syscall.EACCES + default: + return syscall.EHOSTUNREACH + } + case header.ICMPv4TimeExceeded: + return syscall.EHOSTUNREACH + case header.ICMPv4ParamProblem: + return syscall.EPROTO + default: + return nil + } + case uint8(header.ICMPv6ProtocolNumber): + icmpHdr := header.ICMPv6(parsed.transport) + switch icmpHdr.Type() { + case header.ICMPv6DstUnreachable: + switch icmpHdr.Code() { + case header.ICMPv6NetworkUnreachable: + return syscall.ENETUNREACH + case header.ICMPv6Prohibited: + return syscall.EACCES + case header.ICMPv6PortUnreachable: + return syscall.ECONNREFUSED + default: + return syscall.EHOSTUNREACH + } + case header.ICMPv6TimeExceeded: + return syscall.EHOSTUNREACH + case header.ICMPv6ParamProblem: + return syscall.EPROTO + default: + return nil + } + default: + return nil + } +} + +func goICMPPacketTooBig(parsed *forwardPacket) (int, bool) { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + icmpHdr := header.ICMPv4(parsed.transport) + if icmpHdr.Type() != header.ICMPv4DstUnreachable || icmpHdr.Code() != header.ICMPv4FragmentationNeeded { + return 0, false + } + return int(icmpHdr.MTU()), true + case uint8(header.ICMPv6ProtocolNumber): + icmpHdr := header.ICMPv6(parsed.transport) + if icmpHdr.Type() != header.ICMPv6PacketTooBig { + return 0, false + } + return int(icmpHdr.MTU()), true + default: + return 0, false + } +} + +func (e *goEngine) inputPacketTooBig(embedded *embeddedPacket, mtu int) { + if embedded.protocol != uint8(header.TCPProtocolNumber) || mtu == 0 { + return + } + conn := e.flows[embedded.flowKey().reversed()] + if conn == nil || conn.dead || conn.state < goTCPEstablished || len(embedded.payload) < header.TCPAckNumOffset { + return + } + offset := conn.sendOffset(header.TCP(embedded.payload).SequenceNumber()) + if offset < int64(conn.sendUnacked.Load()) || offset >= int64(conn.sendNext()) { + return + } + if conn.ipVersion == 4 { + mtu = max(mtu, header.IPv4MinimumProcessableDatagramSize) + } else { + mtu = max(mtu, header.IPv6MinimumMTU) + } + mss := mtu - goNetworkHeaderLength(conn.ipVersion) - header.TCPMinimumSize + if conn.timestampsEnabled { + mss -= goTimestampOptionLength + } + if mss >= int(conn.effectiveMSS.Load()) { + return + } + e.drainDescriptors(conn) + lost := false + start := conn.sendUnacked.Load() + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.endOffset-start > uint64(mss) && descriptor.flags&(goDescriptorSacked|goDescriptorLost) == 0 { + e.markLost(conn, descriptor) + lost = true + } + start = descriptor.endOffset + } + conn.effectiveMSS.Store(uint32(mss)) + e.refreshPacketsOut(conn) + if lost { + e.enterCWR(conn) + if conn.congestionState < goCongestionRecovery { + e.enterRecovery(conn) + } + e.xmitRetransmitQueue(conn) + } + e.rearmRetransmit(conn) + conn.publishPermit(conn.sendUnacked.Load()) +} diff --git a/stack_go_engine.go b/stack_go_engine.go index da4e63dd..c781aa94 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -1,16 +1,20 @@ package tun import ( + "errors" "hash/maphash" "math" "net" "net/netip" + "os" "runtime" "slices" "sync" "sync/atomic" "time" + tcpip "github.com/sagernet/sing-tun/gtcpip" + "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/buf" @@ -28,6 +32,7 @@ const ( const ( goMessageStackClose uint8 = iota goMessageStackReset + goMessageConnDial goMessageConnEngage goMessageConnClose goMessageConnReadShut @@ -37,9 +42,14 @@ const ( goMessageConnBlocked goMessageConnDropped goMessageConnPacing + goMessageConnThrottle goMessageConnSplice goMessagePacketSplice goMessagePacketClose + goMessageUDPOpen + goMessageUDPClose + goMessageListenOpen + goMessageListenClose goMessageInject ) @@ -78,11 +88,13 @@ func (s *goInjectStack) popAll() *goInjectedFrame { } type goMessage struct { - next *goMessage - conn *GoConn - packet *GoPacketConn - queued atomic.Bool - kind uint8 + next *goMessage + conn *GoConn + packet *GoPacketConn + socket *GoUDPConn + listener *GoListener + queued atomic.Bool + kind uint8 } type goControlStack struct { @@ -159,6 +171,8 @@ type goEngine struct { packetIO goPacketBatchIO packetFlows map[flowKey]*GoPacketConn packetReadOptions atomic.Pointer[N.ReadWaitOptions] + udpSockets map[netip.AddrPort]*GoUDPConn + tcpListeners map[netip.AddrPort]*GoListener flows map[flowKey]*GoConn flowCapacity int @@ -167,6 +181,7 @@ type goEngine struct { sequenceSeed maphash.Seed ackList *GoConn blockedList *GoConn + blockedTail *GoConn dyingList *GoConn reclaimPending bool resetBurst int @@ -195,12 +210,11 @@ func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine stack: stack, platformIO: platformIO, dispatcher: stack.dispatcher, - dispatchStage: stack.dispatcher.NewStage(&goWriteback{platformIO: platformIO}), epoch: time.Now(), frames: make([]goFrame, goReadBatch), reassemblyEntries: make([]goReassemblyEntry, goReassemblyEntries), exitSignal: make(chan struct{}), - flows: make(map[flowKey]*GoConn), + flows: stack.directory.flows, flowCapacity: max(goFlowCapacity/engineCount, 1024), slabPool: newGoSlabPool(stack.memoryPressure, max(goSlabPoolLowWater/engineCount, 8)), descriptorPool: goDescriptorPool{lowWater: max(goDescriptorPoolLowWater/engineCount, 4)}, @@ -211,31 +225,41 @@ func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine spliceIovecs: make([]goIOVector, 0, 8), packetFlows: make(map[flowKey]*GoPacketConn, goReadBatch), packetReceiveBatch: goReceiveBatchMin, + udpSockets: stack.directory.udpSockets, + tcpListeners: stack.directory.listeners, + } + if engineCount > 1 { + engine.flows = make(map[flowKey]*GoConn) + engine.udpSockets = make(map[netip.AddrPort]*GoUDPConn) + engine.tcpListeners = make(map[netip.AddrPort]*GoListener) + } + if stack.dispatcher != nil { + engine.dispatchStage = stack.dispatcher.NewStage(&goWriteback{platformIO: platformIO}) } engine.closeMessage.kind = goMessageStackClose engine.resetMessage.kind = goMessageStackReset engine.injectMessage.kind = goMessageInject engine.packetReadOptions.Store(new(N.ReadWaitOptions)) - engine.delayedAckTickNode.expire = engine.expireDelayedAckTick + engine.delayedAckTickNode.expire = func(now int64) { engine.drainAckList(now, false) } engine.reassemblyTickNode.expire = engine.expireReassemblyTick engine.sweepTickNode.expire = engine.expireSweepTick engine.reclaimTickNode.expire = engine.expireReclaimTick engine.refreshCoarseTime() - engine.wheel.currentTick = engine.now() / goWheelTick - engine.wheel.schedule(&engine.sweepTickNode, engine.now()+int64(goSweepInterval)) - engine.wheel.schedule(&engine.reclaimTickNode, engine.now()+int64(goReclaimInterval)) + engine.wheel.currentTick = engine.coarseTime.Load() / goWheelTick + engine.wheel.schedule(&engine.sweepTickNode, engine.coarseTime.Load()+int64(goSweepInterval)) + engine.wheel.schedule(&engine.reclaimTickNode, engine.coarseTime.Load()+int64(goReclaimInterval)) return engine } func (e *goEngine) run() { defer close(e.exitSignal) defer e.exit() - e.loadWindowStart = e.now() + e.loadWindowStart = e.coarseTime.Load() for { - parkStart := e.readClock() + parkStart := int64(time.Since(e.epoch)) tunReadable, eventCount, err := e.park() - e.parkedNanos += e.now() - parkStart - e.updateLoad(e.now()) + e.parkedNanos += e.coarseTime.Load() - parkStart + e.updateLoad(e.coarseTime.Load()) e.drainControlQueue() if e.stack.closed.Load() { return @@ -246,7 +270,9 @@ func (e *goEngine) run() { e.releaseBlockedWriters() } if err != nil { - e.stack.logger.Error(E.Cause(err, "go: engine wait")) + if !errors.Is(err, os.ErrClosed) { + e.stack.logger.Error(E.Cause(err, "go: engine wait")) + } return } e.resetBurst = 0 @@ -266,9 +292,11 @@ func (e *goEngine) run() { } e.dispatchSocketEvents(eventCount) e.flushSpliceDirty() - e.wheel.advance(e.now()) - e.drainAckList(e.now(), true) - e.dispatchStage.Flush() + e.wheel.advance(e.coarseTime.Load()) + e.drainAckList(e.coarseTime.Load(), true) + if e.dispatchStage != nil { + e.dispatchStage.Flush() + } e.flushPacketUploads() e.flushPacketFrames() e.platformIO.flush() @@ -288,14 +316,7 @@ func (e *goEngine) exit() { e.shutdown() e.releaseReadBuffers() e.engineState.Store(goEngineExited) - message := e.controlStack.head.Swap(nil) - for message != nil { - next := message.next - message.next = nil - message.queued.Store(false) - e.releasePending(message) - message = next - } + e.releaseControlQueue() e.releaseInjected() e.slabPool.close() if e.dyingList != nil { @@ -337,7 +358,7 @@ func (e *goEngine) park() (bool, int, error) { } timeout := goWaitIndefinite if scheduled { - timeout = max(time.Duration(deadline-e.now()), 0) + timeout = max(time.Duration(deadline-e.coarseTime.Load()), 0) } if e.tunPending { timeout = 0 @@ -360,7 +381,7 @@ func (e *goEngine) idleTimeout(timeout time.Duration) time.Duration { if !e.readBuffersHeld { return timeout } - now := e.now() + now := e.coarseTime.Load() if e.idleSince < 0 { e.idleSince = now } @@ -414,7 +435,18 @@ func (e *goEngine) postMessage(message *goMessage) { case goEngineParked: e.platformIO.wake() case goEngineExited: + e.releaseControlQueue() + } +} + +func (e *goEngine) releaseControlQueue() { + message := e.controlStack.head.Swap(nil) + for message != nil { + next := message.next + message.next = nil + message.queued.Store(false) e.releasePending(message) + message = next } } @@ -436,8 +468,9 @@ func (e *goEngine) drainControlQueue() { func (e *goEngine) handleMessage(message *goMessage) { switch message.kind { case goMessageStackReset: - e.dispatcher.ResetNetwork() e.reclaimPending = true + case goMessageConnDial: + e.handleDial(message.conn) case goMessageConnEngage: e.handleEngage(message.conn) case goMessageConnClose: @@ -456,12 +489,24 @@ func (e *goEngine) handleMessage(message *goMessage) { e.handleDroppedFrames(message.conn) case goMessageConnPacing: e.handlePacingRequest(message.conn) + case goMessageConnThrottle: + if !message.conn.dead { + message.conn.wakeTransmitter() + } case goMessageConnSplice: e.handleSpliceEngage(message.conn) case goMessagePacketSplice: e.handlePacketSpliceEngage(message.packet) case goMessagePacketClose: e.handlePacketSpliceClose(message.packet) + case goMessageUDPOpen: + e.handleUDPOpen(message.socket) + case goMessageUDPClose: + e.handleUDPClose(message.socket) + case goMessageListenOpen: + e.handleListenOpen(message.listener) + case goMessageListenClose: + e.handleListenClose(message.listener) case goMessageInject: e.processInjected() } @@ -509,19 +554,21 @@ func (e *goEngine) handleEngage(conn *GoConn) { conn.access.Lock() conn.phase = goPhaseEngaged conn.access.Unlock() - err := conn.writeSynAck() + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) if err != nil { e.sendReset(conn) e.detachConn(conn, E.Cause(err, "go: send SYN-ACK"), goDeathAbortLinger) return } conn.handshakeAttempts = 0 - conn.handshakeDeadline = e.now() + int64(goSynAckRetransmit) + conn.handshakeDeadline = e.coarseTime.Load() + int64(goSynAckRetransmit) e.rearmTimer(conn) } func (e *goEngine) shutdown() { e.closeAllPacketSplices() + e.closeAllUDPSockets() + e.closeAllListeners() for index := range e.reassemblyEntries { e.reassemblyEntries[index].active = false } @@ -533,7 +580,7 @@ func (e *goEngine) shutdown() { conn.drainable = false conn.access.Unlock() } - deadline := e.now() + int64(goShutdownBudget) + deadline := e.coarseTime.Load() + int64(goShutdownBudget) for len(unreset) > 0 { e.releaseReadBuffers() remaining := deadline - e.refreshCoarseTime() @@ -598,22 +645,52 @@ func (e *goEngine) processFrame(frame *goFrame) { if !ok { return } + if e.stack.validateChecksum && parsed.ipVersion == 4 && !header.IPv4(parsed.network).IsChecksumValid() { + return + } if e.dropNonUnicast(&parsed) { return } if parsed.fragment { - e.reassemble(frame.buffer.Bytes(), &parsed) + if parsed.ipVersion == 4 { + e.reassembleIPv4(frame.buffer.Bytes(), &parsed) + } else { + e.reassembleIPv6(frame.buffer.Bytes(), &parsed) + } return } e.processParsed(frame.buffer, frame.meta, &parsed) } func (e *goEngine) processParsed(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + if parsed.protocol == uint8(header.UDPProtocolNumber) { + if len(parsed.transport) < header.UDPMinimumSize { + return + } + length := int(header.UDP(parsed.transport).Length()) + if length < header.UDPMinimumSize || length > len(parsed.transport) { + return + } + parsed.transport = parsed.transport[:length] + } + if e.stack.validateChecksum && !goValidateTransportChecksum(parsed) { + return + } packet := buffer.Bytes() if e.handleLoopbackHairpin(packet, meta, parsed) { return } if parsed.protocol == uint8(header.UDPProtocolNumber) && parsed.hasFlow { + if socket := e.udpSockets[parsed.destination]; socket != nil { + e.inputUDPSocket(socket, parsed) + return + } + if len(e.stack.engines) > 1 { + if owner := e.stack.directory.lookupUDPSocket(parsed.destination); owner != nil && owner.engine != e { + owner.engine.inject(packet, meta) + return + } + } writer := e.packetFlows[parsed.flowKey()] if writer != nil && writer.splice != nil { conn := writer.conn.Load() @@ -630,11 +707,14 @@ func (e *goEngine) processParsed(buffer *buf.Buffer, meta ForwardFrameMeta, pars e.inputTCP(conn, parsed) return } + } else if parsed.hasFlow && parsed.isPureTCPSyn() && e.lookupListener(parsed.destination) != nil { + e.demuxL4(buffer, meta, parsed) + return } - if e.dispatchStage.DispatchParsed(packet, meta, parsed) { + if e.dispatchStage != nil && e.dispatchStage.DispatchParsed(packet, meta, parsed) { return } - if establishedTCP { + if establishedTCP && len(e.stack.engines) > 1 { owner := e.stack.directory.lookup(parsed.flowKey()) if owner != nil && owner.engine != e { owner.engine.inject(packet, meta) @@ -644,6 +724,39 @@ func (e *goEngine) processParsed(buffer *buf.Buffer, meta ForwardFrameMeta, pars e.demuxL4(buffer, meta, parsed) } +func goValidateTransportChecksum(parsed *forwardPacket) bool { + transport := parsed.transport + switch parsed.protocol { + case uint8(header.TCPProtocolNumber): + if len(transport) < header.TCPMinimumSize { + return false + } + case uint8(header.UDPProtocolNumber): + if len(transport) < header.UDPMinimumSize { + return false + } + udpHeader := header.UDP(transport) + length := int(udpHeader.Length()) + if length < header.UDPMinimumSize || length > len(transport) { + return false + } + if udpHeader.Checksum() == 0 { + return parsed.ipVersion == 4 + } + transport = transport[:length] + case uint8(header.ICMPv4ProtocolNumber): + return len(transport) >= header.ICMPv4MinimumSize && checksum.Checksum(transport, 0) == 0xffff + case uint8(header.ICMPv6ProtocolNumber): + if len(transport) < header.ICMPv6MinimumSize { + return false + } + default: + return true + } + partial := header.PseudoHeaderChecksum(tcpip.TransportProtocolNumber(parsed.protocol), parsed.source.Addr().AsSlice(), parsed.destination.Addr().AsSlice(), uint16(len(transport))) + return checksum.Checksum(transport, partial) == 0xffff +} + func (e *goEngine) demuxL4(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { switch parsed.protocol { case uint8(header.TCPProtocolNumber): @@ -651,6 +764,10 @@ func (e *goEngine) demuxL4(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *fo case uint8(header.UDPProtocolNumber): e.demuxUDP(buffer, meta, parsed) case uint8(header.ICMPv4ProtocolNumber), uint8(header.ICMPv6ProtocolNumber): + if parsed.isICMPError() { + e.inputICMPError(parsed) + return + } e.answerEcho(buffer.Bytes(), parsed) } } @@ -694,16 +811,8 @@ func (e *goEngine) handleLoopbackHairpin(packet []byte, meta ForwardFrameMeta, p return true } -func (e *goEngine) now() int64 { - return e.coarseTime.Load() -} - -func (e *goEngine) readClock() int64 { - return int64(time.Since(e.epoch)) -} - func (e *goEngine) refreshCoarseTime() int64 { - now := e.readClock() + now := int64(time.Since(e.epoch)) e.coarseTime.Store(now) return now } @@ -737,14 +846,6 @@ type goReassemblyEntry struct { rangeCount int } -func (e *goEngine) reassemble(packet []byte, parsed *forwardPacket) { - if parsed.ipVersion == 4 { - e.reassembleIPv4(packet, parsed) - } else { - e.reassembleIPv6(packet, parsed) - } -} - func (e *goEngine) reassembleIPv4(packet []byte, parsed *forwardPacket) { ipHdr := header.IPv4(parsed.network) headerLength := int(ipHdr.HeaderLength()) @@ -780,7 +881,7 @@ func (e *goEngine) reassembleIPv6(packet []byte, parsed *forwardPacket) { } func (e *goEngine) reassemblyEntry(ipVersion uint8, protocol uint8, source netip.Addr, destination netip.Addr, ident uint32) *goReassemblyEntry { - now := e.now() + now := e.coarseTime.Load() var free *goReassemblyEntry var oldest *goReassemblyEntry for index := range e.reassemblyEntries { @@ -959,7 +1060,7 @@ func (e *goEngine) answerEcho(packet []byte, parsed *forwardPacket) { default: return } - err := e.platformIO.writeFrame(e.singleFrame(packet), ForwardFrameMeta{}) + err := e.platformIO.writeDatagram(packet, ForwardFrameMeta{}) if err != nil { e.stack.logger.Trace(E.Cause(err, "go: write echo reply")) } diff --git a/stack_go_integration_cc_test.go b/stack_go_integration_cc_test.go index bd414138..d5ecbe58 100644 --- a/stack_go_integration_cc_test.go +++ b/stack_go_integration_cc_test.go @@ -143,7 +143,7 @@ func TestGoKernelTCPSendBudget(t *testing.T) { test.Cleanup(resumeOnce) test.Cleanup(releaseOnce) client, server := fixture.pair(test, false) - mss := int(server.effectiveMSS) + mss := int(server.effectiveMSS.Load()) err := kernelTCPWrite(server, kernelPayload(mss, 19), buffered) if err != nil { test.Fatal(err) @@ -205,7 +205,7 @@ func TestGoKernelTCPWaiting(t *testing.T) { } fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{receiveDelay: 100 * time.Millisecond, checkpoint: checkpoint}) client, server := fixture.pair(test, false) - size := goInitialWindow * int(server.effectiveMSS) + size := goInitialWindow * int(server.effectiveMSS.Load()) warmup := kernelPayload(size, 31) payload := kernelPayload(2*size, 37) readResult := make(chan error, 1) @@ -248,7 +248,7 @@ func TestGoKernelTCPWaiting(t *testing.T) { delivered += event.end - event.sequence rate = max(rate, event.rate) } - allowed := rate*uint64(last-first+2*time.Millisecond)/uint64(time.Second) + 2*uint64(server.effectiveMSS) + allowed := rate*uint64(last-first+2*time.Millisecond)/uint64(time.Second) + 2*uint64(server.effectiveMSS.Load()) if delivered == 0 || delivered > allowed { test.Errorf("pacing envelope: sent=%d bytes over %s at up to %d bytes/s, allowed=%d; recent TCP events: %+v", delivered, last-first, rate, allowed, flow.events) } @@ -329,7 +329,7 @@ func TestGoKernelTCPRecovery(t *testing.T) { configTest.Run(fmt.Sprintf("ipv6=%v/%s/%s", ipv6, writer, pattern), func(test *testing.T) { test.Parallel() client, sender, conn := traffic.pair(test, fixture, ipv6, writer) - mss := int(conn.effectiveMSS) + mss := int(conn.effectiveMSS.Load()) start := uint64(1) for round := range 4 { segments := 8 @@ -405,7 +405,7 @@ func TestGoKernelTCPTransmitLifetime(t *testing.T) { fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{}) client, sender, conn := traffic.pair(test, fixture, ipv6, writer) barrier := newKernelTCPBarrier(test) - mss := int(conn.effectiveMSS) + mss := int(conn.effectiveMSS.Load()) prefix := mss length := 1 switch shape { diff --git a/stack_go_integration_flow_linux_test.go b/stack_go_integration_flow_linux_test.go index 3fb71899..44b36a36 100644 --- a/stack_go_integration_flow_linux_test.go +++ b/stack_go_integration_flow_linux_test.go @@ -159,7 +159,7 @@ func TestGoKernelSACKReneging(t *testing.T) { traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { return kernelTCPAction{drop: event.outgoing && event.sequence == 1 && event.end > event.sequence} }) - mss := int(server.effectiveMSS) + mss := int(server.effectiveMSS.Load()) payload := kernelPayload(8*mss, 137) _, err := server.Write(payload) if err != nil { @@ -296,19 +296,19 @@ func TestGoKernelWindowLimitedRecovery(t *testing.T) { test.Fatalf("delay ACKs: %s: %v", output, err) } payload := kernelPayload(8<<20, 223) - initial := 4 * int(server.effectiveMSS) + initial := 4 * int(server.effectiveMSS.Load()) _, err = server.Write(payload[:initial]) if err != nil { test.Fatal(err) } deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { - if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() > server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() > server.sendUnacked.Load()+2*uint64(server.effectiveMSS.Load()) { break } time.Sleep(time.Millisecond) } - if server.sendPermit.Load()&^goPermitWindowBit != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS) { + if server.sendPermit.Load()&^goPermitWindowBit != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS.Load()) { test.Fatal("window did not shrink below outstanding data") } completed := make(chan kernelIOResult, 1) @@ -388,14 +388,10 @@ func TestGoKernelZeroWindow(t *testing.T) { if err != nil { test.Fatal(err) } - runTC := func(args ...string) { - test.Helper() - output, commandErr := kernelCommand("tc", args...) - if commandErr != nil { - test.Fatalf("tc %v: %s: %v", args, output, commandErr) - } + output, err := kernelCommand("tc", "qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") + if err != nil { + test.Fatalf("tc qdisc replace: %s: %v", output, err) } - runTC("qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") payload := kernelPayload(32768, 107) _, err = server.Write(payload) if err != nil { @@ -418,7 +414,10 @@ func TestGoKernelZeroWindow(t *testing.T) { traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { return kernelTCPAction{drop: windowUpdate(event)} }) - runTC("qdisc", "del", "dev", fixture.options.Name, "root") + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, "root") + if err != nil { + test.Fatalf("tc qdisc del: %s: %v", output, err) + } err = client.SetReadBuffer(4 << 20) if err != nil { test.Fatal(err) diff --git a/stack_go_integration_listen_test.go b/stack_go_integration_listen_test.go new file mode 100644 index 00000000..9e0ffa13 --- /dev/null +++ b/stack_go_integration_listen_test.go @@ -0,0 +1,182 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/netip" + "syscall" + "testing" + "time" +) + +func TestGoListenTCP(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, noHandler: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("duplex", func(test *testing.T) { + listener, err := fixture.stack.ListenTCP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer listener.Close() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + dialer := net.Dialer{Timeout: 3 * time.Second} + kernel, err := dialer.Dial("tcp", listener.Addr().String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(10 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + if conn.LocalAddr().String() != listener.Addr().String() { + test.Fatalf("accepted local address %s, listener %s", conn.LocalAddr(), listener.Addr()) + } + if conn.RemoteAddr().String() != kernel.LocalAddr().String() { + test.Fatalf("accepted remote address %s, kernel %s", conn.RemoteAddr(), kernel.LocalAddr()) + } + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, kernel, kernelPayload(4<<20, 31), false) }() + err = kernelTransfer(kernel, conn, kernelPayload(4<<20, 47), false) + if err != nil { + test.Error("upload:", err) + } + err = <-result + if err != nil { + test.Error("download:", err) + } + }) + addressTest.Run("close", func(test *testing.T) { + listener, err := fixture.stack.ListenTCP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + dialer := net.Dialer{Timeout: 3 * time.Second} + kernel, err := dialer.Dial("tcp", listener.Addr().String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + listener.Close() + _, err = kernel.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + test.Fatalf("kernel read on a connection queued at a closed listener: %v", err) + } + _, err = listener.Accept() + if !errors.Is(err, net.ErrClosed) { + test.Fatalf("accept on a closed listener: %v", err) + } + _, err = dialer.Dial("tcp", listener.Addr().String()) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("connect to a closed listener: %v", err) + } + }) + }) + } +} + +func TestGoListenPrecedence(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + port := uint16(30000 + fixture.port.Add(1)) + local := netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), port) + listener, err := fixture.stack.ListenTCP(local) + if err != nil { + test.Fatal(err) + } + defer listener.Close() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = fixture.stack.ListenTCP(local) + if !errors.Is(err, syscall.EADDRINUSE) { + test.Fatalf("second listener on %s: %v", local, err) + } + kernel, err := fixture.dialer.Dial("tcp", local.String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + test.Fatal(err) + } + defer conn.Close() + _, err = kernel.Write([]byte("listener")) + if err != nil { + test.Fatal(err) + } + received := make([]byte, 8) + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, err = conn.Read(received) + if err != nil { + test.Fatal(err) + } + if string(received) != "listener" { + test.Fatalf("listener received %q", received) + } + }) + } +} + +func TestGoListenUDPPort(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + port := uint16(30000 + fixture.port.Add(1)) + local := netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), port) + socket, err := fixture.stack.ListenUDP(local) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + if socket.LocalAddr().String() != local.String() { + test.Fatalf("bound to %s, want %s", socket.LocalAddr(), local) + } + _, err = fixture.stack.ListenUDP(local) + if !errors.Is(err, syscall.EADDRINUSE) { + test.Fatalf("second socket on %s: %v", local, err) + } + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + payload := kernelPayload(1200, 21) + _, err = peer.WriteToUDPAddrPort(payload, local) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 2000) + n, source, err := socket.ReadFromUDPAddrPort(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("received %d bytes with corrupted payload", n) + } + _, err = socket.WriteToUDPAddrPort(storage[:n], source) + if err != nil { + test.Fatal(err) + } + n, _, err = peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("reply carried %d bytes with corrupted payload", n) + } + }) + } +} diff --git a/stack_go_integration_memory_darwin_test.go b/stack_go_integration_memory_darwin_test.go new file mode 100644 index 00000000..0d613123 --- /dev/null +++ b/stack_go_integration_memory_darwin_test.go @@ -0,0 +1,15 @@ +//go:build darwin && !ios + +package tun + +import ( + "net/netip" + "testing" +) + +func kernelAddUnreachableRoute(t *testing.T, prefix netip.Prefix) bool { + return false +} + +// net.inet.udp.maxdgram caps a datagram at 9216 bytes. +const kernelMaxDatagram = 9216 diff --git a/stack_go_integration_memory_linux_test.go b/stack_go_integration_memory_linux_test.go new file mode 100644 index 00000000..9c3eb2e0 --- /dev/null +++ b/stack_go_integration_memory_linux_test.go @@ -0,0 +1,29 @@ +//go:build linux && !android + +package tun + +import ( + "net" + "net/netip" + "testing" + + "github.com/sagernet/netlink" + + "golang.org/x/sys/unix" +) + +const kernelMaxDatagram = 60000 + +func kernelAddUnreachableRoute(t *testing.T, prefix netip.Prefix) bool { + t.Helper() + route := &netlink.Route{ + Dst: &net.IPNet{IP: prefix.Addr().AsSlice(), Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen())}, + Type: unix.RTN_UNREACHABLE, + } + err := netlink.RouteAdd(route) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { netlink.RouteDel(route) }) + return true +} diff --git a/stack_go_integration_memory_test.go b/stack_go_integration_memory_test.go new file mode 100644 index 00000000..0bce082c --- /dev/null +++ b/stack_go_integration_memory_test.go @@ -0,0 +1,565 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func kernelListen(t *testing.T, fixture *kernelStackFixture, ipv6 bool) (*net.TCPListener, netip.AddrPort) { + t.Helper() + listener, err := net.ListenTCP("tcp", net.TCPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + return listener, listener.Addr().(*net.TCPAddr).AddrPort() +} + +func kernelAcceptTCP(t *testing.T, listener *net.TCPListener) *net.TCPConn { + t.Helper() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn +} + +func TestGoMemoryDial(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("duplex", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + if conn.RemoteAddr().String() != address.String() { + test.Fatalf("remote address %s, want %s", conn.RemoteAddr(), address) + } + if conn.LocalAddr().String() != server.RemoteAddr().String() { + test.Fatalf("local address %s, kernel peer %s", conn.LocalAddr(), server.RemoteAddr()) + } + payload := kernelPayload(4<<20, 31) + response := kernelPayload(4<<20, 47) + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, server, payload, false) }() + err = kernelTransfer(server, conn, response, false) + if err != nil { + test.Error("download:", err) + } + err = <-result + if err != nil { + test.Error("upload:", err) + } + }) + addressTest.Run("close", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + server := kernelAcceptTCP(test, listener) + _, err = conn.Write([]byte("hello")) + if err != nil { + test.Fatal(err) + } + err = conn.Close() + if err != nil { + test.Fatal(err) + } + data, err := io.ReadAll(server) + if err != nil { + test.Fatal(err) + } + if string(data) != "hello" { + test.Fatalf("kernel received %q", data) + } + }) + addressTest.Run("refused", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + listener.Close() + started := time.Now() + _, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("dial to a closed port returned %v", err) + } + if time.Since(started) > time.Second { + test.Fatalf("refusal took %v", time.Since(started)) + } + }) + addressTest.Run("unreachable", func(test *testing.T) { + prefix := netip.MustParsePrefix("198.19.255.0/24") + destination := netip.MustParseAddrPort("198.19.255.254:9") + if ipv6 { + prefix = netip.MustParsePrefix("fd73:ab91:ffff::/48") + destination = netip.MustParseAddrPort("[fd73:ab91:ffff::254]:9") + } + if !kernelAddUnreachableRoute(test, prefix) { + test.Skip("unreachable routes are not configurable on this platform") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), destination) + if !E.IsMulti(err, syscall.EHOSTUNREACH, syscall.ENETUNREACH) { + test.Fatalf("dial to an unroutable address returned %v", err) + } + }) + addressTest.Run("cancel", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + fixture.bridge.pauseOutbound(true) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + fixture.bridge.pauseOutbound(false) + if !errors.Is(err, context.DeadlineExceeded) { + test.Fatalf("cancelled dial returned %v", err) + } + listener.SetDeadline(time.Now().Add(500 * time.Millisecond)) + conn, acceptErr := listener.Accept() + if acceptErr == nil { + conn.SetReadDeadline(time.Now().Add(time.Second)) + _, readErr := conn.Read(make([]byte, 1)) + conn.Close() + if !errors.Is(readErr, syscall.ECONNRESET) { + test.Fatalf("stale handshake was not reset: %v", readErr) + } + } + }) + addressTest.Run("retransmit", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + fixture.bridge.outboundDrop.Store(1) + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + if time.Since(started) < goSynRetransmit { + test.Fatalf("handshake completed in %v with the first SYN dropped", time.Since(started)) + } + server := kernelAcceptTCP(test, listener) + err = kernelTransfer(conn, server, kernelPayload(64<<10, 5), false) + if err != nil { + test.Fatal(err) + } + }) + addressTest.Run("stall", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + payload := kernelPayload(2<<20, 61) + fixture.bridge.pauseOutbound(true) + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, server, payload, false) }() + select { + case err = <-result: + test.Fatalf("transfer finished through a paused link: %v", err) + case <-time.After(300 * time.Millisecond): + } + fixture.bridge.pauseOutbound(false) + err = <-result + if err != nil { + test.Fatal(err) + } + }) + }) + } +} + +func TestGoMemoryUDP(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("listen", func(test *testing.T) { + socket, err := fixture.stack.ListenUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + storage := make([]byte, 65535) + for index, size := range []int{1, 1200, 9000, kernelMaxDatagram} { + payload := kernelPayload(size, uint32(index+1)) + _, err = peer.WriteToUDPAddrPort(payload, socket.LocalAddr().(*net.UDPAddr).AddrPort()) + if err != nil { + test.Fatal(err) + } + n, source, readErr := socket.ReadFrom(storage) + if readErr != nil { + test.Fatalf("size=%d: %v", size, readErr) + } + if source.String() != peer.LocalAddr().String() { + test.Fatalf("size=%d: source %s, want %s", size, source, peer.LocalAddr()) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("size=%d: received %d bytes with corrupted payload", size, n) + } + _, err = socket.WriteTo(payload, source) + if err != nil { + test.Fatal(err) + } + n, _, readErr = peer.ReadFrom(storage) + if readErr != nil { + test.Fatalf("size=%d: reply: %v", size, readErr) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("size=%d: reply carried %d bytes with corrupted payload", size, n) + } + } + }) + addressTest.Run("connected", func(test *testing.T) { + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + stranger, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer stranger.Close() + socket, err := fixture.stack.DialUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0), peer.LocalAddr().(*net.UDPAddr).AddrPort()) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = socket.Write([]byte("ping")) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 64) + n, source, err := peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if string(storage[:n]) != "ping" || source.String() != socket.LocalAddr().String() { + test.Fatalf("kernel received %q from %s", storage[:n], source) + } + _, err = stranger.WriteTo([]byte("stranger"), source) + if err != nil { + test.Fatal(err) + } + _, err = peer.WriteTo([]byte("pong"), source) + if err != nil { + test.Fatal(err) + } + n, err = socket.Read(storage) + if err != nil { + test.Fatal(err) + } + if string(storage[:n]) != "pong" { + test.Fatalf("connected socket received %q", storage[:n]) + } + socket.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + _, err = socket.Read(storage) + if !errors.Is(err, os.ErrDeadlineExceeded) { + test.Fatalf("datagram from another peer leaked into a connected socket: %v", err) + } + }) + addressTest.Run("refused", func(test *testing.T) { + closedPeer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + address := closedPeer.LocalAddr().(*net.UDPAddr).AddrPort() + closedPeer.Close() + socket, err := fixture.stack.DialUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0), address) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = socket.Write([]byte("anyone")) + if err != nil { + test.Fatal(err) + } + _, err = socket.Read(make([]byte, 64)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("datagram to a closed port returned %v", err) + } + }) + }) + } +} + +func TestGoMemoryMTUUpdate(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, server := fixture.pair(test, ipv6) + payload := kernelPayload(4<<20, 79) + result := make(chan error, 1) + go func() { result <- kernelTransfer(server, client, payload, false) }() + time.Sleep(20 * time.Millisecond) + fixture.memoryTun.UpdateMTU(576) + err := <-result + if err != nil { + test.Fatal(err) + } + fixture.bridge.outboundLargest.Store(0) + client, server = fixture.pair(test, ipv6) + err = kernelTransfer(server, client, kernelPayload(1<<20, 83), false) + if err != nil { + test.Fatal(err) + } + largest := fixture.bridge.outboundLargest.Load() + if largest > 576 { + test.Fatalf("stack emitted a %d byte packet after the MTU dropped to 576", largest) + } + fixture.memoryTun.UpdateMTU(1500) + }) + } +} + +func TestGoMemoryNoHandler(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, noHandler: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("tcp", func(test *testing.T) { + dialer := net.Dialer{Timeout: 2 * time.Second} + _, err := dialer.Dial("tcp", fixture.address(ipv6, 8080)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("connect to a handler-less stack returned %v", err) + } + }) + addressTest.Run("udp", func(test *testing.T) { + peer, err := net.Dial("udp", fixture.address(ipv6, 8080)) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(2 * time.Second)) + _, err = peer.Write([]byte("anyone")) + if err != nil { + test.Fatal(err) + } + _, err = peer.Read(make([]byte, 64)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("datagram to a handler-less stack returned %v", err) + } + }) + addressTest.Run("dial", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + err = kernelTransfer(server, conn, kernelPayload(256<<10, 11), false) + if err != nil { + test.Fatal(err) + } + }) + }) + } +} + +func TestGoMemoryOutboundReply(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, conn, destination := fixture.packetPair(test, ipv6) + time.Sleep(100 * time.Millisecond) + payload := kernelPayload(64, 97) + options := N.NewReadWaitOptions(nil, conn) + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + started := time.Now() + err := conn.WritePacket(packet, destination) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(time.Second)) + received := make([]byte, 1500) + n, err := client.Read(received) + elapsed := time.Since(started) + if err != nil || !bytes.Equal(received[:n], payload) { + test.Fatalf("reply: %x, %v", received[:n], err) + } + if elapsed > 100*time.Millisecond { + test.Fatalf("reply reached the kernel after %v", elapsed) + } + }) + } +} + +func TestGoMemoryOutboundBound(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + client, conn, destination := fixture.packetPair(t, false) + err := client.SetReadBuffer(4 << 20) + if err != nil { + t.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + options := N.NewReadWaitOptions(nil, conn) + payload := kernelPayload(100, 113) + sendReply := func() error { + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + return conn.WritePacket(packet, destination) + } + capacity := len(fixture.memoryTun.datagram.packets) + for range 4 * capacity { + err = sendReply() + if err != nil { + t.Fatal(err) + } + } + fixture.bridge.pauseOutbound(false) + received := make([]byte, 1500) + delivered := 0 + for { + client.SetReadDeadline(time.Now().Add(time.Second)) + _, err = client.Read(received) + if err != nil { + break + } + delivered++ + } + if delivered < capacity || delivered > capacity+fixture.memoryTun.batchSize { + t.Fatalf("%d replies delivered after the outbound handler resumed, want between %d and %d", delivered, capacity, capacity+fixture.memoryTun.batchSize) + } + marker := kernelPayload(7, 131) + packet := options.NewBufferSize(len(marker)) + packet.Write(marker) + options.PostReturn(packet) + err = conn.WritePacket(packet, destination) + if err != nil { + t.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(time.Second)) + n, err := client.Read(received) + if err != nil || !bytes.Equal(received[:n], marker) { + t.Fatalf("reply after backlog: %x, %v", received[:n], err) + } +} + +func TestGoMemoryControlPriority(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + _, conn, destination := fixture.packetPair(t, false) + listener, address := kernelListen(t, fixture, false) + fixture.bridge.pauseOutbound(true) + options := N.NewReadWaitOptions(nil, conn) + payload := kernelPayload(100, 113) + for range 4 * len(fixture.memoryTun.datagram.packets) { + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + err := conn.WritePacket(packet, destination) + if err != nil { + t.Fatal(err) + } + } + dialed := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + dialConn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(false).Next(), address) + if err == nil { + dialConn.Close() + } + dialed <- err + }() + time.Sleep(100 * time.Millisecond) + resumed := time.Now() + fixture.bridge.pauseOutbound(false) + server := kernelAcceptTCP(t, listener) + server.Close() + err := <-dialed + if err != nil { + t.Fatal(err) + } + if elapsed := time.Since(resumed); elapsed > 500*time.Millisecond { + t.Fatalf("handshake completed %v after the link resumed, the SYN waited behind the datagram backlog", elapsed) + } +} + +func TestGoMemoryPathMTU(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1408, memoryLink: true}) + var tooBig atomic.Int32 + fixture.bridge.filterOutbound(func(packet []byte) bool { + if header.IPVersion(packet) != header.IPv6Version || len(packet) <= header.IPv6MinimumMTU { + return true + } + reply, built := buildPacketTooBig(header.IPv6(packet), header.IPv6MinimumMTU, 0) + if built { + tooBig.Add(1) + fixture.memoryTun.WritePackets([][]byte{reply}) + } + return false + }) + t.Cleanup(func() { fixture.bridge.filterOutbound(nil) }) + for _, dialed := range []bool{true, false} { + t.Run(fmt.Sprintf("dialed=%v", dialed), func(test *testing.T) { + tooBig.Store(0) + conn, kernel, _ := openConnection(test, fixture, true, dialed) + conn.SetDeadline(time.Now().Add(3 * time.Second)) + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + payload := kernelPayload(32<<10, 151) + written := make(chan error, 1) + go func() { + _, writeErr := conn.Write(payload) + written <- writeErr + }() + received := make([]byte, len(payload)) + _, err := io.ReadFull(kernel, received) + if err != nil { + test.Fatalf("transfer over a %d byte path stalled after %d Packet Too Big messages: %v", header.IPv6MinimumMTU, tooBig.Load(), err) + } + err = <-written + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(received, payload) { + test.Fatal("payload corrupted") + } + if tooBig.Load() == 0 { + test.Fatal("no segment exceeded the path MTU") + } + }) + } +} diff --git a/stack_go_integration_options_test.go b/stack_go_integration_options_test.go new file mode 100644 index 00000000..2126f8a5 --- /dev/null +++ b/stack_go_integration_options_test.go @@ -0,0 +1,458 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "sync" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" +) + +func dialKernel(t *testing.T, fixture *kernelStackFixture, ipv6 bool) (*GoConn, *net.TCPConn) { + t.Helper() + listener, address := kernelListen(t, fixture, ipv6) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, kernelAcceptTCP(t, listener) +} + +func openConnection(t *testing.T, fixture *kernelStackFixture, ipv6 bool, dialed bool) (*GoConn, *net.TCPConn, uint16) { + t.Helper() + if dialed { + conn, kernel := dialKernel(t, fixture, ipv6) + return conn, kernel, conn.LocalAddr().(*net.TCPAddr).AddrPort().Port() + } + kernel, conn := fixture.pair(t, ipv6) + return conn, kernel, kernel.RemoteAddr().(*net.TCPAddr).AddrPort().Port() +} + +func connectionCases(t *testing.T, fixture *kernelStackFixture, run func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16)) { + for _, ipv6 := range []bool{false, true} { + for _, dialed := range []bool{true, false} { + t.Run(fmt.Sprintf("ipv6=%v,dialed=%v", ipv6, dialed), func(test *testing.T) { + conn, kernel, port := openConnection(test, fixture, ipv6, dialed) + run(test, conn, kernel, port) + }) + } + } +} + +type wireSegment struct { + outbound bool + flags header.TCPFlags + sequence uint32 + ack uint32 + window uint32 + payload int +} + +func (s wireSegment) probe() bool { + return s.outbound && s.payload == 0 && s.flags == header.TCPFlagAck +} + +type wireRecorder struct { + access sync.Mutex + port uint16 + segments []wireSegment +} + +func recordWire(t *testing.T, fixture *kernelStackFixture, port uint16) *wireRecorder { + recorder := &wireRecorder{port: port} + fixture.bridge.observe( + func(packet []byte) { recorder.record(packet, false) }, + func(packet []byte) { recorder.record(packet, true) }, + ) + t.Cleanup(func() { fixture.bridge.observe(nil, nil) }) + return recorder +} + +func (r *wireRecorder) record(packet []byte, outbound bool) { + var transport []byte + switch header.IPVersion(packet) { + case 4: + ipHdr := header.IPv4(packet) + if ipHdr.Protocol() != uint8(header.TCPProtocolNumber) { + return + } + transport = ipHdr.Payload() + case 6: + ipHdr := header.IPv6(packet) + if ipHdr.NextHeader() != uint8(header.TCPProtocolNumber) { + return + } + transport = ipHdr.Payload() + default: + return + } + tcpHdr := header.TCP(transport) + if outbound && tcpHdr.SourcePort() != r.port || !outbound && tcpHdr.DestinationPort() != r.port { + return + } + segment := wireSegment{ + outbound: outbound, + flags: tcpHdr.Flags(), + sequence: tcpHdr.SequenceNumber(), + ack: tcpHdr.AckNumber(), + payload: len(transport) - int(tcpHdr.DataOffset()), + } + if outbound { + segment.window = uint32(tcpHdr.WindowSize()) << goLocalWindowShift + } + r.access.Lock() + r.segments = append(r.segments, segment) + r.access.Unlock() +} + +func (r *wireRecorder) take() []wireSegment { + r.access.Lock() + defer r.access.Unlock() + segments := r.segments + r.segments = nil + return segments +} + +func TestGoConnKeepalive(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("vanished", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetKeepAliveConfig(net.KeepAliveConfig{Enable: true, Idle: 500 * time.Millisecond, Interval: 500 * time.Millisecond, Count: 2}) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + fixture.bridge.pauseInbound(true) + defer fixture.bridge.pauseInbound(false) + start := time.Now() + _, err = conn.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ETIMEDOUT) { + test.Fatalf("read after unanswered probes: %v", err) + } + if elapsed := time.Since(start); elapsed > 4*time.Second { + test.Fatalf("keepalive gave up after %v", elapsed) + } + probes := 0 + reset := false + for deadline := time.Now().Add(time.Second); !reset && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + for _, segment := range recorder.take() { + switch { + case segment.probe(): + probes++ + case segment.outbound && segment.flags&header.TCPFlagRst != 0: + reset = true + } + } + } + if probes != 2 || !reset { + test.Fatalf("sent %d probes and reset=%v before giving up on a peer that stopped answering", probes, reset) + } + }) + }) + t.Run("answered", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetKeepAliveConfig(net.KeepAliveConfig{Enable: true, Idle: 100 * time.Millisecond, Interval: 100 * time.Millisecond, Count: 2}) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + time.Sleep(2 * time.Second) + probes := 0 + answered := 0 + var awaiting []uint32 + for _, segment := range recorder.take() { + switch { + case segment.probe(): + probes++ + awaiting = append(awaiting, segment.sequence+1) + case !segment.outbound && segment.payload == 0 && len(awaiting) > 0 && segment.ack == awaiting[0]: + answered++ + awaiting = awaiting[1:] + } + } + if probes < 2 || answered != probes { + test.Fatalf("kernel answered %d of %d probes sent while idle for 2s", answered, probes) + } + err = kernelTransfer(conn, kernel, kernelPayload(1<<20, 5), false) + if err != nil { + test.Fatal("after probes:", err) + } + }) + }) +} + +func TestGoConnNoDelay(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, noDelay := range []bool{true, false} { + t.Run(fmt.Sprintf("nodelay=%v", noDelay), func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetNoDelay(noDelay) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + fixture.bridge.pauseInbound(true) + defer fixture.bridge.pauseInbound(false) + for _, letter := range []byte("abcdef") { + _, err = conn.Write([]byte{letter}) + if err != nil { + test.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + dataSegments := make(map[uint32]bool) + for _, segment := range recorder.take() { + if segment.outbound && segment.payload > 0 { + dataSegments[segment.sequence] = true + } + } + if noDelay && len(dataSegments) != 6 { + test.Fatalf("sent %d data segments for 6 writes without delay", len(dataSegments)) + } + if !noDelay && len(dataSegments) != 1 { + test.Fatalf("sent %d data segments while the first byte was unacknowledged", len(dataSegments)) + } + fixture.bridge.pauseInbound(false) + received := make([]byte, 6) + _, err = io.ReadFull(kernel, received) + if err != nil { + test.Fatal(err) + } + if string(received) != "abcdef" { + test.Fatalf("kernel received %q", received) + } + if !noDelay { + coalesced := false + for _, segment := range recorder.take() { + if segment.outbound && segment.payload == 5 { + coalesced = true + } + } + if !coalesced { + test.Fatal("held bytes were not sent as one segment once the acknowledgement arrived") + } + } + }) + }) + } +} + +func TestGoConnLinger(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("abort", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + _, err := conn.Write([]byte("hello")) + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(kernel, make([]byte, 5)) + if err != nil { + test.Fatal(err) + } + err = conn.SetLinger(0) + if err != nil { + test.Fatal(err) + } + conn.Close() + _, err = kernel.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + test.Fatalf("kernel read after linger 0 close: %v", err) + } + }) + }) + t.Run("prompt", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetLinger(3) + if err != nil { + test.Fatal(err) + } + _, err = conn.Write([]byte("x")) + if err != nil { + test.Fatal(err) + } + start := time.Now() + conn.Close() + if elapsed := time.Since(start); elapsed > time.Second { + test.Fatalf("close waited %v although the peer acknowledged the FIN", elapsed) + } + received, err := io.ReadAll(kernel) + if err != nil { + test.Fatal(err) + } + if string(received) != "x" { + test.Fatalf("kernel received %q", received) + } + }) + }) + t.Run("expire", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetLinger(1) + if err != nil { + test.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + defer fixture.bridge.pauseOutbound(false) + _, err = conn.Write([]byte("x")) + if err != nil { + test.Fatal(err) + } + start := time.Now() + conn.Close() + elapsed := time.Since(start) + if elapsed < time.Second || elapsed > 3*time.Second { + test.Fatalf("close waited %v with the FIN undeliverable and linger 1s", elapsed) + } + fixture.bridge.pauseOutbound(false) + received, err := io.ReadAll(kernel) + if err != nil { + test.Fatal(err) + } + if string(received) != "x" { + test.Fatalf("kernel received %q", received) + } + }) + }) +} + +func TestGoConnBuffers(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("read", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetReadBuffer(goSlabSize) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + payload := kernelPayload(4<<20, 11) + err = kernelTransfer(kernel, conn, payload, false) + if err != nil { + test.Fatal(err) + } + var first *wireSegment + var largest uint32 + for _, segment := range recorder.take() { + if !segment.outbound { + continue + } + if first == nil { + first = &segment + } + if segment.ack-first.ack < uint32(len(payload)/2) { + continue + } + largest = max(largest, segment.window) + } + if largest == 0 || largest > goSlabSize { + test.Fatalf("advertised a %d byte window in the second half of the transfer with a %d byte read buffer", largest, goSlabSize) + } + }) + }) + t.Run("write", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetWriteBuffer(goSlabSize) + if err != nil { + test.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + defer fixture.bridge.pauseOutbound(false) + conn.SetWriteDeadline(time.Now().Add(300 * time.Millisecond)) + n, err := conn.Write(kernelPayload(4<<20, 13)) + if !errors.Is(err, os.ErrDeadlineExceeded) { + test.Fatalf("write against a stalled link: %v", err) + } + if n != goSlabSize { + test.Fatalf("buffered %d bytes with a %d byte write buffer", n, goSlabSize) + } + }) + }) +} + +func TestGoUDPReadBuffer(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + socket, err := fixture.stack.ListenUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + err = socket.SetReadBuffer(8192) + if err != nil { + test.Fatal(err) + } + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + target := socket.LocalAddr().(*net.UDPAddr).AddrPort() + for index := range 64 { + _, err = peer.WriteToUDPAddrPort(kernelPayload(1000, uint32(index+1)), target) + if err != nil { + test.Fatal(err) + } + } + time.Sleep(200 * time.Millisecond) + storage := make([]byte, 2000) + received := 0 + for { + socket.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + n, _, readErr := socket.ReadFromUDPAddrPort(storage) + if readErr != nil { + if !errors.Is(readErr, os.ErrDeadlineExceeded) { + test.Fatal(readErr) + } + break + } + if !bytes.Equal(storage[:n], kernelPayload(1000, uint32(received+1))) { + test.Fatalf("datagram %d out of order or corrupted", received) + } + received++ + } + if received == 0 || received*1000 > 8192 { + test.Fatalf("queued %d datagrams of 1000 bytes with an 8192 byte read buffer", received) + } + payload := kernelPayload(1000, 99) + _, err = peer.WriteToUDPAddrPort(payload, target) + if err != nil { + test.Fatal(err) + } + socket.SetReadDeadline(time.Now().Add(3 * time.Second)) + n, source, err := socket.ReadFromUDPAddrPort(storage) + if err != nil { + test.Fatal("after draining the queue:", err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("datagram after draining carried %d bytes with corrupted payload", n) + } + _, err = socket.WriteToUDPAddrPort(storage[:n], source) + if err != nil { + test.Fatal(err) + } + n, _, err = peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("reply carried %d bytes with corrupted payload", n) + } + }) + } +} diff --git a/stack_go_integration_packet_flow_test.go b/stack_go_integration_packet_flow_test.go index cbae7aaa..8a9bd492 100644 --- a/stack_go_integration_packet_flow_test.go +++ b/stack_go_integration_packet_flow_test.go @@ -26,8 +26,11 @@ func TestGoKernelPacketFragments(t *testing.T) { if runtime.GOOS == "linux" { configs = append(configs, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}) + } for _, config := range configs { - t.Run(fmt.Sprintf("gso=%v/mq=%v", config.gso, config.multiQueue), func(configTest *testing.T) { + t.Run(fmt.Sprintf("gso=%v/mq=%v/memory=%v", config.gso, config.multiQueue, config.memoryLink), func(configTest *testing.T) { fixture := newKernelStackFixture(configTest, config) for _, ipv6 := range []bool{false, true} { configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { @@ -204,10 +207,7 @@ func TestGoKernelPacketMapping(t *testing.T) { for _, ipv6 := range []bool{false, true} { configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { test.Parallel() - address := fixture.options.Inet4Address[0].Addr() - if ipv6 { - address = fixture.options.Inet6Address[0].Addr() - } + address := fixture.kernelAddress(ipv6) client, err := net.ListenUDP("udp", &net.UDPAddr{IP: address.AsSlice()}) if err != nil { test.Fatal(err) diff --git a/stack_go_integration_packet_test.go b/stack_go_integration_packet_test.go index af889661..a5146874 100644 --- a/stack_go_integration_packet_test.go +++ b/stack_go_integration_packet_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net" + "os" "runtime" "strings" "sync" @@ -25,6 +26,49 @@ import ( "github.com/sagernet/sing/protocol/socks" ) +func TestGoKernelPacketWriteAfterClose(t *testing.T) { + configs := []kernelStackConfig{{mtu: 1500}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}) + } + for _, config := range configs { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("gso=%v/memory=%v/ipv6=%v", config.gso, config.memoryLink, ipv6), func(t *testing.T) { + fixture := newKernelStackFixture(t, config) + client, conn, destination := fixture.packetPair(t, ipv6) + defer client.Close() + batchWriter, batchSupported := conn.CreatePacketBatchWriter() + err := fixture.stack.Close() + if err != nil { + t.Fatal(err) + } + options := N.NewReadWaitOptions(nil, conn) + newPacket := func() *buf.Buffer { + packet := options.NewBufferSize(64) + packet.Write([]byte("late UDP reply")) + options.PostReturn(packet) + return packet + } + err = conn.WritePacket(newPacket(), destination) + if !errors.Is(err, os.ErrClosed) { + t.Errorf("packet write after stack close: %v", err) + } + if batchSupported { + packets := []*buf.Buffer{newPacket(), newPacket()} + destinations := []M.Socksaddr{destination, destination} + err = batchWriter.WritePacketBatch(packets, destinations) + if !errors.Is(err, os.ErrClosed) { + t.Errorf("packet batch write after stack close: %v", err) + } + } + }) + } + } +} + func TestGoKernelPacketTimeout(t *testing.T) { fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) for _, ipv6 := range []bool{false, true} { @@ -483,12 +527,6 @@ func (s *kernelSocksEchoServer) NewPacketConnectionEx(ctx context.Context, conn }() } -var ( - _ SpliceSocket = (*kernelPacketSocket)(nil) - _ socks.HandlerEx = (*kernelSocksEchoServer)(nil) - _ socks.PacketListener = (*kernelSocksEchoServer)(nil) -) - type kernelPacketAllocator struct { buf.Allocator active atomic.Int64 diff --git a/stack_go_integration_tcp_test.go b/stack_go_integration_tcp_test.go index 3516a081..9159152e 100644 --- a/stack_go_integration_tcp_test.go +++ b/stack_go_integration_tcp_test.go @@ -368,7 +368,7 @@ func (p *kernelTCPIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (in continue } if action.delay > 0 { - p.delayLocked(flow, event, packet, frame.meta, action.delay) + p.delayLocked(flow, event, packet, frame.meta, nil, 0, action.delay) continue } for copyIndex := 1; copyIndex < action.copies; copyIndex++ { @@ -383,18 +383,18 @@ func (p *kernelTCPIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (in } func (p *kernelTCPIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { - return p.write(frame, meta, false) + return p.write(frame, meta, nil, 0) } func (p *kernelTCPIO) writePacket(packet []byte, meta ForwardFrameMeta) error { - return p.write([][]byte{packet}, meta, false) + return p.write([][]byte{packet}, meta, nil, 0) } -func (p *kernelTCPIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { - return p.write(frame, meta, true) +func (p *kernelTCPIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + return p.write(frame, meta, owner, segmentEnd) } -func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, data bool) error { +func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { packet := slices.Concat(frame...) p.harness.access.Lock() flow, event := p.harness.eventLocked(packet, true) @@ -421,11 +421,11 @@ func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, data bool) er event.dropped = true case action.delay > 0: event.delayed = true - p.delayLocked(flow, event, packet, meta, action.delay) + p.delayLocked(flow, event, packet, meta, owner, segmentEnd, action.delay) default: for range event.copies { - if data { - err = p.goPlatformIO.writeData(frame, meta) + if owner != nil { + err = p.goPlatformIO.writeData(frame, meta, owner, segmentEnd) } else { err = p.goPlatformIO.writeFrame(frame, meta) } @@ -440,14 +440,14 @@ func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, data bool) er } flow.writes-- } - if data { + if owner != nil { p.goPlatformIO.flush() } p.harness.access.Unlock() return err } -func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, packet []byte, meta ForwardFrameMeta, delay time.Duration) { +func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, packet []byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64, delay time.Duration) { if p.harness.stopped { return } @@ -470,7 +470,7 @@ func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, pac event.delivery = true for range event.copies { if event.outgoing { - err := p.goPlatformIO.writeData([][]byte{packet}, meta) + err := p.goPlatformIO.writeData([][]byte{packet}, meta, owner, segmentEnd) p.goPlatformIO.flush() if err != nil && flow.failure == "" { flow.failure = fmt.Sprintf("deliver delayed packet: %v", err) diff --git a/stack_go_integration_test.go b/stack_go_integration_test.go index a6e248a3..1d54efea 100644 --- a/stack_go_integration_test.go +++ b/stack_go_integration_test.go @@ -10,12 +10,14 @@ import ( "io" "net" "net/netip" + "os" "runtime" "sync" "sync/atomic" "testing" "time" + "github.com/sagernet/sing-tun/gtcpip/header" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/buf" "github.com/sagernet/sing/common/bufio" @@ -32,6 +34,9 @@ type kernelStackConfig struct { mtu uint32 gso bool multiQueue bool + memoryLink bool + memoryOutbound bool + noHandler bool prepare func(*testing.T, *Options) configure func(*testing.T, Options) ctx context.Context @@ -50,6 +55,8 @@ type kernelAccept struct { type kernelStackFixture struct { stack *Go + memoryTun *MemoryTun + bridge *memoryBridge options Options access sync.Mutex tcp map[uint16]chan kernelAccept @@ -113,9 +120,20 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF if config.ctx == nil { config.ctx = context.Background() } + var stackTun Tun + stackTun = device + if config.memoryLink { + fixture.bridge = newMemoryBridge(t, device, int(fixture.options.MTU), config.memoryOutbound) + fixture.memoryTun = fixture.bridge.memoryTun + stackTun = fixture.memoryTun + } + var handler Handler = fixture + if config.noHandler { + handler = nil + } stackOptions := StackOptions{ - Context: config.ctx, Tun: device, TunOptions: fixture.options, - Handler: fixture, Logger: logger.NOP(), UDPTimeout: time.Minute, ICMPTimeout: time.Minute, + Context: config.ctx, Tun: stackTun, TunOptions: fixture.options, + Handler: handler, Logger: logger.NOP(), UDPTimeout: time.Minute, ICMPTimeout: time.Minute, UDPMapping: config.udpMapping, UDPFiltering: config.udpFiltering, TCPCongestionControl: config.congestion, } @@ -147,6 +165,191 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF return fixture } +type memoryBridge struct { + device Tun + memoryTun *MemoryTun + inbound bridgeGate + outbound bridgeGate + outboundLargest atomic.Int64 + outboundDrop atomic.Int32 + observerAccess sync.Mutex + inboundObserver func(packet []byte) + outboundObserver func(packet []byte) + outboundFilter func(packet []byte) bool + done chan struct{} +} + +type bridgeGate struct { + access sync.Mutex + resume *sync.Cond + paused bool +} + +func (g *bridgeGate) pass() { + g.access.Lock() + for g.paused { + g.resume.Wait() + } + g.access.Unlock() +} + +func (g *bridgeGate) pause(paused bool) { + g.access.Lock() + g.paused = paused + g.access.Unlock() + g.resume.Broadcast() +} + +func newMemoryBridge(t *testing.T, device Tun, mtu int, callback bool) *memoryBridge { + bridge := &memoryBridge{device: device, done: make(chan struct{})} + bridge.inbound.resume = sync.NewCond(&bridge.inbound.access) + bridge.outbound.resume = sync.NewCond(&bridge.outbound.access) + memoryOptions := MemoryTunOptions{MTU: mtu} + if callback { + memoryOptions.Outbound = bridge.writeOutbound + } + bridge.memoryTun = NewMemoryTun(memoryOptions) + t.Cleanup(func() { bridge.memoryTun.Close() }) + go bridge.pumpInbound() + if !callback { + go bridge.pumpOutbound() + } + t.Cleanup(func() { + close(bridge.done) + bridge.inbound.pause(false) + bridge.outbound.pause(false) + }) + return bridge +} + +func (b *memoryBridge) pumpInbound() { + storage := make([]byte, 65536+PacketOffset) + for { + n, err := b.device.Read(storage) + if err != nil { + return + } + if n <= PacketOffset { + continue + } + b.inbound.pass() + select { + case <-b.done: + return + default: + } + b.observerAccess.Lock() + observer := b.inboundObserver + b.observerAccess.Unlock() + if observer != nil { + observer(storage[PacketOffset:n]) + } + _, err = b.memoryTun.WritePackets([][]byte{storage[PacketOffset:n]}) + if err != nil { + return + } + } +} + +func (b *memoryBridge) pumpOutbound() { + storage := make([][]byte, 64) + for index := range storage { + storage[index] = make([]byte, 65536+PacketOffset) + } + sizes := make([]int, len(storage)) + for { + count, err := b.memoryTun.ReadPackets(storage, sizes, PacketOffset) + if err != nil { + return + } + b.outbound.pass() + select { + case <-b.done: + return + default: + } + for index := range count { + err = b.forwardOutbound(storage[index][:PacketOffset+sizes[index]]) + if err != nil { + return + } + } + } +} + +func (b *memoryBridge) writeOutbound(packets []*buf.Buffer) error { + defer buf.ReleaseMulti(packets) + for _, packet := range packets { + b.outbound.pass() + select { + case <-b.done: + return os.ErrClosed + default: + } + frame := make([]byte, PacketOffset+packet.Len()) + copy(frame[PacketOffset:], packet.Bytes()) + err := b.forwardOutbound(frame) + if err != nil { + return err + } + } + return nil +} + +func (b *memoryBridge) forwardOutbound(frame []byte) error { + packet := frame[PacketOffset:] + if b.outboundDrop.Load() > 0 && b.outboundDrop.Add(-1) >= 0 { + return nil + } + b.observerAccess.Lock() + observer := b.outboundObserver + filter := b.outboundFilter + b.observerAccess.Unlock() + if filter != nil && !filter(packet) { + return nil + } + if observer != nil { + observer(packet) + } + for { + largest := b.outboundLargest.Load() + if int64(len(packet)) <= largest || b.outboundLargest.CompareAndSwap(largest, int64(len(packet))) { + break + } + } + PacketFillHeader(frame, header.IPVersion(packet)) + _, err := b.device.Write(frame) + return err +} + +func (b *memoryBridge) pauseOutbound(paused bool) { + b.outbound.pause(paused) +} + +func (b *memoryBridge) pauseInbound(paused bool) { + b.inbound.pause(paused) +} + +func (b *memoryBridge) observe(inbound func(packet []byte), outbound func(packet []byte)) { + b.observerAccess.Lock() + b.inboundObserver = inbound + b.outboundObserver = outbound + b.observerAccess.Unlock() +} + +func (b *memoryBridge) filterOutbound(filter func(packet []byte) bool) { + b.observerAccess.Lock() + b.outboundFilter = filter + b.observerAccess.Unlock() +} + +func (f *kernelStackFixture) kernelAddress(ipv6 bool) netip.Addr { + if ipv6 { + return f.options.Inet6Address[0].Addr() + } + return f.options.Inet4Address[0].Addr() +} + func (f *kernelStackFixture) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { return FlowVerdict{Action: ActionAccept} } @@ -184,11 +387,7 @@ func (f *kernelStackFixture) NewPacketConnectionEx(_ context.Context, conn N.Pac } func (f *kernelStackFixture) address(ipv6 bool, port uint16) string { - address := f.options.Inet4Address[0].Addr().Next() - if ipv6 { - address = f.options.Inet6Address[0].Addr().Next() - } - return netip.AddrPortFrom(address, port).String() + return netip.AddrPortFrom(f.kernelAddress(ipv6).Next(), port).String() } func (f *kernelStackFixture) pair(t *testing.T, ipv6 bool) (*net.TCPConn, *GoConn) { @@ -362,13 +561,19 @@ func TestGoKernelStream(t *testing.T) { if runtime.GOOS == "linux" { configs = append(configs, kernelStackConfig{mtu: 1500, gso: true}, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}, kernelStackConfig{mtu: 9000, memoryLink: true}) + } for _, config := range configs { - t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v/memory=%v", config.mtu, config.gso, config.multiQueue, config.memoryLink), func(configurationTest *testing.T) { fixture := newKernelStackFixture(configurationTest, config) for _, ipv6 := range []bool{false, true} { configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { addressTest.Parallel() for _, splice := range []bool{false, true} { + if splice && config.memoryLink { + continue + } addressTest.Run(fmt.Sprintf("duplex/splice=%v", splice), func(scenarioTest *testing.T) { scenarioTest.Parallel() var client, server net.Conn @@ -436,11 +641,6 @@ func TestGoKernelStream(t *testing.T) { } } -var ( - _ Handler = (*kernelStackFixture)(nil) - _ SpliceSocket = (*kernelSocket)(nil) -) - func TestGoKernelFlowControl(t *testing.T) { previous := runtime.GOMAXPROCS(4) defer runtime.GOMAXPROCS(previous) diff --git a/stack_go_io.go b/stack_go_io.go index 38a859da..1ef46e7c 100644 --- a/stack_go_io.go +++ b/stack_go_io.go @@ -46,10 +46,14 @@ type goPlatformIO interface { readBurst(frames []goFrame, options N.ReadWaitOptions) (count int, drained bool, err error) writeFrame(frame [][]byte, meta ForwardFrameMeta) error writePacket(packet []byte, meta ForwardFrameMeta) error - writeData(frame [][]byte, meta ForwardFrameMeta) error + writeDatagram(packet []byte, meta ForwardFrameMeta) error + writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error writePacketBatch(frames []goUDPFrame) error + transmitBacklogBelowBatch() bool releaseReadBuffers() flush() + mtu() int + supportsSockets() bool transmitPrefix() int transmitChecksumOffload() bool transmitSegmentOffload() bool @@ -71,10 +75,7 @@ func (w *goWriteback) WriteReturnPackets(packets [][]byte) error { prefix := w.platformIO.transmitPrefix() var writeErr error for _, packet := range packets { - err := goIgnoreDropped(w.platformIO.writePacket(packet[prefix:], ForwardFrameMeta{})) - if err != nil { - writeErr = E.Errors(writeErr, err) - } + writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeDatagram(packet[prefix:], ForwardFrameMeta{}))) } return writeErr } diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go index 1a459b16..31d9e786 100644 --- a/stack_go_io_darwin.go +++ b/stack_go_io_darwin.go @@ -53,6 +53,7 @@ type goDarwinIO struct { stack *Go tunFd int kqueueFd int + pollAccess sync.RWMutex transmitAccess *sync.Mutex receiveBuffers []*buf.Buffer readWaitOptions N.ReadWaitOptions @@ -401,9 +402,12 @@ func (o *goDarwinIO) writePacket(packet []byte, meta ForwardFrameMeta) error { return o.writeFrame(frame[:], meta) } -func (o *goDarwinIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { +func (o *goDarwinIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { o.transmitAccess.Lock() defer o.transmitAccess.Unlock() + if o.closing.Load() { + return os.ErrClosed + } frameLength := 0 for _, segment := range frame { frameLength += len(segment) @@ -460,6 +464,14 @@ func (o *goDarwinIO) writeUnbatchedLocked(frame [][]byte) error { return o.writePacketLocked(frame) } +func (o *goDarwinIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goDarwinIO) transmitBacklogBelowBatch() bool { + return true +} + func (o *goDarwinIO) flush() { o.transmitAccess.Lock() o.flushLocked() @@ -512,6 +524,9 @@ func (o *goDarwinIO) gateRoom(pending int) int { // consumed every frame handed to it: the ones before the failure were delivered, the failing one // is charged to the kernel (an ENOSPC leaks it), the rest are dropped and left to retransmission. func (o *goDarwinIO) flushLocked() { + if o.closing.Load() { + return + } pending := o.batchCount - o.batchStart if pending == 0 { o.gateReleaseLocked() @@ -523,6 +538,9 @@ func (o *goDarwinIO) flushLocked() { messages := o.batchMessages[o.batchStart : o.batchStart+room] n, errno := rawfile.NonBlockingSendMMsg(o.tunFd, messages) if errno == unix.EINTR { + if o.closing.Load() { + return + } continue } if errno == unix.ENOBUFS || errno == unix.EAGAIN { @@ -676,6 +694,9 @@ func (o *goDarwinIO) compactBatch() { } func (o *goDarwinIO) writePacketLocked(frame [][]byte) error { + if o.closing.Load() { + return os.ErrClosed + } defer func() { clear(o.transmitIovecs) }() if o.gateRoom(1) == 0 { o.gateBlockLocked() @@ -730,11 +751,24 @@ func (o *goDarwinIO) transmitChecksumOffload() bool { return false } +func (o *goDarwinIO) mtu() int { + return o.stack.mtu +} + +func (o *goDarwinIO) supportsSockets() bool { + return true +} + func (o *goDarwinIO) transmitSegmentOffload() bool { return false } func (o *goDarwinIO) armTransmitWritable() (bool, error) { + o.pollAccess.RLock() + defer o.pollAccess.RUnlock() + if o.closing.Load() { + return false, os.ErrClosed + } if o.gateBlocked.Load() { return true, nil } @@ -750,17 +784,34 @@ func (o *goDarwinIO) takeTransmitWritable() bool { } func (o *goDarwinIO) wake() { + o.pollAccess.RLock() + defer o.pollAccess.RUnlock() + if o.closing.Load() { + return + } _, _ = unix.Kevent(o.kqueueFd, o.wakeEvent[:], nil, nil) } func (o *goDarwinIO) close() error { o.closing.Store(true) + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + o.pollAccess.Lock() + defer o.pollAccess.Unlock() + clear(o.batchIovecs) + clear(o.batchMessages) + o.batchSpill = nil + o.batchCount = 0 + o.batchStart = 0 return unix.Close(o.kqueueFd) } func (o *goDarwinIO) writePacketBatch(frames []goUDPFrame) error { o.transmitAccess.Lock() defer o.transmitAccess.Unlock() + if o.closing.Load() { + return os.ErrClosed + } defer func() { clear(o.packetMessages[:]) clear(o.packetIovecs[:]) @@ -811,6 +862,9 @@ func (o *goDarwinIO) writePacketBatch(frames []goUDPFrame) error { } written, errno := rawfile.NonBlockingSendMMsg(o.tunFd, o.packetMessages[:count]) if errno == unix.EINTR { + if o.closing.Load() { + return os.ErrClosed + } continue } if (errno == unix.EMSGSIZE || errno == unix.ENOBUFS) && count > 1 { diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go index 494731f8..31819d09 100644 --- a/stack_go_io_linux.go +++ b/stack_go_io_linux.go @@ -5,6 +5,7 @@ import ( "net/netip" "os" "runtime" + "sync" "sync/atomic" "time" "unsafe" @@ -40,6 +41,7 @@ type goLinuxIO struct { ownsFd bool epollFd int eventFd int + fdAccess sync.RWMutex receiveBuffers []*buf.Buffer readWaitOptions N.ReadWaitOptions events [goSocketEventBatch + 2]unix.EpollEvent @@ -368,6 +370,9 @@ func (o *goLinuxIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { o.device.droppedEngineFrames.record(o.device.stack.logger, "engine frames") return errGoFrameDropped default: + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } return E.Cause(errno, "go: write tun") } } @@ -378,7 +383,7 @@ func (o *goLinuxIO) writePacket(packet []byte, meta ForwardFrameMeta) error { return o.writeFrame(frame[:], meta) } -func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { +func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { delay := goTransmitBackoffMin waited := time.Duration(0) for { @@ -407,14 +412,30 @@ func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { waited += delay delay = min(2*delay, goTransmitBackoffMax) default: + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } return E.Cause(errno, "go: write tun") } } } func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Errno { + o.fdAccess.RLock() + defer o.fdAccess.RUnlock() + if o.device.closing.Load() { + return unix.EBADF + } var headerStorage [virtioNetHdrLen]byte var iovecStorage [goPacketBatchSize + 2]unix.Iovec + iovecs := iovecStorage[:] + iovecCapacity := len(frame) + if o.device.vnetHeader { + iovecCapacity++ + } + if iovecCapacity > len(iovecs) { + iovecs = make([]unix.Iovec, iovecCapacity) + } iovecCount := 0 if o.device.vnetHeader { prefix := goEmptyVirtioHeader[:] @@ -435,7 +456,7 @@ func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Er } vector := unix.Iovec{Base: &prefix[0]} vector.SetLen(len(prefix)) - iovecStorage[iovecCount] = vector + iovecs[iovecCount] = vector iovecCount++ } for _, segment := range frame { @@ -444,14 +465,22 @@ func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Er } vector := unix.Iovec{Base: &segment[0]} vector.SetLen(len(segment)) - iovecStorage[iovecCount] = vector + iovecs[iovecCount] = vector iovecCount++ } //nolint:staticcheck - _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecStorage[0])), uintptr(iovecCount)) + _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(iovecCount)) return errno } +func (o *goLinuxIO) mtu() int { + return o.device.stack.mtu +} + +func (o *goLinuxIO) supportsSockets() bool { + return true +} + func (o *goLinuxIO) transmitPrefix() int { if o.device.vnetHeader { return virtioNetHdrLen @@ -476,6 +505,11 @@ func (o *goLinuxIO) takeTransmitWritable() bool { } func (o *goLinuxIO) wake() { + o.fdAccess.RLock() + defer o.fdAccess.RUnlock() + if o.device.closing.Load() { + return + } var value [8]byte binary.NativeEndian.PutUint64(value[:], 1) _, _ = unix.Write(o.eventFd, value[:]) @@ -483,6 +517,8 @@ func (o *goLinuxIO) wake() { func (o *goLinuxIO) close() error { o.device.closing.Store(true) + o.fdAccess.Lock() + defer o.fdAccess.Unlock() err := E.Errors(unix.Close(o.epollFd), unix.Close(o.eventFd)) if o.ownsFd { err = E.Errors(err, unix.Close(o.tunFd)) @@ -490,10 +526,21 @@ func (o *goLinuxIO) close() error { return err } +func (o *goLinuxIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goLinuxIO) transmitBacklogBelowBatch() bool { + return true +} + func (o *goLinuxIO) flush() { } func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { + if o.device.closing.Load() { + return os.ErrClosed + } var writeError error var segments [goPacketBatchSize + 1][]byte for index := 0; index < len(frames); { @@ -554,8 +601,12 @@ func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { } errno := o.transmitFrame(segments[:end-index+1], meta) if errno == unix.EINTR { + frame.header = packetHeader continue } + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } if errno != 0 && end-index > 1 { switch errno { case unix.EINVAL, unix.EIO, unix.EOPNOTSUPP: diff --git a/stack_go_io_memory.go b/stack_go_io_memory.go new file mode 100644 index 00000000..86cb036c --- /dev/null +++ b/stack_go_io_memory.go @@ -0,0 +1,304 @@ +package tun + +import ( + "os" + "time" + + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +type goMemoryIO struct { + stack *Go + tun *MemoryTun + timer *time.Timer + handed []*buf.Buffer + dropped goDropCounter + stop chan struct{} +} + +func (o *goMemoryIO) start() error { + o.tun.attached.Store(true) + if o.tun.outboundHandler != nil { + o.stop = make(chan struct{}) + go o.deliverOutbound() + } + return nil +} + +func (o *goMemoryIO) inboundReady() (bool, error) { + if o.tun.closed.Load() { + return false, os.ErrClosed + } + o.tun.inboundAccess.Lock() + defer o.tun.inboundAccess.Unlock() + return !o.tun.inbound.empty(), nil +} + +func (o *goMemoryIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + ready, err := o.inboundReady() + if ready || err != nil || timeout == 0 { + return ready, 0, err + } + if timeout < 0 { + select { + case <-o.tun.engineWake: + case <-o.tun.closeSignal: + } + } else { + if o.timer == nil { + o.timer = time.NewTimer(timeout) + } else { + o.timer.Reset(timeout) + } + select { + case <-o.tun.engineWake: + o.timer.Stop() + case <-o.tun.closeSignal: + o.timer.Stop() + case <-o.timer.C: + } + } + ready, err = o.inboundReady() + return ready, 0, err +} + +func (o *goMemoryIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + return nil +} + +func (o *goMemoryIO) updateSocket(socket *goSocket, interest uint8) error { + return nil +} + +func (o *goMemoryIO) unregisterSocket(socket *goSocket) { +} + +func (o *goMemoryIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + o.releaseReadBuffers() + if o.tun.closed.Load() { + return 0, false, os.ErrClosed + } + o.tun.inboundHeadroom.Store(int32(options.FrontHeadroom)) + o.tun.inboundRearSpace.Store(int32(options.RearHeadroom)) + o.tun.inboundAccess.Lock() + defer o.tun.inboundAccess.Unlock() + count := 0 + for count < len(frames) && !o.tun.inbound.empty() { + buffer, _, _ := o.tun.inbound.pop() + if options.NeedHeadroom() { + buffer = options.Copy(buffer) + options.PostReturn(buffer) + } + o.handed = append(o.handed, buffer) + frames[count] = goFrame{buffer: buffer} + count++ + } + return count, o.tun.inbound.empty(), nil +} + +func (o *goMemoryIO) releaseReadBuffers() { + buf.ReleaseMulti(o.handed) + o.handed = o.handed[:0] +} + +type goOutboundClass uint8 + +const ( + goOutboundControl goOutboundClass = iota + goOutboundDatagram + goOutboundData +) + +func (o *goMemoryIO) enqueue(segments [][]byte, class goOutboundClass, owner *GoConn, segmentEnd uint64) error { + length := 0 + for _, segment := range segments { + length += len(segment) + } + o.tun.outboundAccess.Lock() + defer o.tun.outboundAccess.Unlock() + if o.tun.closed.Load() || !o.tun.attached.Load() { + return os.ErrClosed + } + ring := &o.tun.control + switch class { + case goOutboundDatagram: + ring = &o.tun.datagram + case goOutboundData: + ring = &o.tun.data + } + if ring.full() { + if class == goOutboundData { + return errGoTransmitBlocked + } + return errGoFrameDropped + } + if o.tun.control.empty() && o.tun.datagram.empty() && o.tun.data.empty() { + defer o.tun.readerWake.notify() + } + packet := o.tun.newOutboundPacket(length) + for _, segment := range segments { + packet.Write(segment) + } + mtu := o.tun.MTU() + if length <= mtu { + o.pushOutboundLocked(ring, packet, owner, segmentEnd) + return nil + } + defer packet.Release() + fragments, fragmentsBuilt := goFragmentPacket(&o.stack.fragmentIdentification, packet.Bytes(), mtu) + if !fragmentsBuilt { + return errGoFrameDropped + } + for _, fragment := range fragments { + if ring.full() { + return errGoFrameDropped + } + fragmentPacket := o.tun.newOutboundPacket(len(fragment)) + fragmentPacket.Write(fragment) + o.pushOutboundLocked(ring, fragmentPacket, owner, segmentEnd) + } + return nil +} + +func (o *goMemoryIO) pushOutboundLocked(ring *goPacketRing, packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + if owner != nil { + owner.frameEnqueued(packet.Len()) + o.tun.dataQueued.Add(1) + } + ring.push(packet, owner, segmentEnd) +} + +func (o *goMemoryIO) deliverOutbound() { + batch := make([]*buf.Buffer, o.tun.batchSize) + for { + o.tun.outboundAccess.Lock() + count := 0 + for count < len(batch) { + buffer, owner, segmentEnd := o.tun.popOutboundLocked() + if buffer == nil { + break + } + if owner != nil { + owner.frameDequeued(buffer.Len(), segmentEnd) + } + batch[count] = buffer + count++ + } + wakeEngine := o.tun.takeTransmitWritableLocked() + o.tun.outboundAccess.Unlock() + if wakeEngine { + o.tun.engineWake.notify() + } + if count > 0 { + err := o.tun.outboundHandler(batch[:count]) + if err != nil { + for range count { + o.dropped.record(o.stack.logger, "outbound frames") + } + } + clear(batch[:count]) + continue + } + select { + case <-o.tun.readerWake: + case <-o.tun.closeSignal: + return + case <-o.stop: + return + } + } +} + +func (o *goMemoryIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + return o.enqueue(frame, goOutboundControl, nil, 0) +} + +func (o *goMemoryIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + segments := [1][]byte{packet} + return o.enqueue(segments[:], goOutboundControl, nil, 0) +} + +func (o *goMemoryIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + segments := [1][]byte{packet} + return o.enqueue(segments[:], goOutboundDatagram, nil, 0) +} + +func (o *goMemoryIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + return o.enqueue(frame, goOutboundData, owner, segmentEnd) +} + +func (o *goMemoryIO) writePacketBatch(frames []goUDPFrame) error { + var writeErr error + for index := range frames { + frame := &frames[index] + segments := [2][]byte{frame.header[:frame.length], frame.payload} + writeErr = E.Errors(writeErr, o.enqueue(segments[:], goOutboundDatagram, nil, 0)) + } + return writeErr +} + +func (o *goMemoryIO) transmitBacklogBelowBatch() bool { + return int(o.tun.dataQueued.Load()) < o.tun.batchSize +} + +func (o *goMemoryIO) flush() { +} + +func (o *goMemoryIO) mtu() int { + return o.tun.MTU() +} + +func (o *goMemoryIO) supportsSockets() bool { + return false +} + +func (o *goMemoryIO) transmitPrefix() int { + return 0 +} + +func (o *goMemoryIO) transmitChecksumOffload() bool { + return false +} + +func (o *goMemoryIO) transmitSegmentOffload() bool { + return false +} + +func (o *goMemoryIO) armTransmitWritable() (bool, error) { + o.tun.outboundAccess.Lock() + defer o.tun.outboundAccess.Unlock() + if !o.tun.data.full() { + return false, nil + } + o.tun.transmitInterest = true + return true, nil +} + +func (o *goMemoryIO) takeTransmitWritable() bool { + o.tun.outboundAccess.Lock() + defer o.tun.outboundAccess.Unlock() + writable := o.tun.transmitWritable + o.tun.transmitWritable = false + return writable +} + +func (o *goMemoryIO) wake() { + o.tun.engineWake.notify() +} + +func (o *goMemoryIO) close() error { + o.releaseReadBuffers() + o.tun.attached.Store(false) + if o.stop != nil { + close(o.stop) + } + o.tun.inboundAccess.Lock() + o.tun.inbound.releaseAll() + o.tun.inboundAccess.Unlock() + if o.timer != nil { + o.timer.Stop() + } + return nil +} diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index 6544d365..df782a78 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -2,6 +2,7 @@ package tun import ( "os" + "sync" "sync/atomic" "time" "unsafe" @@ -43,6 +44,7 @@ type goWindowsIO struct { // CreateEventW(&SecurityAttributes, FALSE, FALSE, NULL)). ringDrained bool droppedEngineFrames goDropCounter + wakeAccess sync.RWMutex closing atomic.Bool droppedDataFrames goDropCounter } @@ -228,9 +230,6 @@ func (o *goWindowsIO) registerSocket(socket *goSocket, token uint32, interest ui } entry := &goAFDEntry{baseHandle: baseHandle, token: token, interest: interest} entry.pinner.Pin(entry) - if o.entries == nil { - o.entries = make(map[*goAFDEntry]struct{}) - } o.entries[entry] = struct{}{} socket.entry = entry if interest == 0 { @@ -358,7 +357,7 @@ func (o *goWindowsIO) writePacket(packet []byte, meta ForwardFrameMeta) error { return o.writeFrame(frame[:], meta) } -func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { +func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { backoff := goTransmitBackoffMin waited := time.Duration(0) for { @@ -380,6 +379,9 @@ func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta) error { } func (o *goWindowsIO) transmitFrame(frame [][]byte) error { + if o.closing.Load() { + return os.ErrClosed + } err := o.tun.transmitGather(frame) if err == nil || err == windows.ERROR_BUFFER_OVERFLOW || err == os.ErrClosed { return err @@ -395,6 +397,14 @@ func (o *goWindowsIO) transmitChecksumOffload() bool { return false } +func (o *goWindowsIO) mtu() int { + return o.stack.mtu +} + +func (o *goWindowsIO) supportsSockets() bool { + return true +} + func (o *goWindowsIO) transmitSegmentOffload() bool { return false } @@ -408,11 +418,21 @@ func (o *goWindowsIO) takeTransmitWritable() bool { } func (o *goWindowsIO) wake() { + o.wakeAccess.RLock() + defer o.wakeAccess.RUnlock() + if o.closing.Load() { + return + } _ = windows.PostQueuedCompletionStatus(o.iocp, 0, goCompletionKeyWake, nil) } func (o *goWindowsIO) close() error { - o.closing.Store(true) + o.wakeAccess.Lock() + closing := o.closing.Swap(true) + o.wakeAccess.Unlock() + if closing { + return nil + } var err error if o.waitPacket != nil { err = E.Errors(o.waitPacket.Cancel(), o.waitPacket.Close()) @@ -421,13 +441,81 @@ func (o *goWindowsIO) close() error { <-o.bridgeDone err = E.Errors(windows.CloseHandle(o.bridgeArm), windows.CloseHandle(o.bridgeClose)) } - err = E.Errors(err, o.afd.Close(), windows.CloseHandle(o.iocp)) for entry := range o.entries { - entry.pinner.Unpin() + entry.interest = 0 + entry.cancelled = true + if entry.armed { + err = E.Errors(err, o.afd.Cancel(&entry.ioStatusBlock)) + } else { + o.releaseEntry(entry) + } + } + err = E.Errors(err, o.afd.Close()) + err = E.Errors(err, o.drainClosingEntries(goShutdownBudget)) + if len(o.entries) == 0 { + return E.Errors(err, windows.CloseHandle(o.iocp)) } - clear(o.entries) + go o.finishClose() return err } +func (o *goWindowsIO) drainClosingEntries(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for len(o.entries) > 0 { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil + } + waitMillis := uint32((remaining + time.Millisecond - 1) / time.Millisecond) + var removed uint32 + errno := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) + if errno == windows.WAIT_TIMEOUT { + return nil + } + if errno != 0 { + return E.Cause(errno, "go: drain cancelled socket polls") + } + for index := range removed { + completion := &o.completions[index] + if completion.CompletionKey != 0 { + continue + } + entry := (*goAFDEntry)(unsafe.Pointer(completion.Overlapped)) + if _, pending := o.entries[entry]; pending { + entry.armed = false + o.releaseEntry(entry) + } + } + clear(o.completions[:removed]) + } + return nil +} + +func (o *goWindowsIO) finishClose() { + logged := false + for len(o.entries) > 0 { + err := o.drainClosingEntries(time.Second) + if err != nil { + if !logged { + o.stack.logger.Error(err) + logged = true + } + time.Sleep(time.Second) + } + } + err := windows.CloseHandle(o.iocp) + if err != nil { + o.stack.logger.Error(E.Cause(err, "go: close completion port")) + } +} + +func (o *goWindowsIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goWindowsIO) transmitBacklogBelowBatch() bool { + return true +} + func (o *goWindowsIO) flush() { } diff --git a/stack_go_listen.go b/stack_go_listen.go new file mode 100644 index 00000000..b9ef6c58 --- /dev/null +++ b/stack_go_listen.go @@ -0,0 +1,240 @@ +package tun + +import ( + "net" + "net/netip" + "os" + "sync" + "syscall" + "time" + + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/pipe" +) + +const goListenBacklog = 4096 + +type GoListener struct { + engine *goEngine + local netip.AddrPort + access sync.Mutex + pending []*GoConn + pendingHead int + backlog int + closed bool + openErr error + openSignal chan struct{} + closeSignal chan struct{} + closeDone chan struct{} + acceptSignal goSignal + deadline pipe.Deadline + openMessage goMessage + closeMessage goMessage +} + +func (s *Go) ListenTCP(local netip.AddrPort) (*GoListener, error) { + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + if !local.Addr().IsValid() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(local) + if err != nil { + return nil, err + } + listener := &GoListener{ + engine: engine, + local: local, + openSignal: make(chan struct{}), + closeSignal: make(chan struct{}), + closeDone: make(chan struct{}), + acceptSignal: make(goSignal, 1), + deadline: pipe.MakeDeadline(), + } + listener.openMessage = goMessage{kind: goMessageListenOpen, listener: listener} + listener.closeMessage = goMessage{kind: goMessageListenClose, listener: listener} + engine.postMessage(&listener.openMessage) + select { + case <-listener.openSignal: + case <-s.ctx.Done(): + listener.Close() + return nil, E.Cause(s.ctx.Err(), "go: listen") + } + listener.access.Lock() + err = listener.openErr + listener.access.Unlock() + if err != nil { + return nil, err + } + return listener, nil +} + +func (e *goEngine) lookupListener(local netip.AddrPort) *GoListener { + listener := e.tcpListeners[local] + if listener == nil && len(e.stack.engines) > 1 { + listener = e.stack.directory.lookupListener(local) + } + return listener +} + +func (e *goEngine) handleListenOpen(listener *GoListener) { + listener.access.Lock() + closed := listener.closed + listener.access.Unlock() + if closed { + close(listener.openSignal) + return + } + available := func(port uint16) bool { + return e.lookupListener(netip.AddrPortFrom(listener.local.Addr(), port)) == nil + } + port := listener.local.Port() + if port == 0 { + allocated := false + port, allocated = goAllocatePort(available) + if !allocated { + listener.access.Lock() + listener.openErr = E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted") + listener.access.Unlock() + close(listener.openSignal) + return + } + } else if !available(port) { + listener.access.Lock() + listener.openErr = E.Cause(syscall.EADDRINUSE, "go: listen ", listener.local) + listener.access.Unlock() + close(listener.openSignal) + return + } + listener.local = netip.AddrPortFrom(listener.local.Addr(), port) + e.stack.directory.insertListener(listener) + close(listener.openSignal) +} + +func (e *goEngine) handleListenClose(listener *GoListener) { + e.stack.directory.removeListener(listener) + listener.drain() + close(listener.closeDone) +} + +func (e *goEngine) closeAllListeners() { + for _, listener := range e.tcpListeners { + e.stack.directory.removeListener(listener) + listener.markClosed() + listener.drain() + } +} + +func (l *GoListener) reserve() bool { + l.access.Lock() + defer l.access.Unlock() + if l.closed || l.backlog >= goListenBacklog { + return false + } + l.backlog++ + return true +} + +func (l *GoListener) abandon() { + l.access.Lock() + l.backlog-- + l.access.Unlock() +} + +func (l *GoListener) deliver(conn *GoConn) { + l.access.Lock() + if l.closed { + l.backlog-- + l.access.Unlock() + conn.fail(net.ErrClosed) + return + } + l.pending = append(l.pending, conn) + l.access.Unlock() + l.acceptSignal.notify() +} + +func (l *GoListener) AcceptTCP() (*GoConn, error) { + for { + l.access.Lock() + if l.pendingHead < len(l.pending) { + conn := l.pending[l.pendingHead] + l.pending[l.pendingHead] = nil + l.pendingHead++ + if l.pendingHead == len(l.pending) { + l.pending = l.pending[:0] + l.pendingHead = 0 + } else if l.pendingHead >= len(l.pending)/2 { + remaining := copy(l.pending, l.pending[l.pendingHead:]) + clear(l.pending[remaining:]) + l.pending = l.pending[:remaining] + l.pendingHead = 0 + } + l.backlog-- + l.access.Unlock() + return conn, nil + } + closed := l.closed + l.access.Unlock() + if closed { + return nil, net.ErrClosed + } + select { + case <-l.acceptSignal: + case <-l.closeSignal: + case <-l.deadline.Wait(): + return nil, os.ErrDeadlineExceeded + } + } +} + +func (l *GoListener) Accept() (net.Conn, error) { + conn, err := l.AcceptTCP() + if err != nil { + return nil, err + } + return conn, nil +} + +func (l *GoListener) markClosed() bool { + l.access.Lock() + if l.closed { + l.access.Unlock() + return false + } + l.closed = true + l.access.Unlock() + close(l.closeSignal) + return true +} + +func (l *GoListener) drain() { + l.access.Lock() + pending := l.pending[l.pendingHead:] + l.backlog -= len(pending) + l.pending = nil + l.pendingHead = 0 + l.access.Unlock() + for _, conn := range pending { + conn.fail(net.ErrClosed) + } +} + +func (l *GoListener) Close() error { + if l.markClosed() { + l.engine.postMessage(&l.closeMessage) + } + select { + case <-l.closeDone: + case <-l.engine.exitSignal: + } + return nil +} + +func (l *GoListener) Addr() net.Addr { + return net.TCPAddrFromAddrPort(l.local) +} + +func (l *GoListener) SetDeadline(t time.Time) error { + l.deadline.Set(t) + return nil +} diff --git a/stack_go_ring.go b/stack_go_ring.go index 522ff990..e6344f28 100644 --- a/stack_go_ring.go +++ b/stack_go_ring.go @@ -649,10 +649,6 @@ func (s *goScoreboard) split(index int, offset uint64) { s.entries[index].endOffset = offset } -func (s *goScoreboard) reset() { - s.entries = nil -} - func (s *goScoreboard) trim() { if len(s.entries) == 0 && cap(s.entries) > goScoreboardInlineCapacity { s.entries = nil diff --git a/stack_go_socket_unix.go b/stack_go_socket_unix.go index 6ec31402..69799ca6 100644 --- a/stack_go_socket_unix.go +++ b/stack_go_socket_unix.go @@ -8,6 +8,7 @@ import ( "syscall" "unsafe" + "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" M "github.com/sagernet/sing/common/metadata" @@ -30,21 +31,13 @@ type goSocket struct { } func goSpliceSocket(conn syscall.Conn) (goSocket, error) { - rawConn, err := conn.SyscallConn() - if err != nil { - return goSocket{}, err - } - socket := goSocket{fd: -1} - var duplicateError error - err = rawConn.Control(func(fd uintptr) { - socket.fd, duplicateError = unix.FcntlInt(fd, unix.F_DUPFD_CLOEXEC, 0) + fd, err := control.Conn0[int](conn, func(fd uintptr) (int, error) { + return unix.FcntlInt(fd, unix.F_DUPFD_CLOEXEC, 0) }) - err = E.Errors(err, duplicateError) if err != nil { - socket.close() return goSocket{}, err } - return socket, nil + return goSocket{fd: fd}, nil } func (s *goSocket) close() { diff --git a/stack_go_splice.go b/stack_go_splice.go index 54a208e9..3602f43d 100644 --- a/stack_go_splice.go +++ b/stack_go_splice.go @@ -67,6 +67,9 @@ func (e *goEngine) allocateSpliceToken(stream *GoConn, packet *GoPacketConn) uin } slot := &e.spliceSlots[index] slot.generation = (slot.generation + 1) & (1<> goSpliceTokenShift - if index >= uint32(len(e.spliceSlots)) { - return nil - } - slot := &e.spliceSlots[index] + slot := &e.spliceSlots[token>>goSpliceTokenShift] if slot.generation != token&(1< w.mtu { + if len(data)+w.templateLength > w.platformIO.mtu() { e.flushPacketFrames() buffer := buf.NewSize(w.FrontHeadroom() + len(data)) buffer.Resize(w.FrontHeadroom(), 0) diff --git a/stack_go_tcp_ack.go b/stack_go_tcp_ack.go index 6752261f..d8de7b51 100644 --- a/stack_go_tcp_ack.go +++ b/stack_go_tcp_ack.go @@ -193,7 +193,7 @@ func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, } sample.ackDelayed = ackFlags&goAckMaybeDelayed != 0 delivered := conn.delivered - priorDelivered - e.rateGenerate(conn, sample, delivered, conn.lost-priorLost, e.now()) + e.rateGenerate(conn, sample, delivered, conn.lost-priorLost, e.coarseTime.Load()) e.validateWindow(conn) e.congestionControl(conn, delivered, ackFlags, sample) e.updateFrameLimit(conn) @@ -245,13 +245,13 @@ func (e *goEngine) applyDuplicateSack(conn *GoConn, block goRawSackBlock, priorU if end-start > conn.maxPeerWindow { return } - packets := (end - start + uint64(conn.effectiveMSS) - 1) / uint64(conn.effectiveMSS) + packets := (end - start + uint64(conn.effectiveMSS.Load()) - 1) / uint64(conn.effectiveMSS.Load()) conn.duplicateSegments += packets if conn.duplicateSegments > conn.totalRetransmits { return } *flags |= goAckDSACK - if end-start <= uint64(conn.effectiveMSS) && conn.probeHighSeq != 0 && conn.probeRetransmitted && end == conn.probeHighSeq { + if end-start <= uint64(conn.effectiveMSS.Load()) && conn.probeHighSeq != 0 && conn.probeRetransmitted && end == conn.probeHighSeq { *flags |= goAckDSACKProbe } if conn.reorderingSeen != 0 && *flags&goAckDSACKProbe == 0 { @@ -392,8 +392,9 @@ func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFac flags |= goAckDataAcked start = descriptor.endOffset } - if conn.finSent && ackOffset > conn.finOffset { + if conn.finSent && !conn.finAcked && ackOffset > conn.finOffset { conn.finAcked = true + close(conn.finAckedSignal) } if newUnacked > priorUnacked { conn.scoreboard.advance(newUnacked) @@ -404,7 +405,7 @@ func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFac } } e.refreshPacketsOut(conn) - now := e.now() + now := e.coarseTime.Load() nowStamp := conn.stamp(now) sequenceRoundTrip := int64(-1) sackRoundTrip := int64(-1) @@ -412,7 +413,7 @@ func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFac if sampled && !unsampled && flags&goAckRetransmittedAcked == 0 { sequenceRoundTrip = goStampMicros(nowStamp - firstStamp) controlRoundTrip = goStampMicros(nowStamp - lastStamp) - if packetsAcked == 1 && priorSacked == 0 && newUnacked-priorUnacked < uint64(conn.effectiveMSS) && + if packetsAcked == 1 && priorSacked == 0 && newUnacked-priorUnacked < uint64(conn.effectiveMSS.Load()) && sample.priorDelivered+1 == conn.delivered && flags&goAckAlert == 0 { flags |= goAckMaybeDelayed } @@ -438,7 +439,7 @@ func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFac *ackSample = goAckSample{ packetsAcked: packetsAcked, roundTripMicros: sample.roundTripMicros, - inFlightBytes: uint64(conn.effectiveMSS) * uint64(conn.delivered-sample.priorDelivered), + inFlightBytes: uint64(conn.effectiveMSS.Load()) * uint64(conn.delivered-sample.priorDelivered), } conn.congestion.packetsAcked(conn, ackSample) } @@ -583,26 +584,22 @@ func (e *goEngine) accountSplit(conn *GoConn, flags uint16) { } func (c *GoConn) descriptorSentTime(descriptor *goSentDescriptor) int64 { - now := c.engine.now() + now := c.engine.coarseTime.Load() units := (now - c.epoch) / goTimeUnit age := int64(int32(units) - descriptor.sentAt) return c.epoch + (units-age)*goTimeUnit } -func (c *GoConn) descriptorTimestamp(descriptor *goSentDescriptor) uint32 { - return goTimestampAt(c.descriptorSentTime(descriptor)) -} - func (c *GoConn) spuriousRetransmit(descriptor *goSentDescriptor) bool { - return descriptor.flags&goDescriptorRetransmitted != 0 && c.echoedTimestampBefore(c.descriptorTimestamp(descriptor)) + return descriptor.flags&goDescriptorRetransmitted != 0 && c.echoedTimestampBefore(goTimestampAt(c.descriptorSentTime(descriptor))) } func (e *goEngine) rackAdvance(conn *GoConn, descriptor *goSentDescriptor, endOffset uint64) { if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { return } - roundTrip := goStampMicros(conn.stamp(e.now()) - descriptor.sentAt) - if roundTrip < int64(conn.minRoundTripMicros()) && descriptor.flags&goDescriptorRetransmitted != 0 { + roundTrip := goStampMicros(conn.stamp(e.coarseTime.Load()) - descriptor.sentAt) + if roundTrip < int64(conn.roundTripMin.get()) && descriptor.flags&goDescriptorRetransmitted != 0 { return } conn.rackAdvanced = true @@ -622,7 +619,7 @@ func (e *goEngine) rackReorderWindow(conn *GoConn) int64 { return 0 } } - return min(int64(conn.minRoundTripMicros()>>2)*int64(conn.reorderWindowSteps), int64(conn.smoothedRoundTrip)) + return min(int64(conn.roundTripMin.get()>>2)*int64(conn.reorderWindowSteps), int64(conn.smoothedRoundTrip)) } func (e *goEngine) rackDescriptorTimeout(conn *GoConn, descriptor *goSentDescriptor, reorderWindow int64, now int64) int64 { @@ -633,7 +630,7 @@ func (e *goEngine) rackDetectLoss(conn *GoConn) int64 { if conn.rackEndOffset == 0 { return 0 } - now := e.now() + now := e.coarseTime.Load() reorderWindow := e.rackReorderWindow(conn) timeout := int64(0) ordered := conn.flight.retransmitted == 0 @@ -670,7 +667,7 @@ func (e *goEngine) rackMarkLost(conn *GoConn) bool { if timeout == 0 { return false } - conn.reorderDeadline = e.now() + (timeout+goRackTimeoutMinMicros)*int64(time.Microsecond) + conn.reorderDeadline = e.coarseTime.Load() + (timeout+goRackTimeoutMinMicros)*int64(time.Microsecond) e.rearmTimer(conn) return true } @@ -787,7 +784,7 @@ func (e *goEngine) fastretransAlert(conn *GoConn, priorUnacked uint64, numDupack } if *flags&goAckReneging != 0 && *flags&goAckUnackedAdvanced != 0 { delay := max(int64(conn.smoothedRoundTrip)/2, int64(10*time.Millisecond/time.Microsecond)) - conn.retransmitDeadline = e.now() + delay*int64(time.Microsecond) + conn.retransmitDeadline = e.coarseTime.Load() + delay*int64(time.Microsecond) e.rearmTimer(conn) *flags |= goAckKeepTimer return goRexmitNone @@ -883,7 +880,7 @@ func (e *goEngine) xmitRetransmitQueue(conn *GoConn) { } if conn.congestion.pacing { stamp := conn.pacingStamp.Load() - if stamp > e.now()+goWheelTick { + if stamp > e.coarseTime.Load()+goWheelTick { conn.pacingDeadline = stamp - goWheelTick e.rearmTimer(conn) return @@ -906,7 +903,7 @@ func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int, p if conn.peerWindow == 0 || offset >= limit { return 0 } - length = int(min(uint64(length), uint64(conn.effectiveMSS), limit-offset)) + length = int(min(uint64(length), uint64(conn.effectiveMSS.Load()), limit-offset)) for _, descriptor := range conn.scoreboard.entries { if descriptor.endOffset > offset { length = min(length, int(descriptor.endOffset-offset)) @@ -918,7 +915,7 @@ func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int, p return 0 } conn.totalRetransmits++ - conn.advancePacing(e.now(), length) + conn.advancePacing(e.coarseTime.Load(), length) end := offset + uint64(length) marked := e.markRetransmitted(conn, offset, end, probe) if !probe && marked > 0 { @@ -932,14 +929,14 @@ func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int, p conn.undoRetransmits += int32(marked) } if conn.retransmitStamp == 0 { - conn.retransmitStamp = goTimestampAt(e.now()) + conn.retransmitStamp = goTimestampAt(e.coarseTime.Load()) } return length } func (e *goEngine) markRetransmitted(conn *GoConn, start uint64, end uint64, probe bool) uint32 { previous := conn.sendUnacked.Load() - stamp := conn.stamp(e.now()) + stamp := conn.stamp(e.coarseTime.Load()) marked := uint32(0) appLimited := conn.appLimited.Load() != 0 scoreboard := &conn.scoreboard @@ -996,7 +993,7 @@ func (c *GoConn) publishPermit(ackOffset uint64) { limit := max(c.flight.packetsEnd, unacked) inFlight := c.flight.inFlight() if c.congestionWindow > inFlight { - limit += uint64(c.congestionWindow-inFlight) * uint64(c.effectiveMSS) + limit += uint64(c.congestionWindow-inFlight) * uint64(c.effectiveMSS.Load()) } windowEnd := ackOffset + c.peerWindow permit := max(min(limit, windowEnd), unacked) @@ -1009,8 +1006,7 @@ func (c *GoConn) publishPermit(ackOffset uint64) { c.wakeWriter() } -func (c *GoConn) gsoMaxSize() uint32 { - mss := uint32(c.effectiveMSS) +func (c *GoConn) gsoMaxSize(mss uint32) uint32 { if c.engine.platformIO.transmitSegmentOffload() { return max(goGSOMaxPayload/mss*mss, mss) } @@ -1024,8 +1020,8 @@ func (e *goEngine) updateFrameLimit(conn *GoConn) { return } bytes := rate >> goPacingShift - shift := conn.minRoundTripMicros() >> goTSORoundTripShift - gsoMax := uint64(conn.gsoMaxSize()) + shift := conn.roundTripMin.get() >> goTSORoundTripShift + gsoMax := uint64(conn.gsoMaxSize(conn.effectiveMSS.Load())) if shift < 64 { bytes += gsoMax >> shift } @@ -1034,7 +1030,7 @@ func (e *goEngine) updateFrameLimit(conn *GoConn) { if conn.congestion.minTSOSegments != nil { minSegments = conn.congestion.minTSOSegments(conn) } - mss := uint64(conn.effectiveMSS) + mss := uint64(conn.effectiveMSS.Load()) segments := max(uint32(bytes/mss), minSegments) conn.frameLimit.Store(uint32(min(uint64(segments)*mss, gsoMax))) } @@ -1067,14 +1063,14 @@ func (e *goEngine) armProbe(conn *GoConn) { if conn.smoothedRoundTrip > 0 { timeout = max(2*conn.smoothedRoundTrip, goProbeFloorMicros) } - if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS) { + if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS.Load()) { timeout += goProbeDelayedAck } if timeout >= conn.retransmitTimeout { conn.probeDeadline = 0 return } - conn.probeDeadline = e.now() + int64(timeout)*int64(time.Microsecond) + conn.probeDeadline = e.coarseTime.Load() + int64(timeout)*int64(time.Microsecond) } func (e *goEngine) expireProbe(conn *GoConn, now int64) { @@ -1097,6 +1093,11 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { e.rearmTimer(conn) return } + if conn.flightHeadQueued() { + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + return + } conn.probeAttempts++ e.summarizeFlight(conn) sent := conn.sentTail.Load() @@ -1109,7 +1110,7 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { conn.access.Unlock() switch { case pending > 0 && committed == sent && windowEnd > sent && !writing: - length := min(pending, uint64(conn.effectiveMSS), windowEnd-sent) + length := min(pending, uint64(conn.effectiveMSS.Load()), windowEnd-sent) conn.probeRetransmitted = false conn.probeHighSeq = sent + length permit := conn.sendPermit.Load() @@ -1136,8 +1137,8 @@ func (e *goEngine) expireProbe(conn *GoConn, now int64) { end = committed } } - if end-start > uint64(conn.effectiveMSS) { - start = end - uint64(conn.effectiveMSS) + if end-start > uint64(conn.effectiveMSS.Load()) { + start = end - uint64(conn.effectiveMSS.Load()) } if e.transmitRetransmit(conn, start, int(end-start), true) > 0 { conn.probeRetransmitted = true @@ -1168,7 +1169,7 @@ func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { conn.probeDeadline = 0 conn.probeAttempts = 0 conn.reorderDeadline = 0 - if conn.hasOutstandingData() && conn.sendUnacked.Load() >= conn.transmittedTail.Load() { + if conn.hasOutstandingData() && conn.sendUnacked.Load() >= conn.transmittedTail.Load() || conn.flightHeadQueued() { conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) e.rearmTimer(conn) return @@ -1184,7 +1185,7 @@ func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { conn.retransmitAttempts++ unacked := conn.sendUnacked.Load() if conn.hasOutstandingData() { - length := int(min(conn.sentTail.Load()-unacked, uint64(conn.effectiveMSS))) + length := int(min(conn.sentTail.Load()-unacked, uint64(conn.effectiveMSS.Load()))) e.transmitRetransmit(conn, unacked, length, false) } else { e.retransmitFrom(conn, unacked) diff --git a/stack_go_tcp_bbr.go b/stack_go_tcp_bbr.go index a034b4f3..91a782a7 100644 --- a/stack_go_tcp_bbr.go +++ b/stack_go_tcp_bbr.go @@ -91,22 +91,18 @@ type goBBRState struct { } func goBBRJiffies(c *GoConn) uint32 { - return uint32(c.engine.now() / int64(time.Millisecond)) + return uint32(c.engine.coarseTime.Load() / int64(time.Millisecond)) } func goBBRAfter(a uint32, b uint32) bool { return int32(a-b) > 0 } -func (s *goBBRState) maxBandwidth() uint32 { - return s.bandwidth.get() -} - func (s *goBBRState) currentBandwidth() uint32 { if s.longTermUseBandwidth { return s.longTermBandwidth } - return s.maxBandwidth() + return s.bandwidth.get() } func (s *goBBRState) extraAckedMax() uint32 { @@ -114,7 +110,7 @@ func (s *goBBRState) extraAckedMax() uint32 { } func goBBRRateBytesPerSecond(c *GoConn, rate uint64, gain uint32) uint64 { - rate *= uint64(c.effectiveMSS) + rate *= uint64(c.effectiveMSS.Load()) rate *= uint64(gain) rate >>= goBBRScale rate *= uint64(time.Second/time.Microsecond) / 100 * (100 - goBBRPacingMarginPercent) @@ -150,7 +146,7 @@ func goBBRMinTSOSegments(c *GoConn) uint32 { func goBBRTSOSegmentsGoal(c *GoConn) uint32 { bytes := min(c.pacingRate.Load()>>goPacingShift, uint64(0xffff-1-goHeaderScratchSize)) - segments := max(uint32(bytes/uint64(c.effectiveMSS)), goBBRMinTSOSegments(c)) + segments := max(uint32(bytes/uint64(c.effectiveMSS.Load())), goBBRMinTSOSegments(c)) return min(segments, 0x7f) } @@ -166,7 +162,7 @@ func goBBRTxStart(c *GoConn, _ int64) { s := c.congestionPrivate.(*goBBRState) if c.appLimited.Load() != 0 { s.idleRestart = true - s.ackEpochStamp = c.engine.now() + s.ackEpochStamp = c.engine.coarseTime.Load() s.ackEpochAcked = 0 switch s.mode { case goBBRModeProbeBandwidth: @@ -199,7 +195,7 @@ func (s *goBBRState) inflight(c *GoConn, bandwidth uint32, gain uint32) uint32 { } func (s *goBBRState) packetsInNetAtDeparture(c *GoConn, inflightNow uint32) uint32 { - now := c.engine.now() + now := c.engine.coarseTime.Load() departure := max(c.pacingStamp.Load(), now) intervalMicros := uint64(departure-now) / uint64(time.Microsecond) intervalDelivered := uint32(uint64(s.currentBandwidth()) * intervalMicros >> goBBRBandwidthScale) @@ -274,7 +270,7 @@ func (s *goBBRState) isNextCyclePhase(c *GoConn, sample *goRateSample) bool { return fullLength } inflight := s.packetsInNetAtDeparture(c, sample.priorInFlight) - bandwidth := s.maxBandwidth() + bandwidth := s.bandwidth.get() if s.pacingGain > goBBRUnit { return fullLength && (sample.losses > 0 || inflight >= s.inflight(c, bandwidth, s.pacingGain)) } @@ -406,7 +402,7 @@ func (s *goBBRState) updateBandwidth(c *GoConn, sample *goRateSample) { } s.longTermSample(c, sample) bandwidth := uint64(sample.delivered) * goBBRBandwidthUnit / uint64(sample.intervalMicros) - if !sample.appLimited || bandwidth >= uint64(s.maxBandwidth()) { + if !sample.appLimited || bandwidth >= uint64(s.bandwidth.get()) { s.bandwidth.runningMax(goBBRBandwidthRounds, s.roundTripCount, uint32(min(bandwidth, uint64(^uint32(0))))) } } @@ -446,8 +442,8 @@ func (s *goBBRState) checkFullBandwidthReached(sample *goRateSample) { return } threshold := uint32(uint64(s.fullBandwidth) * goBBRFullBandwidthThresh >> goBBRScale) - if s.maxBandwidth() >= threshold { - s.fullBandwidth = s.maxBandwidth() + if s.bandwidth.get() >= threshold { + s.fullBandwidth = s.bandwidth.get() s.fullBandwidthCount = 0 return } @@ -458,9 +454,9 @@ func (s *goBBRState) checkFullBandwidthReached(sample *goRateSample) { func (s *goBBRState) checkDrain(c *GoConn) { if s.mode == goBBRModeStartup && s.fullBandwidthReached { s.mode = goBBRModeDrain - c.slowStartThreshold = s.inflight(c, s.maxBandwidth(), goBBRUnit) + c.slowStartThreshold = s.inflight(c, s.bandwidth.get(), goBBRUnit) } - if s.mode == goBBRModeDrain && s.packetsInNetAtDeparture(c, c.flight.inFlight()) <= s.inflight(c, s.maxBandwidth(), goBBRUnit) { + if s.mode == goBBRModeDrain && s.packetsInNetAtDeparture(c, c.flight.inFlight()) <= s.inflight(c, s.bandwidth.get(), goBBRUnit) { s.resetProbeBandwidthMode(c) } } @@ -548,13 +544,13 @@ func goBBRInit(c *GoConn) { c.slowStartThreshold = goInfiniteSlowStartThreshold s.nextRoundDelivered = c.delivered s.previousCAState = goCongestionOpen - s.minRoundTripMicros = c.minRoundTripMicros() + s.minRoundTripMicros = c.roundTripMin.get() s.minRoundTripStamp = goBBRJiffies(c) s.bandwidth.reset(s.roundTripCount, 0) goBBRInitPacingRateFromRoundTrip(c, s) s.resetLongTermSampling(c) s.mode = goBBRModeStartup - s.ackEpochStamp = c.engine.now() + s.ackEpochStamp = c.engine.coarseTime.Load() } func goBBRUndoWindow(c *GoConn) uint32 { @@ -592,7 +588,3 @@ var goBBROps = goCongestionOps{ setState: goBBRSetState, pacing: true, } - -func init() { - goRegisterCongestionControl(&goBBROps) -} diff --git a/stack_go_tcp_cong.go b/stack_go_tcp_cong.go index da900d76..cf5fbde3 100644 --- a/stack_go_tcp_cong.go +++ b/stack_go_tcp_cong.go @@ -123,23 +123,15 @@ type goCongestionOps struct { pacing bool } -var goCongestionRegistry = make(map[string]*goCongestionOps) - -const goDefaultCongestionControl = "cubic" - -func goRegisterCongestionControl(ops *goCongestionOps) { - if ops.slowStartThreshold == nil || ops.undoWindow == nil || (ops.congAvoid == nil && ops.congControl == nil) { - panic("go: congestion control " + ops.name + " does not implement required ops") - } - if _, exists := goCongestionRegistry[ops.name]; exists { - panic("go: congestion control " + ops.name + " already registered") - } - goCongestionRegistry[ops.name] = ops +var goCongestionRegistry = map[string]*goCongestionOps{ + goRenoOps.name: &goRenoOps, + goCubicOps.name: &goCubicOps, + goBBROps.name: &goBBROps, } func goLookupCongestionControl(name string) (*goCongestionOps, error) { if name == "" { - name = goDefaultCongestionControl + name = goCubicOps.name } ops, found := goCongestionRegistry[name] if !found { @@ -152,10 +144,10 @@ func (c *GoConn) initCongestionControl(ops *goCongestionOps) { c.congestion = ops c.congestionWindow = goInitialWindow c.slowStartThreshold = goInfiniteSlowStartThreshold - c.congestionClamp = max(uint32(goMaxCongestionSize/uint64(c.effectiveMSS)), 2) + c.congestionClamp = max(uint32(goMaxCongestionSize/uint64(c.effectiveMSS.Load())), 2) c.reordering = goInitialReordering c.reorderWindowSteps = 1 - c.deliveredTime = c.engine.now() + c.deliveredTime = c.engine.coarseTime.Load() c.undoRetransmits = -1 c.priorSlowStartThreshold = 0 c.roundTripMin.reset(0, ^uint32(0)) @@ -213,10 +205,6 @@ func (c *GoConn) isWindowLimited() bool { return false } -func (c *GoConn) minRoundTripMicros() uint32 { - return c.roundTripMin.get() -} - func (c *GoConn) slowStart(acked uint32) uint32 { window := min(c.congestionWindow+acked, c.slowStartThreshold) acked -= window - c.congestionWindow @@ -265,10 +253,6 @@ var goRenoOps = goCongestionOps{ undoWindow: goRenoUndoWindow, } -func init() { - goRegisterCongestionControl(&goRenoOps) -} - func (e *goEngine) initUndo(conn *GoConn) { conn.undoMarker = conn.sendUnacked.Load() conn.undoRetransmits = int32(conn.flight.retransmitOut) @@ -328,7 +312,7 @@ func (e *goEngine) endWindowReduction(conn *GoConn) { } if conn.slowStartThreshold < goInfiniteSlowStartThreshold && (conn.congestionState == goCongestionCWR || conn.undoMarker != 0) { conn.setCongestionWindow(conn.slowStartThreshold) - conn.congestionWindowStamp = e.now() + conn.congestionWindowStamp = e.coarseTime.Load() } conn.congestionEvent(goCongestionEventCompleteCWR) } @@ -366,7 +350,7 @@ func (e *goEngine) enterLoss(conn *GoConn) { } else if !conn.sackPermitted { conn.renoSacked = 0 } - now := e.now() + now := e.coarseTime.Load() for index := range conn.scoreboard.entries { descriptor := &conn.scoreboard.entries[index] if reneging { @@ -386,7 +370,7 @@ func (e *goEngine) enterLoss(conn *GoConn) { } conn.setCongestionWindow(conn.flight.inFlight() + 1) conn.congestionWindowCount = 0 - conn.congestionWindowStamp = e.now() + conn.congestionWindowStamp = e.coarseTime.Load() if conn.congestionState <= goCongestionDisorder && conn.flight.sackedOut >= goInitialReordering { conn.reordering = min(conn.reordering, goInitialReordering) } @@ -434,7 +418,7 @@ func (e *goEngine) undoWindowReduction(conn *GoConn, unmarkLoss bool) { conn.slowStartThreshold = conn.priorSlowStartThreshold } } - conn.congestionWindowStamp = e.now() + conn.congestionWindowStamp = e.coarseTime.Load() conn.undoMarker = 0 conn.rackAdvanced = true } @@ -530,7 +514,7 @@ func (e *goEngine) checkSackReordering(conn *GoConn, lowOffset uint64) { return } metric := fack - lowOffset - mss := uint64(conn.effectiveMSS) + mss := uint64(conn.effectiveMSS.Load()) if metric > uint64(conn.reordering)*mss { conn.reordering = uint32(min((metric+mss-1)/mss, goMaxReordering)) } @@ -553,13 +537,13 @@ func (e *goEngine) congestionControl(conn *GoConn, ackedSacked uint32, flags goA e.reduceWindow(conn, ackedSacked, sample.losses, flags) } else if e.mayRaiseWindow(conn, flags) { conn.congestion.congAvoid(conn, ackedSacked) - conn.congestionWindowStamp = e.now() + conn.congestionWindowStamp = e.coarseTime.Load() } e.updatePacingRate(conn) } func (e *goEngine) updatePacingRate(conn *GoConn) { - rate := uint64(conn.effectiveMSS) * (uint64(time.Second/time.Microsecond) / 100) + rate := uint64(conn.effectiveMSS.Load()) * (uint64(time.Second/time.Microsecond) / 100) if conn.congestionWindow < conn.slowStartThreshold/2 { rate *= goPacingSlowStartRatio } else { @@ -584,7 +568,7 @@ func (e *goEngine) validateWindow(conn *GoConn) { conn.maxPacketsOut = peakPackets conn.windowUsageSeq = conn.sentTail.Load() } - now := e.now() + now := e.coarseTime.Load() if conn.isWindowLimited() { conn.congestionWindowUsed = 0 conn.congestionWindowStamp = now @@ -603,7 +587,7 @@ func (e *goEngine) armIdleRestart(conn *GoConn) { conn.idleDeadline = 0 return } - conn.idleDeadline = e.now() + int64(conn.retransmitTimeout)*int64(time.Microsecond) + conn.idleDeadline = e.coarseTime.Load() + int64(conn.retransmitTimeout)*int64(time.Microsecond) } func (e *goEngine) expireIdleRestart(conn *GoConn, now int64) { diff --git a/stack_go_tcp_cubic.go b/stack_go_tcp_cubic.go index 9530ac66..bc22403d 100644 --- a/stack_go_tcp_cubic.go +++ b/stack_go_tcp_cubic.go @@ -68,11 +68,11 @@ func (s *goCubicState) reset() { } func goCubicJiffies(c *GoConn) uint32 { - return uint32(c.engine.now()/int64(time.Millisecond)) + 1 + return uint32(c.engine.coarseTime.Load()/int64(time.Millisecond)) + 1 } func goCubicMicros(c *GoConn) int64 { - return c.engine.now() / int64(time.Microsecond) + return c.engine.coarseTime.Load() / int64(time.Microsecond) } func (s *goCubicState) hystartReset(c *GoConn) { @@ -228,7 +228,7 @@ func goCubicHystartAckDelay(c *GoConn) uint32 { if rate == 0 { return 0 } - return uint32(min(uint64(time.Second/time.Microsecond)/1000, uint64(c.gsoMaxSize())*4*uint64(time.Second/time.Microsecond)/rate)) + return uint32(min(uint64(time.Second/time.Microsecond)/1000, uint64(c.gsoMaxSize(c.effectiveMSS.Load()))*4*uint64(time.Second/time.Microsecond)/rate)) } func (s *goCubicState) hystartUpdate(c *GoConn, delay uint32) { @@ -292,7 +292,3 @@ var goCubicOps = goCongestionOps{ windowEventTxStart: goCubicTxStart, packetsAcked: goCubicAcked, } - -func init() { - goRegisterCongestionControl(&goCubicOps) -} diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index 46a10cc4..b6088e5e 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -94,7 +94,7 @@ func goBuildNoFlowReset(scratch []byte, parsed *forwardPacket, ident uint16) ([] packet := scratch[:ipHeaderLength+header.TCPMinimumSize] replyTCP := header.TCP(packet[ipHeaderLength:]) encodeResetTCP(replyTCP, tcpHdr) - pseudoSum := goEncodeNetworkHeader(packet, parsed.ipVersion, parsed.destination.Addr(), parsed.source.Addr(), header.TCPMinimumSize, ident) + pseudoSum := goEncodeNetworkHeader(packet, parsed.ipVersion, header.TCPProtocolNumber, parsed.destination.Addr(), parsed.source.Addr(), header.TCPMinimumSize, ident) replyTCP.SetChecksum(^replyTCP.CalculateChecksum(pseudoSum)) return packet, true } @@ -105,6 +105,9 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { clientISN := tcpHdr.SequenceNumber() existing := e.flows[key] if existing != nil { + if existing.state == goTCPSynSent { + return + } if existing.clientISN == clientISN { e.handleDuplicateSyn(existing) return @@ -118,6 +121,11 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { } e.removeFlow(existing) } + listener := e.lookupListener(parsed.destination) + if listener == nil && e.stack.handler == nil { + e.answerNoFlow(parsed) + return + } if len(e.flows) >= e.flowCapacity && !e.evictFlow() { return } @@ -125,8 +133,11 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { return } + if listener != nil && !listener.reserve() { + return + } synOptions := header.ParseSynOptions(tcpHdr[header.TCPMinimumSize:dataOffset], false) - now := e.now() + now := e.coarseTime.Load() conn := new(GoConn) source := M.SocksaddrFromNetIP(parsed.source) destination := M.SocksaddrFromNetIP(parsed.destination) @@ -142,23 +153,37 @@ func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { } localMSS := e.localMSS(parsed.ipVersion) conn.peerMSS = synOptions.MSS - conn.effectiveMSS = min(synOptions.MSS, localMSS) + effectiveMSS := min(synOptions.MSS, localMSS) if synOptions.TS { conn.timestampsEnabled = true conn.tsRecent.Store(synOptions.TSVal) - conn.effectiveMSS -= goTimestampOptionLength + effectiveMSS -= goTimestampOptionLength + } + conn.effectiveMSS.Store(uint32(effectiveMSS)) + e.initializeSession(conn, now, tcpHdr.WindowSize()) + conn.buildHandshake(localMSS, synOptions.WS >= 0, synOptions.SACKPermitted, header.TCPFlagSyn|header.TCPFlagAck, clientISN+1) + conn.keyed = true + e.stack.directory.insert(key, conn) + if listener != nil { + conn.listener = listener + conn.socket = true + e.handleEngage(conn) + return } - conn.receiveCapacityMax = min(uint64(goReceiveCapacityMax), uint64(0xffff)<= goSynAckAttempts { conn.handshakeDeadline = 0 e.detachConn(conn, E.New("go: handshake timeout"), goDeathAbortLinger) return } - err := conn.writeSynAck() + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) if err != nil { e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) } @@ -1209,7 +1247,7 @@ func (e *goEngine) retransmitFrom(conn *GoConn, offset uint64) { sent := conn.sentTail.Load() offset = max(offset, 1) if offset < sent { - e.transmitRetransmit(conn, offset, int(min(sent-offset, uint64(conn.effectiveMSS))), false) + e.transmitRetransmit(conn, offset, int(min(sent-offset, uint64(conn.effectiveMSS.Load()))), false) return } if conn.finSent && !conn.finAcked { @@ -1240,6 +1278,9 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { } if !established { close(conn.establishedSignal) + if conn.listener != nil { + conn.listener.abandon() + } } close(conn.closeSignal) conn.retransmitDeadline = 0 @@ -1252,9 +1293,9 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { conn.finPending = false switch class { case goDeathFinLinger: - conn.lingerDeadline = e.now() + int64(goFinLinger) + conn.lingerDeadline = e.coarseTime.Load() + int64(goFinLinger) case goDeathAbortLinger: - conn.lingerDeadline = e.now() + int64(goAbortLinger) + conn.lingerDeadline = e.coarseTime.Load() + int64(goAbortLinger) default: conn.lingerDeadline = 0 } @@ -1264,7 +1305,7 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { } if !conn.onDyingList { conn.onDyingList = true - conn.dyingSince = e.now() + conn.dyingSince = e.coarseTime.Load() conn.dyingNext = e.dyingList e.dyingList = conn } @@ -1290,8 +1331,10 @@ func (e *goEngine) removeFlowKey(conn *GoConn) { return } conn.keyed = false - delete(e.flows, conn.key) e.stack.directory.remove(conn.key, conn) + if e.dispatchStage == nil { + return + } reason := FlowCloseReset if conn.finReceived && conn.finSent { reason = FlowCloseFinished @@ -1301,7 +1344,7 @@ func (e *goEngine) removeFlowKey(conn *GoConn) { func (e *goEngine) reapDying() { var kept *GoConn - now := e.now() + now := e.coarseTime.Load() for conn := e.dyingList; conn != nil; { next := conn.dyingNext conn.dyingNext = nil @@ -1310,7 +1353,7 @@ func (e *goEngine) reapDying() { } else { if now-conn.dyingSince > int64(goDyingLeakTimeout) { conn.dyingSince = now - e.stack.logger.Warn("go: connection stuck in teardown: ", conn.destination) + e.stack.logger.Warn("go: connection stuck in teardown: ", conn.local) } conn.dyingNext = kept kept = conn @@ -1342,9 +1385,12 @@ func (c *GoConn) reclaim() bool { c.engine.platformIO.flush() c.receiveChain.releaseAll() c.transmitStore.releaseAll() + c.blockedValid = false + c.blockedFrame = nil + c.transmitSegments = nil c.oooRanges = nil c.descriptors.release() - c.scoreboard.reset() + c.scoreboard.entries = nil c.releaseCongestionControl() if c.receiveTarget.buffer != nil { c.receiveTarget.buffer.Release() @@ -1376,6 +1422,9 @@ func (e *goEngine) expireSweepTick(now int64) { conn.lastActivity = now } e.keepalive(conn, now) + if conn.dead { + continue + } conn.receiveChain.releaseBelow(conn.consumedTail.Load()) conn.releaseTransmitted(conn.sendUnacked.Load()) conn.releaseDrainedSlabs() @@ -1393,6 +1442,10 @@ func (e *goEngine) expireSweepTick(now int64) { func (e *goEngine) keepalive(conn *GoConn, now int64) { conn.access.Lock() userClosed := conn.userClosed + enabled := conn.keepaliveEnabled + idleThreshold := conn.keepaliveIdle + interval := conn.keepaliveInterval + count := conn.keepaliveCount conn.access.Unlock() if conn.state == goTCPFinWait2 && userClosed && now-conn.finWait2Since > int64(goFinWait2Timeout) { e.sendReset(conn) @@ -1400,19 +1453,19 @@ func (e *goEngine) keepalive(conn *GoConn, now int64) { return } idle := now - conn.lastActivity - if idle < int64(tcpEstablishedTimeout) { + if !enabled || idle < int64(idleThreshold) { conn.keepaliveProbes = 0 return } - if conn.state == goTCPSynReceived || conn.hasOutstanding() { + if conn.handshaking() || conn.hasOutstanding() { return } - sinceIdle := idle - int64(tcpEstablishedTimeout) - due := int64(conn.keepaliveProbes) * int64(goKeepaliveInterval) + sinceIdle := idle - int64(idleThreshold) + due := int64(conn.keepaliveProbes) * int64(interval) if sinceIdle < due { return } - if conn.keepaliveProbes >= goKeepaliveCount { + if conn.keepaliveProbes >= count { e.sendReset(conn) e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: keepalive timeout"), goDeathImmediate) return diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go index 216483ee..022f7b1c 100644 --- a/stack_go_tcp_output.go +++ b/stack_go_tcp_output.go @@ -7,6 +7,7 @@ import ( "os" "time" + "github.com/sagernet/sing-tun/gtcpip" "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" "github.com/sagernet/sing-tun/gtcpip/seqnum" @@ -109,6 +110,7 @@ func (c *GoConn) writeLocked(p []byte) (int, error) { c.access.Lock() dead := c.dead writeShut := c.writeShut + capacity := c.transmitCapacity c.access.Unlock() if dead { return total, c.closeError() @@ -124,7 +126,7 @@ func (c *GoConn) writeLocked(p []byte) (int, error) { if len(p) == 0 { return total, nil } - budget := c.writeBudget(len(p)) + budget := c.writeBudget(len(p), capacity) if budget <= 0 { err := c.parkWriter(1) if err != nil { @@ -146,15 +148,11 @@ func (c *GoConn) writeLocked(p []byte) (int, error) { func (c *GoConn) writeBufferLocked(buffer *buf.Buffer) error { length := buffer.Len() - if length > goTransmitCapacityMax { - defer buffer.Release() - _, err := c.writeLocked(buffer.Bytes()) - return err - } for { c.access.Lock() dead := c.dead writeShut := c.writeShut + capacity := c.transmitCapacity c.access.Unlock() if dead { buffer.Release() @@ -164,6 +162,11 @@ func (c *GoConn) writeBufferLocked(buffer *buf.Buffer) error { buffer.Release() return net.ErrClosed } + if uint64(length) > capacity { + defer buffer.Release() + _, err := c.writeLocked(buffer.Bytes()) + return err + } select { case <-c.writeDeadline.Wait(): buffer.Release() @@ -174,7 +177,7 @@ func (c *GoConn) writeBufferLocked(buffer *buf.Buffer) error { buffer.Release() return nil } - budget := c.writeBudget(length) + budget := c.writeBudget(length, capacity) if budget < length { if budget > 0 && c.storeEmpty() { defer buffer.Release() @@ -204,7 +207,7 @@ func (c *GoConn) FrontHeadroom() int { } func (c *GoConn) WriterMTU() int { - return int(c.gsoMaxSize()) + return int(c.gsoMaxSize(c.effectiveMSS.Load())) } func (c *GoConn) publishBuffered(tail uint64) { @@ -229,10 +232,10 @@ func (c *GoConn) transmitPublished() { case goTransmitBlocked: c.wakeTransmitterGoroutine() return - case goTransmitPaced: + case goTransmitPaced, goTransmitThrottled: return } - if c.closed() || c.nextTransmitLength() == 0 { + if c.closed() || c.nextTransmitLength(c.effectiveMSS.Load()) == 0 { return } } @@ -268,7 +271,7 @@ func (e *goEngine) transmitOnEngine(conn *GoConn, budget int) { case goTransmitBlocked: e.handleTransmitBlocked(conn) case goTransmitDone: - if !conn.dead && conn.nextTransmitLength() != 0 { + if !conn.dead && conn.nextTransmitLength(conn.effectiveMSS.Load()) != 0 { e.postMessage(&conn.transmitMessage) } } @@ -286,10 +289,17 @@ func (e *goEngine) handleTransmitRequest(conn *GoConn) { } func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { + if conn.dead { + return + } if !conn.transmitAccess.TryLock() { conn.transmitSignal.notify() return } + if !conn.blockedValid { + conn.transmitAccess.Unlock() + return + } segment := conn.blockedSegment end := segment.offset + uint64(segment.length) err := conn.transmitFrame(&segment) @@ -314,7 +324,7 @@ func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { return } if conn.congestion.pacing { - conn.advancePacing(e.now(), segment.length) + conn.advancePacing(e.coarseTime.Load(), segment.length) } conn.blockedValid = false conn.blockedFrame = nil @@ -373,7 +383,7 @@ func (c *GoConn) runTransmitter() { } } -func (c *GoConn) nextTransmitLength() int { +func (c *GoConn) nextTransmitLength(mss uint32) int { sent := c.sentTail.Load() pending := c.bufferedTail.Load() - sent if pending == 0 { @@ -392,12 +402,12 @@ func (c *GoConn) nextTransmitLength() int { c.windowLimitedSince.Store(true) return 0 } - length := int(min(pending, permit-sent, uint64(packets)*uint64(c.effectiveMSS))) - if pending < uint64(c.effectiveMSS) && sent > c.sendUnacked.Load() { + length := int(min(pending, permit-sent, uint64(packets)*uint64(mss))) + if pending < uint64(mss) && sent > c.sendUnacked.Load() { c.access.Lock() - writing := c.writing + held := c.writing || c.nagle && !c.finRequested c.access.Unlock() - if writing { + if held { return 0 } } @@ -405,17 +415,17 @@ func (c *GoConn) nextTransmitLength() int { // grows while every window is drained by one segment, so a window narrower than two segments is // always covered by two frames. allowance := permit - c.sendUnacked.Load() - if allowance < 2*uint64(c.effectiveMSS) && packets > 1 { + if allowance < 2*uint64(mss) && packets > 1 { length = min(length, int((allowance+1)/2)) } - return c.frameLength(length) + return c.frameLength(length, mss) } -func (c *GoConn) writeBudget(pending int) int { +func (c *GoConn) writeBudget(pending int, capacity uint64) int { released := c.sendReleased.Load() buffered := c.bufferedTail.Load() stored := buffered - released - if stored >= goTransmitCapacityMax { + if stored >= capacity { return 0 } if stored >= goTransmitReadAhead { @@ -426,7 +436,7 @@ func (c *GoConn) writeBudget(pending int) int { } unacked := c.sendUnacked.Load() permit := max(c.sendPermit.Load()&^goPermitWindowBit, unacked) - limit := min(permit+max(permit-unacked, goTransmitReadAhead), released+goTransmitCapacityMax) + limit := min(permit+max(permit-unacked, goTransmitReadAhead), released+capacity) budget := int64(limit) - int64(buffered) if pending > 0 { budget = min(budget, int64(pending)) @@ -441,8 +451,14 @@ func (c *GoConn) storeEmpty() bool { return c.bufferedTail.Load() == c.sendReleased.Load() } +func (c *GoConn) loadTransmitCapacity() uint64 { + c.access.Lock() + defer c.access.Unlock() + return c.transmitCapacity +} + func (c *GoConn) writeReady(required int) bool { - budget := c.writeBudget(required) + budget := c.writeBudget(required, c.loadTransmitCapacity()) return budget >= required || (budget > 0 && c.storeEmpty()) } @@ -501,6 +517,7 @@ const ( goTransmitDone goTransmitResult = iota goTransmitBlocked goTransmitPaced + goTransmitThrottled ) func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResult, bool) { @@ -523,7 +540,8 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu if budget > 0 && written >= budget { break } - length := c.nextTransmitLength() + mss := c.effectiveMSS.Load() + length := c.nextTransmitLength(mss) if length == 0 { sent := c.sentTail.Load() pending := c.bufferedTail.Load() - sent @@ -536,18 +554,22 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu } var now int64 if paced { - now = c.engine.readClock() + now = int64(time.Since(c.engine.epoch)) if stamp := c.pacingStamp.Load(); c.dataSegmentsOut.Load() >= goPacingUnpacedSegments && stamp > now+goWheelTick { c.requestPacing(stamp - goWheelTick) result = goTransmitPaced break } } + if c.queueThrottle(length) { + result = goTransmitThrottled + break + } written += length sent := c.sentTail.Load() segment := goSegment{offset: sent, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} - if length > int(c.effectiveMSS) { - segment.gsoSize = c.effectiveMSS + if length > int(mss) { + segment.gsoSize = uint16(mss) } var flags uint16 unacked := c.sendUnacked.Load() @@ -557,9 +579,9 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu armed = true } if !paced { - now = c.engine.readClock() + now = int64(time.Since(c.engine.epoch)) } - c.pushDescriptors(sent, length, c.stamp(now), flags, flightEmpty) + c.pushDescriptors(sent, length, mss, c.stamp(now), flags, flightEmpty) c.sentTail.Store(sent + uint64(length)) creditBase := c.packetCreditBase.Load() flight := uint64(c.dataSegmentsOut.Load() - creditBase) @@ -624,7 +646,7 @@ func (c *GoConn) retryBlocked() bool { return false } if c.congestion.pacing { - c.advancePacing(c.engine.readClock(), c.blockedSegment.length) + c.advancePacing(int64(time.Since(c.engine.epoch)), c.blockedSegment.length) } c.blockedValid = false c.transmittedTail.Store(end) @@ -639,13 +661,13 @@ func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint16) { c.descriptors.push(goSentDescriptor{endOffset: offset + uint64(length), sentAt: int32(length), flags: goDescriptorAmend | flags}) } -func (c *GoConn) pushDescriptors(offset uint64, length int, sentAt int32, flags uint16, flightEmpty bool) { +func (c *GoConn) pushDescriptors(offset uint64, length int, mss uint32, sentAt int32, flags uint16, flightEmpty bool) { delivered, deliveredStamp, firstSentStamp, appLimited := c.snapshotDelivery(sentAt, flightEmpty) flags |= goDescriptorRated if appLimited { flags |= goDescriptorAppLimited } - step := int(c.effectiveMSS) + step := int(mss) for length > 0 { span := min(length, step) offset += uint64(span) @@ -663,12 +685,12 @@ func (c *GoConn) pushDescriptors(offset uint64, length int, sentAt int32, flags } } -func (c *GoConn) frameLength(pending int) int { - limit := int(c.effectiveMSS) +func (c *GoConn) frameLength(pending int, mss uint32) int { + limit := int(mss) if c.engine.platformIO.transmitSegmentOffload() { - limit = int(c.gsoMaxSize()) + limit = int(c.gsoMaxSize(mss)) if frameLimit := int(c.frameLimit.Load()); frameLimit > 0 { - limit = min(limit, max(frameLimit, int(c.effectiveMSS))) + limit = min(limit, max(frameLimit, int(mss))) } } return min(pending, limit) @@ -692,6 +714,42 @@ func (c *GoConn) advancePacing(now int64, length int) { } } +const goQueueLimitShift = 8 + +func (c *GoConn) queueThrottle(length int) bool { + limit := max(2*int64(length), int64(c.pacingRate.Load()>>goQueueLimitShift)) + if c.queuedBytes.Load()+int64(length) <= limit || c.engine.platformIO.transmitBacklogBelowBatch() { + return false + } + if c.sentTail.Load() == c.sendUnacked.Load() { + return false + } + c.queueThrottled.Store(true) + if c.queuedBytes.Load()+int64(length) <= limit { + c.queueThrottled.Store(false) + return false + } + return true +} + +func (c *GoConn) frameEnqueued(length int) { + c.queuedBytes.Add(int64(length)) +} + +func (c *GoConn) frameDequeued(length int, segmentEnd uint64) { + c.queuedBytes.Add(-int64(length)) + if segmentEnd > c.departedTail.Load() { + c.departedTail.Store(segmentEnd) + } + if c.queueThrottled.Swap(false) { + c.engine.postMessage(&c.throttleMessage) + } +} + +func (c *GoConn) flightHeadQueued() bool { + return c.queuedBytes.Load() > 0 && c.sendUnacked.Load() >= c.departedTail.Load() +} + func (c *GoConn) requestPacing(deadline int64) { c.pacingRequest.Store(deadline) c.engine.postMessage(&c.pacingMessage) @@ -717,7 +775,7 @@ func (c *GoConn) transmitFrame(segment *goSegment) error { c.transmitSegments = frame } } - err := c.engine.platformIO.writeData(frame, meta) + err := c.engine.platformIO.writeData(frame, meta, c, segment.offset+uint64(segment.length)) switch err { case nil, errGoFrameDropped: c.blockedFrame = nil @@ -772,8 +830,8 @@ func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *go c.dataSentEdge.Store(edge) } tcpHdr.Encode(&header.TCPFields{ - SrcPort: c.destination.Port, - DstPort: c.source.Port, + SrcPort: c.local.Port, + DstPort: c.peer.Port, SeqNum: uint32(uint64(c.sendISN) + segment.offset), AckNum: uint32(uint64(c.clientISN) + receiveAck), DataOffset: uint8(layout.tcpHeaderLength), @@ -782,12 +840,12 @@ func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *go }) options := packet[layout.ipHeaderLength+header.TCPMinimumSize:] if layout.timestampLength > 0 { - goEncodeTimestampOption(options, goTimestampAt(c.engine.readClock()), c.tsRecent.Load()) + goEncodeTimestampOption(options, goTimestampAt(int64(time.Since(c.engine.epoch))), c.tsRecent.Load()) } if len(layout.sackBlocks) > 0 { goEncodeSackOption(options[layout.timestampLength:], layout.sackBlocks, c.clientISN) } - return goEncodeNetworkHeader(packet, c.ipVersion, c.destination.Addr, c.source.Addr, layout.tcpHeaderLength+segment.length, uint16(c.ident.Add(1))) + return goEncodeNetworkHeader(packet, c.ipVersion, header.TCPProtocolNumber, c.local.Addr, c.peer.Addr, layout.tcpHeaderLength+segment.length, uint16(c.ident.Add(1))) } func goNetworkHeaderLength(ipVersion uint8) int { @@ -797,29 +855,29 @@ func goNetworkHeaderLength(ipVersion uint8) int { return header.IPv6MinimumSize } -func goEncodeNetworkHeader(packet []byte, ipVersion uint8, source netip.Addr, destination netip.Addr, tcpLength int, ident uint16) uint16 { +func goEncodeNetworkHeader(packet []byte, ipVersion uint8, protocol tcpip.TransportProtocolNumber, source netip.Addr, destination netip.Addr, transportLength int, ident uint16) uint16 { if ipVersion == 4 { ipHdr := header.IPv4(packet) ipHdr.Encode(&header.IPv4Fields{ - TotalLength: uint16(header.IPv4MinimumSize + tcpLength), + TotalLength: uint16(header.IPv4MinimumSize + transportLength), ID: ident, TTL: synthesizedTTL, - Protocol: uint8(header.TCPProtocolNumber), + Protocol: uint8(protocol), SrcAddr: source, DstAddr: destination, }) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - return header.PseudoHeaderChecksum(header.TCPProtocolNumber, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(tcpLength)) + return header.PseudoHeaderChecksum(protocol, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(transportLength)) } ipHdr := header.IPv6(packet) ipHdr.Encode(&header.IPv6Fields{ - PayloadLength: uint16(tcpLength), - TransportProtocol: header.TCPProtocolNumber, + PayloadLength: uint16(transportLength), + TransportProtocol: protocol, HopLimit: synthesizedTTL, SrcAddr: source, DstAddr: destination, }) - return header.PseudoHeaderChecksum(header.TCPProtocolNumber, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(tcpLength)) + return header.PseudoHeaderChecksum(protocol, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(transportLength)) } func (c *GoConn) offloadMeta(tcpHdr header.TCP, pseudoSum uint16, layout *goFrameLayout, segment *goSegment) ForwardFrameMeta { @@ -910,43 +968,39 @@ func goEncodeSackOption(options []byte, blocks []goSackBlock, base uint32) { header.AddTCPOptionPadding(options, optionsLength) } -func (c *GoConn) buildSynAck(synOptions header.TCPSynOptions, localMSS uint16) { +func (c *GoConn) buildHandshake(localMSS uint16, windowScale bool, sackPermitted bool, flags header.TCPFlags, ackNumber uint32) { ipHeaderLength := goNetworkHeaderLength(c.ipVersion) - optionsStorage := c.synAckImage[ipHeaderLength+header.TCPMinimumSize:] + optionsStorage := c.handshakeImage[ipHeaderLength+header.TCPMinimumSize:] optionsLength := header.EncodeMSSOption(uint32(localMSS), optionsStorage) - if synOptions.WS >= 0 { + if windowScale { optionsLength += header.EncodeWSOption(int(c.localWindowShift), optionsStorage[optionsLength:]) } - if synOptions.SACKPermitted { + if sackPermitted { optionsLength += header.EncodeSACKPermittedOption(optionsStorage[optionsLength:]) } if c.timestampsEnabled { - optionsLength += header.EncodeTSOption(goTimestampAt(c.engine.readClock()), c.tsRecent.Load(), optionsStorage[optionsLength:]) + optionsLength += header.EncodeTSOption(goTimestampAt(int64(time.Since(c.engine.epoch))), c.tsRecent.Load(), optionsStorage[optionsLength:]) } optionsLength += header.AddTCPOptionPadding(optionsStorage, optionsLength) tcpHeaderLength := header.TCPMinimumSize + optionsLength totalLength := ipHeaderLength + tcpHeaderLength - c.synAckLength = uint8(totalLength) - packet := c.synAckImage[:totalLength] + c.handshakeLength = uint8(totalLength) + packet := c.handshakeImage[:totalLength] tcpHdr := header.TCP(packet[ipHeaderLength:]) tcpHdr.Encode(&header.TCPFields{ - SrcPort: c.destination.Port, - DstPort: c.source.Port, + SrcPort: c.local.Port, + DstPort: c.peer.Port, SeqNum: c.sendISN, - AckNum: c.clientISN + 1, + AckNum: ackNumber, DataOffset: uint8(tcpHeaderLength), - Flags: header.TCPFlagSyn | header.TCPFlagAck, + Flags: flags, WindowSize: uint16(min(c.receiveCapacity, 0xffff)), }) - pseudoSum := goEncodeNetworkHeader(packet, c.ipVersion, c.destination.Addr, c.source.Addr, tcpHeaderLength, uint16(c.ident.Add(1))) + pseudoSum := goEncodeNetworkHeader(packet, c.ipVersion, header.TCPProtocolNumber, c.local.Addr, c.peer.Addr, tcpHeaderLength, uint16(c.ident.Add(1))) tcpHdr.SetChecksum(0) tcpHdr.SetChecksum(^checksum.Checksum(tcpHdr, pseudoSum)) } -func (c *GoConn) writeSynAck() error { - return goIgnoreDropped(c.engine.platformIO.writePacket(c.synAckImage[:c.synAckLength], ForwardFrameMeta{})) -} - var errGoTransmitBlocked = E.New("go: transmit blocked") func goGSOType(ipVersion uint8) uint8 { diff --git a/stack_go_tcp_rate.go b/stack_go_tcp_rate.go index 20262f62..848528e5 100644 --- a/stack_go_tcp_rate.go +++ b/stack_go_tcp_rate.go @@ -78,7 +78,7 @@ func (e *goEngine) rateGenerate(conn *GoConn, sample *goRateSample, delivered ui receiveMicros := goStampMicros(stamp - sample.priorStamp) sample.intervalMicros = max(sendMicros, receiveMicros) sample.receiveIntervalMicros = receiveMicros - if sample.intervalMicros < int64(conn.minRoundTripMicros()) { + if sample.intervalMicros < int64(conn.roundTripMin.get()) { sample.intervalMicros = -1 return } @@ -91,7 +91,7 @@ func (e *goEngine) rateGenerate(conn *GoConn, sample *goRateSample, delivered ui func (c *GoConn) countDelivered(packets uint32) { c.delivered += packets - c.deliveredTime = c.engine.now() + c.deliveredTime = c.engine.coarseTime.Load() c.deliveredStamp = c.stamp(c.deliveredTime) c.publishDelivery() } @@ -111,7 +111,7 @@ func (c *GoConn) publishDelivery() { func (c *GoConn) checkAppLimited(pending uint64, permit uint64, sent uint64) { segments := c.dataSegmentsOut.Load() - if pending >= uint64(c.effectiveMSS) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerWaiting.Load() != 0 { + if pending >= uint64(c.effectiveMSS.Load()) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerWaiting.Load() != 0 { return } c.access.Lock() diff --git a/stack_go_udp.go b/stack_go_udp.go index 1b7a38be..3c933c37 100644 --- a/stack_go_udp.go +++ b/stack_go_udp.go @@ -58,7 +58,8 @@ func (e *goEngine) processUDPSegments(frame *goFrame) { return } count := (len(packet) - headerLength + int(meta.gsoSize) - 1) / int(meta.gsoSize) - if count == 0 || count > goGSOMaxSegments || headerLength+int(meta.gsoSize) > e.stack.mtu { + mtu := e.platformIO.mtu() + if count == 0 || count > goGSOMaxSegments || headerLength+int(meta.gsoSize) > mtu { return } options := *e.packetReadOptions.Load() @@ -70,7 +71,7 @@ func (e *goEngine) processUDPSegments(frame *goFrame) { for index := range count { buffer := e.gsoBuffers[index] if buffer == nil { - buffer = options.NewBufferSize(e.stack.mtu) + buffer = options.NewBufferSize(mtu) e.gsoBuffers[index] = buffer } buffer.Reset() @@ -101,6 +102,17 @@ func (e *goEngine) demuxUDP(packetBuffer *buf.Buffer, meta ForwardFrameMeta, par if len(parsed.transport) < header.UDPMinimumSize { return } + if e.udpNat == nil { + meta.completeChecksum(packet) + reply, ok := buildReject(parsed, 0) + if ok { + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write port unreachable")) + } + } + return + } switch parsed.verdict.Action { case ActionReject: meta.completeChecksum(packet) @@ -130,17 +142,16 @@ func (e *goEngine) demuxUDP(packetBuffer *buf.Buffer, meta ForwardFrameMeta, par meta: meta, ipVersion: parsed.ipVersion, } - conn, ok := e.udpNat.getOrCreate(key, source, destination, userData) + conn, ok := e.udpNat.getOrCreate(key, source, destination, userData, func(conn *UDPNatConn) { + e.attachUDPSession(conn, userData.created, &parsed.verdict) + userData.created = nil + }) if !ok { if userData.owner != nil { userData.owner.inject(packet, meta) } return } - if userData.created != nil { - e.attachUDPSession(conn, userData.created, &parsed.verdict) - userData.created = nil - } if writer, isGoPacketConn := conn.writer.(*GoPacketConn); isGoPacketConn { if writer.engine != e { writer.engine.inject(packet, meta) @@ -213,7 +224,6 @@ func (s *Go) prepareUDPConnection(source M.Socksaddr, destination M.Socksaddr, u engine: data.engine, key: data.key, platformIO: data.engine.platformIO, - mtu: s.mtu, checksumOffload: data.engine.platformIO.transmitChecksumOffload(), snapshot: slices.Clone(data.packet), snapshotMeta: data.meta, @@ -242,7 +252,6 @@ type GoPacketConn struct { spliceCloseError atomic.Pointer[goPacketCloseError] spliceMessage goMessage closeMessage goMessage - mtu int ipVersion uint8 templateLength int checksumOffload bool @@ -347,10 +356,6 @@ func (w *GoPacketConn) FrontHeadroom() int { } func (w *GoPacketConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - if !destination.IsIP() { - buffer.Release() - return E.Cause(os.ErrInvalid, "invalid destination") - } defer buffer.Release() return w.transmit(buffer, destination) } @@ -373,11 +378,9 @@ func (w *GoPacketConn) WritePacketBatch(buffers []*buf.Buffer, destinations []M. clear(frames[:count]) goUDPFramePool.Put(frames) }() + mtu := w.platformIO.mtu() for index, buffer := range buffers { - if !destinations[index].IsIP() { - return os.ErrInvalid - } - if buffer.Len()+w.templateLength > w.mtu { + if buffer.Len()+w.templateLength > mtu { if count > 0 { err := goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) if err != nil { @@ -420,7 +423,8 @@ func (w *GoPacketConn) WritePacketBatch(buffers []*buf.Buffer, destinations []M. func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) error { payload := buffer.Bytes() - fragmented := w.templateLength+len(payload) > w.mtu + mtu := w.platformIO.mtu() + fragmented := w.templateLength+len(payload) > mtu meta, err := w.preparePacketHeader(buffer.ExtendHeader(w.templateLength), payload, destination, w.checksumOffload && !fragmented) if err != nil { if w.ipVersion == 4 && destination.IsIPv6() { @@ -430,34 +434,32 @@ func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) err } packet := buffer.Bytes() if !fragmented { - return goIgnoreDropped(w.platformIO.writePacket(packet, meta)) - } - var ( - fragments [][]byte - fragmentsBuilt bool - ) - if w.ipVersion == 4 { - ipHdr := header.IPv4(packet) - ipHdr.SetID(uint16(w.engine.stack.udpIdentification.Add(1))) - ipHdr.SetChecksum(0) - ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - fragments, fragmentsBuilt = fragmentIPv4Packet(ipHdr, uint32(w.mtu)) - } else { - fragments, fragmentsBuilt = fragmentIPv6Packet(header.IPv6(packet), uint32(w.mtu), w.engine.stack.udpIdentification.Add(1)) + return goIgnoreDropped(w.platformIO.writeDatagram(packet, meta)) } + return goWriteFragmented(w.platformIO, &w.engine.stack.fragmentIdentification, packet, mtu) +} + +func goWriteFragmented(platformIO goPlatformIO, identification *atomic.Uint32, packet []byte, mtu int) error { + fragments, fragmentsBuilt := goFragmentPacket(identification, packet, mtu) if !fragmentsBuilt { return nil } var writeErr error for _, fragment := range fragments { - err = goIgnoreDropped(w.platformIO.writePacket(fragment, ForwardFrameMeta{})) - if err != nil { - writeErr = E.Errors(writeErr, err) - } + writeErr = E.Errors(writeErr, goIgnoreDropped(platformIO.writeDatagram(fragment, ForwardFrameMeta{}))) } return writeErr } +func goFragmentPacket(identification *atomic.Uint32, packet []byte, mtu int) ([][]byte, bool) { + if header.IPVersion(packet) == header.IPv4Version { + ipHdr := header.IPv4(packet) + ipHdr.SetID(uint16(identification.Add(1))) + return fragmentIPv4Packet(ipHdr, uint32(mtu)) + } + return fragmentIPv6Packet(header.IPv6(packet), uint32(mtu), identification.Add(1)) +} + func (w *GoPacketConn) HandshakeSuccess() error { w.snapshotAccess.Lock() w.snapshot = nil @@ -525,12 +527,8 @@ func (w *GoPacketConn) handleSessionClose(err error) { } var ( - _ N.PacketWriter = (*GoPacketConn)(nil) _ N.FrontHeadroom = (*GoPacketConn)(nil) _ N.HandshakeSuccess = (*GoPacketConn)(nil) _ N.HandshakeFailure = (*GoPacketConn)(nil) - _ io.Closer = (*GoPacketConn)(nil) - _ FlowHandle = (*GoPacketConn)(nil) _ N.PacketBatchWriteCreator = (*GoPacketConn)(nil) - _ N.PacketBatchWriter = (*GoPacketConn)(nil) ) diff --git a/stack_go_udp_socket.go b/stack_go_udp_socket.go new file mode 100644 index 00000000..abdb053b --- /dev/null +++ b/stack_go_udp_socket.go @@ -0,0 +1,462 @@ +package tun + +import ( + "context" + "io" + "net" + "net/netip" + "os" + "sync" + "syscall" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/common/pipe" +) + +// SK_RMEM_MAX in include/net/sock.h is the initial receive budget of every socket, and +// SOCK_MIN_RCVBUF the floor SO_RCVBUF is clamped to. +const ( + goUDPReceiveCapacity = 212992 + goUDPReceiveMinimum = 2304 +) + +type goUDPDatagram struct { + buffer *buf.Buffer + source netip.AddrPort +} + +func (d goUDPDatagram) memoryCost() int { + return d.buffer.RawCap() + int(unsafe.Sizeof(d)) + int(unsafe.Sizeof(*d.buffer)) +} + +type GoUDPConn struct { + engine *goEngine + local netip.AddrPort + remote netip.AddrPort + ipVersion uint8 + connected bool + access sync.Mutex + queue []goUDPDatagram + queueHead int + queuedBytes int + receiveCapacity int + readWaitOptions N.ReadWaitOptions + closed bool + openErr error + pendingErr error + openSignal chan struct{} + closeSignal chan struct{} + closeDone chan struct{} + readSignal goSignal + readDeadline pipe.Deadline + writeDeadline pipe.Deadline + openMessage goMessage + closeMessage goMessage +} + +func (s *Go) DialUDP(local netip.AddrPort, destination netip.AddrPort) (*GoUDPConn, error) { + destination = netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !destination.IsValid() || destination.Port() == 0 { + return nil, errGoInvalidAddress + } + return s.openUDP(local, destination) +} + +func (s *Go) ListenUDP(local netip.AddrPort) (*GoUDPConn, error) { + return s.openUDP(local, netip.AddrPort{}) +} + +func (s *Go) openUDP(local netip.AddrPort, destination netip.AddrPort) (*GoUDPConn, error) { + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + if !local.Addr().IsValid() || destination.IsValid() && local.Addr().Is4() != destination.Addr().Is4() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(local) + if err != nil { + return nil, err + } + socket := &GoUDPConn{ + engine: engine, + local: local, + remote: destination, + ipVersion: goIPVersion(local.Addr()), + connected: destination.IsValid(), + receiveCapacity: min(goUDPReceiveCapacity, goReceiveCapacityMax), + openSignal: make(chan struct{}), + closeSignal: make(chan struct{}), + closeDone: make(chan struct{}), + readSignal: make(goSignal, 1), + readDeadline: pipe.MakeDeadline(), + writeDeadline: pipe.MakeDeadline(), + } + socket.openMessage = goMessage{kind: goMessageUDPOpen, socket: socket} + socket.closeMessage = goMessage{kind: goMessageUDPClose, socket: socket} + engine.postMessage(&socket.openMessage) + select { + case <-socket.openSignal: + case <-s.ctx.Done(): + socket.Close() + return nil, E.Cause(context.Cause(s.ctx), "go: open UDP socket") + } + socket.access.Lock() + err = socket.openErr + socket.access.Unlock() + if err != nil { + return nil, err + } + return socket, nil +} + +func (e *goEngine) handleUDPOpen(socket *GoUDPConn) { + socket.access.Lock() + closed := socket.closed + socket.access.Unlock() + if closed { + close(socket.openSignal) + return + } + available := func(port uint16) bool { + local := netip.AddrPortFrom(socket.local.Addr(), port) + return e.udpSockets[local] == nil && (len(e.stack.engines) == 1 || e.stack.directory.lookupUDPSocket(local) == nil) + } + port := socket.local.Port() + if port == 0 { + allocated := false + port, allocated = goAllocatePort(available) + if !allocated { + socket.access.Lock() + socket.openErr = E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted") + socket.access.Unlock() + close(socket.openSignal) + return + } + } else if !available(port) { + socket.access.Lock() + socket.openErr = E.Cause(syscall.EADDRINUSE, "go: bind ", socket.local) + socket.access.Unlock() + close(socket.openSignal) + return + } + socket.local = netip.AddrPortFrom(socket.local.Addr(), port) + e.stack.directory.insertUDPSocket(socket) + close(socket.openSignal) +} + +func (e *goEngine) handleUDPClose(socket *GoUDPConn) { + e.stack.directory.removeUDPSocket(socket) + socket.drain() + close(socket.closeDone) +} + +func (e *goEngine) closeAllUDPSockets() { + for _, socket := range e.udpSockets { + e.stack.directory.removeUDPSocket(socket) + socket.markClosed(net.ErrClosed) + socket.drain() + } +} + +func (e *goEngine) inputUDPSocket(socket *GoUDPConn, parsed *forwardPacket) { + if socket.connected && parsed.source != socket.remote { + return + } + payload := header.UDP(parsed.transport).Payload() + socket.access.Lock() + if socket.closed || socket.queuedBytes >= socket.receiveCapacity { + socket.access.Unlock() + return + } + buffer := socket.readWaitOptions.NewBufferSize(len(payload)) + buffer.Write(payload) + socket.readWaitOptions.PostReturn(buffer) + datagram := goUDPDatagram{buffer: buffer, source: parsed.source} + socket.queue = append(socket.queue, datagram) + socket.queuedBytes += datagram.memoryCost() + socket.access.Unlock() + if socket.readSignal.notify() { + e.wokeHandlerThisBurst = true + } +} + +func (c *GoUDPConn) deliverError(err error) { + c.access.Lock() + c.pendingErr = err + c.access.Unlock() + c.readSignal.notify() +} + +func (c *GoUDPConn) InitializeReadWaiter(options N.ReadWaitOptions) bool { + c.access.Lock() + c.readWaitOptions = options + c.access.Unlock() + return false +} + +func (c *GoUDPConn) waitDatagram() (goUDPDatagram, error) { + for { + select { + case <-c.readDeadline.Wait(): + return goUDPDatagram{}, os.ErrDeadlineExceeded + default: + } + c.access.Lock() + if c.pendingErr != nil { + err := c.pendingErr + c.pendingErr = nil + c.access.Unlock() + return goUDPDatagram{}, err + } + if c.queueHead < len(c.queue) { + datagram := c.queue[c.queueHead] + c.queue[c.queueHead] = goUDPDatagram{} + c.queueHead++ + if c.queueHead == len(c.queue) { + c.queue = c.queue[:0] + c.queueHead = 0 + } else if c.queueHead >= len(c.queue)/2 { + remaining := copy(c.queue, c.queue[c.queueHead:]) + clear(c.queue[remaining:]) + c.queue = c.queue[:remaining] + c.queueHead = 0 + } + c.queuedBytes -= datagram.memoryCost() + datagram.buffer = c.readWaitOptions.Copy(datagram.buffer) + c.access.Unlock() + return datagram, nil + } + closed := c.closed + c.access.Unlock() + if closed { + return goUDPDatagram{}, c.closeError() + } + select { + case <-c.readSignal: + case <-c.closeSignal: + case <-c.readDeadline.Wait(): + return goUDPDatagram{}, os.ErrDeadlineExceeded + } + } +} + +func (c *GoUDPConn) WaitReadPacket() (*buf.Buffer, M.Socksaddr, error) { + datagram, err := c.waitDatagram() + if err != nil { + return nil, M.Socksaddr{}, err + } + return datagram.buffer, M.SocksaddrFromNetIP(datagram.source), nil +} + +func (c *GoUDPConn) ReadPacket(buffer *buf.Buffer) (M.Socksaddr, error) { + datagram, err := c.waitDatagram() + if err != nil { + return M.Socksaddr{}, err + } + defer datagram.buffer.Release() + if buffer.FreeLen() < datagram.buffer.Len() { + return M.SocksaddrFromNetIP(datagram.source), io.ErrShortBuffer + } + buffer.Write(datagram.buffer.Bytes()) + return M.SocksaddrFromNetIP(datagram.source), nil +} + +func (c *GoUDPConn) ReadFromUDPAddrPort(p []byte) (int, netip.AddrPort, error) { + datagram, err := c.waitDatagram() + if err != nil { + return 0, netip.AddrPort{}, err + } + defer datagram.buffer.Release() + return copy(p, datagram.buffer.Bytes()), datagram.source, nil +} + +func (c *GoUDPConn) ReadFrom(p []byte) (int, net.Addr, error) { + n, source, err := c.ReadFromUDPAddrPort(p) + if err != nil { + return 0, nil, err + } + return n, net.UDPAddrFromAddrPort(source), nil +} + +func (c *GoUDPConn) Read(p []byte) (int, error) { + n, _, err := c.ReadFromUDPAddrPort(p) + return n, err +} + +func (c *GoUDPConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + defer buffer.Release() + if !destination.IsIP() { + return E.Cause(os.ErrInvalid, "go: invalid packet destination") + } + return c.transmit(buffer.Bytes(), destination.AddrPort()) +} + +func (c *GoUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) { + destination := M.SocksaddrFromNet(addr) + if !destination.IsIP() { + return 0, E.Cause(os.ErrInvalid, "go: invalid packet destination") + } + return c.WriteToUDPAddrPort(p, destination.AddrPort()) +} + +func (c *GoUDPConn) WriteToUDPAddrPort(p []byte, addr netip.AddrPort) (int, error) { + err := c.transmit(p, addr) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (c *GoUDPConn) Write(p []byte) (int, error) { + if !c.connected { + return 0, E.Cause(syscall.EDESTADDRREQ, "go: socket not connected") + } + return c.WriteToUDPAddrPort(p, c.remote) +} + +func (c *GoUDPConn) transmit(payload []byte, destination netip.AddrPort) error { + c.access.Lock() + closed := c.closed + c.access.Unlock() + if closed { + return c.closeError() + } + select { + case <-c.writeDeadline.Wait(): + return os.ErrDeadlineExceeded + default: + } + remote := netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !remote.IsValid() || remote.Addr().Is4() != c.local.Addr().Is4() || remote.Port() == 0 { + return E.Cause(syscall.EAFNOSUPPORT, "go: packet destination ", remote) + } + headerLength := goNetworkHeaderLength(c.ipVersion) + header.UDPMinimumSize + maximumPayload := 65535 - header.UDPMinimumSize + if c.ipVersion == 4 { + maximumPayload -= header.IPv4MinimumSize + } + if len(payload) > maximumPayload { + return E.Cause(syscall.EMSGSIZE, "go: packet payload ", len(payload)) + } + packet := buf.NewSize(headerLength + len(payload)) + defer packet.Release() + goEncodeUDPPacket(packet.Extend(headerLength+len(payload)), c.ipVersion, c.local, remote, payload, uint16(c.engine.stack.fragmentIdentification.Add(1))) + platformIO := c.engine.platformIO + mtu := platformIO.mtu() + if packet.Len() <= mtu { + return goIgnoreDropped(platformIO.writeDatagram(packet.Bytes(), ForwardFrameMeta{})) + } + return goWriteFragmented(platformIO, &c.engine.stack.fragmentIdentification, packet.Bytes(), mtu) +} + +func goEncodeUDPPacket(packet []byte, ipVersion uint8, source netip.AddrPort, destination netip.AddrPort, payload []byte, ident uint16) { + udpLength := uint16(header.UDPMinimumSize + len(payload)) + pseudoSum := goEncodeNetworkHeader(packet, ipVersion, header.UDPProtocolNumber, source.Addr(), destination.Addr(), int(udpLength), ident) + udpHdr := header.UDP(packet[goNetworkHeaderLength(ipVersion):]) + udpHdr.Encode(&header.UDPFields{ + SrcPort: source.Port(), + DstPort: destination.Port(), + Length: udpLength, + }) + copy(udpHdr[header.UDPMinimumSize:], payload) + sum := ^checksum.Checksum(payload, udpHdr.CalculateChecksum(pseudoSum)) + if sum == 0 { + sum = 0xffff + } + udpHdr.SetChecksum(sum) +} + +func (c *GoUDPConn) markClosed(err error) bool { + c.access.Lock() + if c.closed { + c.access.Unlock() + return false + } + c.closed = true + if c.openErr == nil { + c.openErr = err + } + c.access.Unlock() + close(c.closeSignal) + return true +} + +func (c *GoUDPConn) closeError() error { + c.access.Lock() + defer c.access.Unlock() + if c.openErr != nil { + return c.openErr + } + return net.ErrClosed +} + +func (c *GoUDPConn) drain() { + c.access.Lock() + for _, datagram := range c.queue[c.queueHead:] { + datagram.buffer.Release() + } + clear(c.queue) + c.queue = c.queue[:0] + c.queueHead = 0 + c.queuedBytes = 0 + c.access.Unlock() +} + +func (c *GoUDPConn) Close() error { + if c.markClosed(net.ErrClosed) { + c.engine.postMessage(&c.closeMessage) + } + select { + case <-c.closeDone: + case <-c.engine.exitSignal: + } + return nil +} + +func (c *GoUDPConn) LocalAddr() net.Addr { + return net.UDPAddrFromAddrPort(c.local) +} + +func (c *GoUDPConn) RemoteAddr() net.Addr { + if !c.connected { + return nil + } + return net.UDPAddrFromAddrPort(c.remote) +} + +func (c *GoUDPConn) SetDeadline(t time.Time) error { + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil +} + +func (c *GoUDPConn) SetReadDeadline(t time.Time) error { + c.readDeadline.Set(t) + return nil +} + +func (c *GoUDPConn) SetReadBuffer(size int) error { + c.access.Lock() + c.receiveCapacity = min(max(size, goUDPReceiveMinimum), goReceiveCapacityMax) + c.access.Unlock() + return nil +} + +func (c *GoUDPConn) SetWriteBuffer(size int) error { + return nil +} + +func (c *GoUDPConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline.Set(t) + return nil +} + +var ( + _ N.NetPacketConn = (*GoUDPConn)(nil) + _ N.PacketReadWaiter = (*GoUDPConn)(nil) +) diff --git a/stack_go_udp_test.go b/stack_go_udp_test.go new file mode 100644 index 00000000..7f03fbc3 --- /dev/null +++ b/stack_go_udp_test.go @@ -0,0 +1,183 @@ +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/netip" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/logger" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +type goUDPImmediateCloseHandler struct { + Handler + finished chan struct{} + tracker *goUDPCloseTracker +} + +func (h *goUDPImmediateCloseHandler) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionFlow, Port: goUDPUnavailablePort{}, NewTracker: func() FlowTracker { + select { + case <-h.finished: + case <-time.After(50 * time.Millisecond): + } + return h.tracker + }} +} + +func (h *goUDPImmediateCloseHandler) NewPacketConnectionEx(_ context.Context, conn N.PacketConn, _, _ M.Socksaddr, onClose N.CloseHandlerFunc) { + conn.Close() + onClose(nil) + close(h.finished) +} + +type goUDPUnavailablePort struct{ Port } + +func (goUDPUnavailablePort) PortAddresses() (netip.Addr, netip.Addr) { + return netip.Addr{}, netip.Addr{} +} + +type goUDPCloseTracker struct { + attached atomic.Bool + closed chan bool +} + +func (t *goUDPCloseTracker) AttachFlow(FlowHandle) { t.attached.Store(true) } +func (*goUDPCloseTracker) CountForward(int) {} +func (*goUDPCloseTracker) CountReverse(int) {} +func (*goUDPCloseTracker) FlowEstablished() {} +func (t *goUDPCloseTracker) CloseFlow(FlowCloseReason) { + t.closed <- t.attached.Load() +} + +func TestGoUDPSessionImmediateClose(t *testing.T) { + tracker := &goUDPCloseTracker{closed: make(chan bool, 1)} + handler := &goUDPImmediateCloseHandler{finished: make(chan struct{}), tracker: tracker} + device := NewMemoryTun(MemoryTunOptions{MTU: 1500}) + t.Cleanup(func() { device.Close() }) + stack, err := NewGo(StackOptions{ + Context: context.Background(), Tun: device, TunOptions: Options{MTU: 1500}, + Handler: handler, Logger: logger.NOP(), UDPTimeout: time.Minute, + }) + if err != nil { + t.Fatal(err) + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(packet, 4, netip.MustParseAddrPort("198.18.0.2:12345"), netip.MustParseAddrPort("203.0.113.1:443"), []byte{1}, 1) + _, err = device.Write(packet) + if err != nil { + t.Fatal(err) + } + select { + case attached := <-tracker.closed: + if !attached { + t.Fatal("packet session closed before its tracker was attached") + } + case <-time.After(time.Second): + t.Fatal("packet session finished without closing its tracker") + } + select { + case <-handler.finished: + case <-time.After(time.Second): + t.Fatal("packet handler did not finish") + } +} + +func TestGoUDPReceiveBudget(t *testing.T) { + for _, payloadSize := range []int{0, 1} { + t.Run(fmt.Sprintf("payload=%d", payloadSize), func(t *testing.T) { + device := NewMemoryTun(MemoryTunOptions{MTU: 1500}) + t.Cleanup(func() { device.Close() }) + stack, err := NewGo(StackOptions{Context: context.Background(), Tun: device, TunOptions: Options{MTU: 1500}, Logger: logger.NOP()}) + if err != nil { + t.Fatal(err) + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + local := netip.MustParseAddrPort("198.18.0.2:12345") + remote := netip.MustParseAddrPort("203.0.113.1:443") + socket, err := stack.ListenUDP(local) + if err != nil { + t.Fatal(err) + } + defer socket.Close() + socket.SetReadBuffer(1) + barrier, err := stack.ListenUDP(netip.AddrPortFrom(local.Addr(), local.Port()+1)) + if err != nil { + t.Fatal(err) + } + defer barrier.Close() + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+payloadSize) + goEncodeUDPPacket(packet, 4, remote, local, make([]byte, payloadSize), 1) + const sent = 200 + packets := make([][]byte, sent+1) + for index := range sent { + packets[index] = packet + } + barrierPacket := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(barrierPacket, 4, remote, netip.AddrPortFrom(local.Addr(), local.Port()+1), []byte{1}, 1) + packets[sent] = barrierPacket + accepted, err := device.WritePackets(packets) + if err != nil || accepted != len(packets) { + t.Fatalf("inject receive burst: accepted %d/%d: %v", accepted, len(packets), err) + } + barrier.SetReadDeadline(time.Now().Add(time.Second)) + _, err = barrier.Read(make([]byte, 1)) + if err != nil { + t.Fatal("receive burst did not finish:", err) + } + socket.SetReadDeadline(time.Now().Add(20 * time.Millisecond)) + received := 0 + for { + buffer, source, readErr := socket.WaitReadPacket() + if errors.Is(readErr, os.ErrDeadlineExceeded) { + break + } + if readErr != nil { + t.Fatal(readErr) + } + valid := buffer.Len() == payloadSize && source.AddrPort() == remote + buffer.Release() + if !valid { + t.Fatal("receive burst payload or source changed") + } + received++ + } + if received == 0 || received >= sent { + t.Fatalf("receive buffer retained %d/%d datagrams despite its small budget", received, sent) + } + socket.SetReadDeadline(time.Now().Add(time.Second)) + payload := []byte("capacity restored after reading") + packet = make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+len(payload)) + goEncodeUDPPacket(packet, 4, remote, local, payload, 2) + _, err = device.Write(packet) + if err != nil { + t.Fatal(err) + } + buffer, source, err := socket.WaitReadPacket() + if err != nil { + t.Fatal("receive after draining:", err) + } + defer buffer.Release() + if !bytes.Equal(buffer.Bytes(), payload) || source.AddrPort() != remote { + t.Fatal("receive after draining changed the datagram") + } + }) + } +} diff --git a/stack_gvisor.go b/stack_gvisor.go index c226d05d..5d4f3ac3 100644 --- a/stack_gvisor.go +++ b/stack_gvisor.go @@ -81,12 +81,11 @@ func NewGVisor( inet6LoopbackAddress: options.TunOptions.Inet6LoopbackAddress, icmpTimeout: options.ICMPTimeout, udpNATOptions: UDPNatOptions{ - Timeout: options.UDPTimeout, - Mapping: options.UDPMapping, - Filtering: options.UDPFiltering, - MaxSize: options.UDPNATMax, - InterfaceFinder: options.InterfaceFinder, - ExcludeInterface: []string{options.TunOptions.Name}, + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, }, broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), handler: options.Handler, diff --git a/stack_gvisor_filter.go b/stack_gvisor_filter.go index 96a245f5..3cf9e014 100644 --- a/stack_gvisor_filter.go +++ b/stack_gvisor_filter.go @@ -10,8 +10,6 @@ import ( "github.com/sagernet/gvisor/pkg/tcpip/stack" ) -var _ stack.LinkEndpoint = (*LinkEndpointFilter)(nil) - type LinkEndpointFilter struct { stack.LinkEndpoint BroadcastAddress netip.Addr @@ -40,8 +38,6 @@ func (w *LinkEndpointFilter) Attach(dispatcher stack.NetworkDispatcher) { }) } -var _ stack.NetworkDispatcher = (*networkDispatcherFilter)(nil) - type networkDispatcherFilter struct { stack.NetworkDispatcher broadcastAddress netip.Addr diff --git a/stack_gvisor_tcp_conn.go b/stack_gvisor_tcp_conn.go index ad48d42f..ff3db0b4 100644 --- a/stack_gvisor_tcp_conn.go +++ b/stack_gvisor_tcp_conn.go @@ -18,10 +18,7 @@ import ( N "github.com/sagernet/sing/common/network" ) -var ( - _ net.Conn = (*gTCPConn)(nil) - _ N.ReadWaiter = (*gTCPConn)(nil) -) +var _ N.ReadWaiter = (*gTCPConn)(nil) type gTCPConn struct { gTCPDeadline diff --git a/stack_system.go b/stack_system.go index ad87433f..8a01174a 100644 --- a/stack_system.go +++ b/stack_system.go @@ -84,12 +84,11 @@ func NewSystem(options StackOptions) (Stack, error) { inet6Prefixes: options.TunOptions.Inet6Address, broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), udpNATOptions: UDPNatOptions{ - Timeout: options.UDPTimeout, - Mapping: options.UDPMapping, - Filtering: options.UDPFiltering, - MaxSize: options.UDPNATMax, - InterfaceFinder: options.InterfaceFinder, - ExcludeInterface: []string{options.TunOptions.Name}, + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, }, bindInterface: options.ForwarderBindInterface, interfaceFinder: options.InterfaceFinder, diff --git a/tun_linux_gvisor.go b/tun_linux_gvisor.go index 48e142a9..43b58bb1 100644 --- a/tun_linux_gvisor.go +++ b/tun_linux_gvisor.go @@ -166,5 +166,3 @@ func (e *linuxTUNEndpoint) dispatchLoop() { } } } - -var _ stack.LinkEndpoint = (*linuxTUNEndpoint)(nil) diff --git a/tun_memory.go b/tun_memory.go new file mode 100644 index 00000000..1ba12c45 --- /dev/null +++ b/tun_memory.go @@ -0,0 +1,290 @@ +package tun + +import ( + "os" + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common/buf" +) + +const ( + memoryTunDefaultBatchSize = 128 + memoryTunMaxPacketSize = 65535 +) + +type MemoryTunOptions struct { + MTU int + Headroom int + RearSpace int + BatchSize int + Outbound func(packets []*buf.Buffer) error +} + +type MemoryTun struct { + mtu atomic.Int32 + headroom int + rearSpace int + batchSize int + outboundHandler func(packets []*buf.Buffer) error + attached atomic.Bool + closed atomic.Bool + inboundAccess sync.Mutex + inbound goPacketRing + inboundHeadroom atomic.Int32 + inboundRearSpace atomic.Int32 + outboundAccess sync.Mutex + control goPacketRing + datagram goPacketRing + data goPacketRing + dataTurn bool + dataQueued atomic.Int32 + transmitInterest bool + transmitWritable bool + engineWake goSignal + readerWake goSignal + closeSignal chan struct{} +} + +func NewMemoryTun(options MemoryTunOptions) *MemoryTun { + batchSize := options.BatchSize + if batchSize <= 0 { + batchSize = memoryTunDefaultBatchSize + } + memoryTun := &MemoryTun{ + headroom: options.Headroom, + rearSpace: options.RearSpace, + batchSize: batchSize, + outboundHandler: options.Outbound, + inbound: newGoPacketRing(4 * batchSize), + control: newGoPacketRing(4 * batchSize), + datagram: newGoPacketRing(4 * batchSize), + data: newGoPacketRing(2 * batchSize), + engineWake: make(goSignal, 1), + readerWake: make(goSignal, 1), + closeSignal: make(chan struct{}), + } + memoryTun.mtu.Store(int32(options.MTU)) + return memoryTun +} + +func (t *MemoryTun) Name() (string, error) { + return "memory", nil +} + +func (t *MemoryTun) Start() error { + return nil +} + +func (t *MemoryTun) UpdateRouteOptions(tunOptions Options) error { + return nil +} + +func (t *MemoryTun) MTU() int { + return int(t.mtu.Load()) +} + +func (t *MemoryTun) UpdateMTU(mtu int) { + t.mtu.Store(int32(mtu)) +} + +func (t *MemoryTun) Read(p []byte) (int, error) { + buffers := [1][]byte{p} + var sizes [1]int + _, err := t.ReadPackets(buffers[:], sizes[:], 0) + if err != nil { + return 0, err + } + return sizes[0], nil +} + +func (t *MemoryTun) Write(p []byte) (int, error) { + packets := [1][]byte{p} + _, err := t.WritePackets(packets[:]) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (t *MemoryTun) WritePackets(packets [][]byte) (int, error) { + t.inboundAccess.Lock() + if t.closed.Load() { + t.inboundAccess.Unlock() + return 0, os.ErrClosed + } + if !t.attached.Load() { + t.inboundAccess.Unlock() + return 0, nil + } + headroom := int(t.inboundHeadroom.Load()) + rearSpace := int(t.inboundRearSpace.Load()) + wasEmpty := t.inbound.empty() + accepted := 0 + for _, packet := range packets { + if len(packet) == 0 || len(packet) > memoryTunMaxPacketSize { + continue + } + if t.inbound.full() { + continue + } + buffer := buf.NewSize(headroom + len(packet) + rearSpace) + buffer.Resize(headroom, 0) + buffer.Write(packet) + t.inbound.push(buffer, nil, 0) + accepted++ + } + t.inboundAccess.Unlock() + if wasEmpty && accepted > 0 { + t.engineWake.notify() + } + return accepted, nil +} + +func (t *MemoryTun) ReadPackets(buffers [][]byte, sizes []int, offset int) (int, error) { + if len(buffers) == 0 { + return 0, nil + } + if t.outboundHandler != nil { + return 0, os.ErrInvalid + } + for { + t.outboundAccess.Lock() + count := 0 + for count < len(buffers) { + buffer, owner, segmentEnd := t.popOutboundLocked() + if buffer == nil { + break + } + packet := buffer.Bytes() + if owner != nil { + owner.frameDequeued(len(packet), segmentEnd) + } + target := buffers[count][offset:] + if len(packet) > len(target) { + buffer.Release() + continue + } + sizes[count] = copy(target, packet) + buffer.Release() + count++ + } + wakeEngine := t.takeTransmitWritableLocked() + t.outboundAccess.Unlock() + if wakeEngine { + t.engineWake.notify() + } + if count > 0 { + return count, nil + } + if t.closed.Load() { + return 0, os.ErrClosed + } + select { + case <-t.readerWake: + case <-t.closeSignal: + } + } +} + +func (t *MemoryTun) popOutboundLocked() (*buf.Buffer, *GoConn, uint64) { + if !t.control.empty() { + return t.control.pop() + } + if !t.datagram.empty() && (t.data.empty() || !t.dataTurn) { + t.dataTurn = true + return t.datagram.pop() + } + if !t.data.empty() { + t.dataTurn = false + t.dataQueued.Add(-1) + return t.data.pop() + } + return nil, nil, 0 +} + +func (t *MemoryTun) takeTransmitWritableLocked() bool { + if !t.transmitInterest || t.data.full() { + return false + } + t.transmitInterest = false + t.transmitWritable = true + return true +} + +func (t *MemoryTun) Close() error { + if t.closed.Swap(true) { + return nil + } + t.inboundAccess.Lock() + t.inbound.releaseAll() + t.inboundAccess.Unlock() + t.outboundAccess.Lock() + t.control.releaseAll() + t.datagram.releaseAll() + t.data.releaseAll() + t.dataQueued.Store(0) + t.outboundAccess.Unlock() + close(t.closeSignal) + return nil +} + +func (t *MemoryTun) newOutboundPacket(length int) *buf.Buffer { + packet := buf.NewSize(t.headroom + length + t.rearSpace) + packet.Resize(t.headroom, 0) + return packet +} + +type goPacketRing struct { + packets []*buf.Buffer + owners []*GoConn + segmentEnds []uint64 + head int + count int +} + +func newGoPacketRing(capacity int) goPacketRing { + return goPacketRing{ + packets: make([]*buf.Buffer, capacity), + owners: make([]*GoConn, capacity), + segmentEnds: make([]uint64, capacity), + } +} + +func (r *goPacketRing) empty() bool { + return r.count == 0 +} + +func (r *goPacketRing) full() bool { + return r.count == len(r.packets) +} + +func (r *goPacketRing) push(packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + index := (r.head + r.count) % len(r.packets) + r.packets[index] = packet + r.owners[index] = owner + r.segmentEnds[index] = segmentEnd + r.count++ +} + +func (r *goPacketRing) pop() (*buf.Buffer, *GoConn, uint64) { + packet := r.packets[r.head] + owner := r.owners[r.head] + segmentEnd := r.segmentEnds[r.head] + r.packets[r.head] = nil + r.owners[r.head] = nil + r.segmentEnds[r.head] = 0 + r.head = (r.head + 1) % len(r.packets) + r.count-- + return packet, owner, segmentEnd +} + +func (r *goPacketRing) releaseAll() { + for !r.empty() { + packet, owner, segmentEnd := r.pop() + if owner != nil { + owner.frameDequeued(packet.Len(), segmentEnd) + } + packet.Release() + } +} diff --git a/tun_windows_gvisor.go b/tun_windows_gvisor.go index 9e68b444..cf9683d2 100644 --- a/tun_windows_gvisor.go +++ b/tun_windows_gvisor.go @@ -21,8 +21,6 @@ func (t *NativeTun) NewEndpoint() (stack.LinkEndpoint, stack.NICOptions, error) return &WintunEndpoint{tun: t}, stack.NICOptions{}, nil } -var _ stack.LinkEndpoint = (*WintunEndpoint)(nil) - type WintunEndpoint struct { tun *NativeTun access sync.RWMutex diff --git a/udp_egress.go b/udp_egress.go index 0ace53e8..df4e46f6 100644 --- a/udp_egress.go +++ b/udp_egress.go @@ -24,7 +24,6 @@ type UDPEgressPoolOptions struct { Control control.Func InterfaceFinder control.InterfaceFinder InterfaceMonitor DefaultInterfaceMonitor - ExcludeInterface string IsExempt func() bool } @@ -34,7 +33,6 @@ type UDPEgressPool struct { control control.Func interfaceFinder control.InterfaceFinder interfaceMonitor DefaultInterfaceMonitor - excludeInterface string isExempt func() bool access sync.Mutex port uint16 @@ -70,7 +68,6 @@ func NewUDPEgressPool(options UDPEgressPoolOptions) *UDPEgressPool { control: options.Control, interfaceFinder: options.InterfaceFinder, interfaceMonitor: options.InterfaceMonitor, - excludeInterface: options.ExcludeInterface, isExempt: options.IsExempt, anchorInterfaceIndex: -1, members: make(map[udpEgressSpec]*udpEgressMember), @@ -182,8 +179,7 @@ func (p *UDPEgressPool) rebuildLocked() { networkInterface.Flags&net.FlagLoopback != 0 || networkInterface.Flags&net.FlagPointToPoint != 0 || networkInterface.Flags&net.FlagBroadcast == 0 || - networkInterface.Index == p.anchorInterfaceIndex || - networkInterface.Name == p.excludeInterface { + networkInterface.Index == p.anchorInterfaceIndex { continue } for _, prefix := range networkInterface.Addresses { diff --git a/udp_nat.go b/udp_nat.go index 32a8b64d..3f19b38d 100644 --- a/udp_nat.go +++ b/udp_nat.go @@ -14,6 +14,7 @@ import ( "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" "github.com/sagernet/sing/common/canceler" "github.com/sagernet/sing/common/control" M "github.com/sagernet/sing/common/metadata" @@ -51,8 +52,7 @@ type UDPNatOptions struct { Filtering NATFiltering MaxSize uint32 - InterfaceFinder control.InterfaceFinder - ExcludeInterface []string + InterfaceFinder control.InterfaceFinder } type udpNatSessionKey struct { @@ -95,7 +95,6 @@ type UDPNat struct { filterCache *freelru.Cache[udpNatFilterKey, *UDPNatConn] nextFilterSessionID atomic.Uint64 interfaceFinder control.InterfaceFinder - excludeInterface []string interfaceElement *list.Element[control.InterfaceUpdateCallback] egress atomic.Pointer[udpNatEgressTable] classAccess sync.Mutex @@ -128,17 +127,16 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { filterCache = common.Must1(freelru.New[udpNatFilterKey, *UDPNatConn](maxSize, filterHasher.Hash32, options.Shared)) } service := &UDPNat{ - handler: options.Handler, - prepare: options.Prepare, - timeout: options.Timeout, - mapping: options.Mapping, - filtering: options.Filtering, - cache: cache, - filterCache: filterCache, - interfaceFinder: options.InterfaceFinder, - excludeInterface: options.ExcludeInterface, - classConns: make(map[uint32]map[*UDPNatConn]struct{}), - cleanupDone: make(chan struct{}), + handler: options.Handler, + prepare: options.Prepare, + timeout: options.Timeout, + mapping: options.Mapping, + filtering: options.Filtering, + cache: cache, + filterCache: filterCache, + interfaceFinder: options.InterfaceFinder, + classConns: make(map[uint32]map[*UDPNatConn]struct{}), + cleanupDone: make(chan struct{}), } service.cleanup = newUDPNatCleanupQueue(service) cache.SetLifetime(options.Timeout) @@ -198,9 +196,6 @@ func (s *UDPNat) updateInterfaces(interfaces []control.Interface) { networkInterface.Flags&net.FlagBroadcast == 0 { continue } - if slices.Contains(s.excludeInterface, networkInterface.Name) { - continue - } for _, prefix := range networkInterface.Addresses { if !prefix.Addr().IsGlobalUnicast() { continue @@ -316,7 +311,7 @@ func (s *UDPNat) unregisterClass(conn *UDPNatConn) { } func (s *UDPNat) NewPacket(bufferSlices [][]byte, source M.Socksaddr, destination M.Socksaddr, userData any) { - conn, ok := s.getOrCreate(s.sessionKey(source, destination), source, destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(source, destination), source, destination, userData, nil) if !ok { return } @@ -350,7 +345,7 @@ func (s *UDPNat) sessionKey(source M.Socksaddr, destination M.Socksaddr) udpNatS return key } -func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destination M.Socksaddr, userData any) (*UDPNatConn, bool) { +func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destination M.Socksaddr, userData any, initialize func(*UDPNatConn)) (*UDPNatConn, bool) { if s.state.Load() != udpNatStateStarted { return nil, false } @@ -364,13 +359,14 @@ func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destinati return nil, false } newConn := &UDPNatConn{ - service: s, - key: key, - writer: writer, - localAddr: source, - packetChan: make(chan *N.PacketBuffer, 64), - doneChan: make(chan struct{}), - readDeadline: pipe.MakeDeadline(), + service: s, + key: key, + writer: writer, + localAddr: source, + packetChan: make(chan *N.PacketBuffer, 64), + doneChan: make(chan struct{}), + readDeadline: pipe.MakeDeadline(), + writeDeadline: pipe.MakeDeadline(), } newConn.cleanupEntry = &udpNatCleanupEntry{ conn: newConn, @@ -411,7 +407,13 @@ func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destinati } if !loaded { s.cleanup.addOrUpdate(conn.cleanupEntry, time.Now().Add(s.timeout)) + if initialize != nil { + initialize(conn) + } if conn.isClosed() { + if newOnClose != nil { + newOnClose(io.ErrClosedPipe) + } return nil, false } go s.handler.NewPacketConnectionEx(newContext, conn, source, destination, newOnClose) @@ -449,7 +451,7 @@ func (s *UDPNat) NewPacketBatch(buffers []*buf.Buffer, sources []M.Socksaddr, de return } for index, buffer := range buffers { - conn, ok := s.getOrCreate(s.sessionKey(sources[index], destination), sources[index], destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(sources[index], destination), sources[index], destination, userData, nil) if !ok { buffer.Release() continue @@ -478,7 +480,6 @@ func (s *UDPNat) PurgeExpired() { } var ( - _ N.PacketConn = (*UDPNatConn)(nil) _ canceler.PacketConn = (*UDPNatConn)(nil) _ N.PacketBatchReadWaitCreator = (*UDPNatConn)(nil) _ N.PacketBatchWriteCreator = (*UDPNatConn)(nil) @@ -495,6 +496,7 @@ type UDPNatConn struct { closeOnce sync.Once doneChan chan struct{} readDeadline pipe.Deadline + writeDeadline pipe.Deadline readWaitOptions atomic.Pointer[N.ReadWaitOptions] readBatch *udpNatReadBatch cleanupEntry *udpNatCleanupEntry @@ -595,6 +597,11 @@ func (c *UDPNatConn) allowPeer(destination M.Socksaddr) bool { } func (c *UDPNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error) { + select { + case <-c.readDeadline.Wait(): + return M.Socksaddr{}, os.ErrDeadlineExceeded + default: + } select { case p := <-c.packetChan: if buffer.IsFull() { @@ -614,6 +621,11 @@ func (c *UDPNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error } func (c *UDPNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + err := c.writeError() + if err != nil { + buffer.Release() + return err + } if !c.allowPeer(destination) { buffer.Release() return nil @@ -625,13 +637,41 @@ func (c *UDPNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { if c.service.filtering != NATFilteringEndpointIndependent { return nil, false } - if creator, isCreator := c.writer.(N.PacketBatchWriteCreator); isCreator { - return creator.CreatePacketBatchWriter() + writer, created := bufio.CreatePacketBatchWriter(c.writer) + if !created { + return nil, false } - if writer, isWriter := c.writer.(N.PacketBatchWriter); isWriter { - return writer, true + return &udpNatPacketBatchWriter{conn: c, writer: writer}, true +} + +type udpNatPacketBatchWriter struct { + conn *UDPNatConn + writer N.PacketBatchWriter +} + +func (w *udpNatPacketBatchWriter) WritePacketBatch(buffers []*buf.Buffer, destinations []M.Socksaddr) error { + err := w.conn.writeError() + if err != nil { + buf.ReleaseMulti(buffers) + return err + } + return w.writer.WritePacketBatch(buffers, destinations) +} + +func (w *udpNatPacketBatchWriter) Upstream() any { + return w.conn +} + +func (c *UDPNatConn) writeError() error { + if c.isClosed() { + return os.ErrClosed + } + select { + case <-c.writeDeadline.Wait(): + return os.ErrDeadlineExceeded + default: + return nil } - return nil, false } func (c *UDPNatConn) InitializeReadWaiter(options N.ReadWaitOptions) (needCopy bool) { @@ -644,6 +684,11 @@ func (c *UDPNatConn) WaitReadPacket() (buffer *buf.Buffer, destination M.Socksad } func (c *UDPNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buffer, destination M.Socksaddr, err error) { + select { + case <-c.readDeadline.Wait(): + return nil, M.Socksaddr{}, os.ErrDeadlineExceeded + default: + } select { case packet := <-c.packetChan: buffer = options.Copy(packet.Buffer) @@ -779,16 +824,28 @@ func (c *UDPNatConn) RemoteAddr() net.Addr { } func (c *UDPNatConn) SetDeadline(t time.Time) error { - return os.ErrInvalid + if c.isClosed() { + return io.ErrClosedPipe + } + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil } func (c *UDPNatConn) SetReadDeadline(t time.Time) error { + if c.isClosed() { + return io.ErrClosedPipe + } c.readDeadline.Set(t) return nil } func (c *UDPNatConn) SetWriteDeadline(t time.Time) error { - return os.ErrInvalid + if c.isClosed() { + return io.ErrClosedPipe + } + c.writeDeadline.Set(t) + return nil } func (c *UDPNatConn) Upstream() any { From 1b5507070ed7298f5292acbe331e97addbce2e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Thu, 17 Sep 2026 11:06:58 +0800 Subject: [PATCH 12/25] Fix iptables DNS hijack skipped by input connmark --- redirect_iptables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redirect_iptables.go b/redirect_iptables.go index df985065..93f85478 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -299,7 +299,7 @@ func (r *autoRedirect) iptablesAddExcludeRules(builder *iptablesBuilder, hook ip } if kind == iptablesKindNAT { builder.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") - builder.add("-m", "connmark", "--mark", iptablesMark(options.AutoRedirectInputMark, mask), "-j", "RETURN") + builder.add("-p", "tcp", "-m", "connmark", "--mark", iptablesMark(options.AutoRedirectInputMark, mask), "-j", "RETURN") } if hook == iptablesHookPrerouting { builder.add("-i", options.Name, "-j", "RETURN") From 8ce6d6dd2e9a9508085177bd8c9acf08beca055a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Thu, 17 Sep 2026 22:28:47 +0800 Subject: [PATCH 13/25] Refactor go stack memory --- flow_dispatch.go | 13 +++ go.mod | 2 +- go.sum | 4 +- stack_go_conn.go | 23 +++- stack_go_conn_options.go | 3 + stack_go_engine.go | 74 ++++++------ stack_go_io_darwin.go | 27 ++--- stack_go_io_linux.go | 46 +++----- stack_go_io_windows.go | 47 +++----- stack_go_memory.go | 113 +++++++++++++++++++ stack_go_memory_arena.go | 218 ++++++++++++++++++++++++++++++++++++ stack_go_memory_darwin.go | 17 +++ stack_go_memory_linux.go | 26 +++++ stack_go_memory_other.go | 24 ++++ stack_go_profile_default.go | 1 - stack_go_profile_ios.go | 1 - stack_go_ring.go | 96 +++------------- stack_go_slab_other.go | 10 -- stack_go_slab_unix.go | 23 ---- stack_go_splice.go | 10 +- stack_go_tcp_ack.go | 2 +- stack_go_tcp_input.go | 138 ++++++++++++++--------- stack_go_tcp_output.go | 13 ++- stack_go_udp.go | 15 +-- stack_go_wheel.go | 72 +++++++++--- 25 files changed, 691 insertions(+), 327 deletions(-) create mode 100644 stack_go_memory.go create mode 100644 stack_go_memory_arena.go create mode 100644 stack_go_memory_darwin.go create mode 100644 stack_go_memory_linux.go create mode 100644 stack_go_memory_other.go delete mode 100644 stack_go_slab_other.go delete mode 100644 stack_go_slab_unix.go diff --git a/flow_dispatch.go b/flow_dispatch.go index cf321fbc..fdc61974 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -762,6 +762,19 @@ func (s *ForwardStage) Flush() { } } +func (s *ForwardStage) sweepDue() (time.Duration, bool) { + if s == nil { + return 0, false + } + s.access.Lock() + count := len(s.table) + s.access.Unlock() + if count == 0 { + return 0, false + } + return max(time.Duration(s.lastSweep+int64(flowSweepInterval)-s.dispatcher.now()), 0), true +} + func entryExpired(entry *flowEntry, now int64) bool { if now <= entry.deadline { return false diff --git a/go.mod b/go.mod index 56b24c84..deffaaf4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf github.com/sagernet/nftables v0.3.0-mod.4 - github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd + github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0 github.com/stretchr/testify v1.11.1 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc diff --git a/go.sum b/go.sum index 40a12c3a..d644c5de 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf h1:b/3rm+KvxzuYRc github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= github.com/sagernet/nftables v0.3.0-mod.4 h1:vnOtcDYeSXv2e5RoRuGH0lrpttQFJ8iC4ICS2nhlDSo= github.com/sagernet/nftables v0.3.0-mod.4/go.mod h1:8kslHG4VvYNihcco+i6uxIX7qbT8A56T0y5q7U44ZaQ= -github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd h1:awZQVpsmNdo/EZgEW1nwj9sMQbtgfD130oFIH/7fIDM= -github.com/sagernet/sing v0.9.4-0.20260912053229-7776850263cd/go.mod h1:K3Owt3xPhHugvlnlPPxZJ/exXdaJfEPOTNorGk4AXjo= +github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0 h1:rn2MLe/qiqCxvRIHfT9WRc22eNLBTRCKlY39pZTWRdg= +github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0/go.mod h1:K3Owt3xPhHugvlnlPPxZJ/exXdaJfEPOTNorGk4AXjo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= diff --git a/stack_go_conn.go b/stack_go_conn.go index d6f29143..1a09fbcc 100644 --- a/stack_go_conn.go +++ b/stack_go_conn.go @@ -92,7 +92,6 @@ type GoConn struct { ackCovered uint64 lastAckSent uint64 lastActivity int64 - sweepSentTail uint64 windowLeft1 int64 windowLeft2 int64 keepaliveProbes uint8 @@ -108,6 +107,7 @@ type GoConn struct { idleDeadline int64 pacingDeadline int64 reorderDeadline int64 + keepaliveDeadline int64 retransmitAttempts uint8 probeAttempts uint8 handshakeAttempts uint8 @@ -292,6 +292,8 @@ type GoConn struct { pacingMessage goMessage throttleMessage goMessage spliceMessage goMessage + keepaliveMessage goMessage + transmittedMessage goMessage slabHolder goSlabHolder } @@ -332,6 +334,15 @@ func (c *GoConn) initialize(engine *goEngine, key flowKey, peer M.Socksaddr, loc c.pacingMessage = goMessage{kind: goMessageConnPacing, conn: c} c.throttleMessage = goMessage{kind: goMessageConnThrottle, conn: c} c.spliceMessage = goMessage{kind: goMessageConnSplice, conn: c} + c.keepaliveMessage = goMessage{kind: goMessageConnKeepalive, conn: c} + c.transmittedMessage = goMessage{kind: goMessageConnTransmitted, conn: c} +} + +func (c *GoConn) transmitted(tail uint64) { + c.transmittedTail.Store(tail) + if c.sendReleased.Load() < min(c.sendUnacked.Load(), tail) { + c.engine.postMessage(&c.transmittedMessage) + } } func (c *GoConn) handshaking() bool { @@ -611,7 +622,15 @@ func (c *GoConn) copyReceived(target []byte, consumed uint64, available uint64) } func (c *GoConn) advanceConsumed(n int) { - if c.consumedTail.Add(uint64(n)) >= c.windowUpdateAt.Load() { + consumed := c.consumedTail.Add(uint64(n)) + if consumed >= c.windowUpdateAt.Load() { + c.engine.postMessage(&c.windowMessage) + return + } + if c.receiveChain.held.Load() == 0 { + return + } + if consumed/goSlabSize != (consumed-uint64(n))/goSlabSize || consumed == c.receiveAvailable.Load() { c.engine.postMessage(&c.windowMessage) } } diff --git a/stack_go_conn_options.go b/stack_go_conn_options.go index 6a2be215..5a45fbb1 100644 --- a/stack_go_conn_options.go +++ b/stack_go_conn_options.go @@ -16,6 +16,7 @@ func (c *GoConn) SetKeepAlive(keepalive bool) error { c.access.Lock() c.keepaliveEnabled = keepalive c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) return nil } @@ -23,6 +24,7 @@ func (c *GoConn) SetKeepAlivePeriod(d time.Duration) error { c.access.Lock() c.setKeepaliveIdle(d) c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) return nil } @@ -43,6 +45,7 @@ func (c *GoConn) SetKeepAliveConfig(config net.KeepAliveConfig) error { c.keepaliveCount = uint8(min(config.Count, goKeepaliveMaxCount)) } c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) return nil } diff --git a/stack_go_engine.go b/stack_go_engine.go index c781aa94..0294dab1 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -44,6 +44,8 @@ const ( goMessageConnPacing goMessageConnThrottle goMessageConnSplice + goMessageConnKeepalive + goMessageConnTransmitted goMessagePacketSplice goMessagePacketClose goMessageUDPOpen @@ -138,8 +140,7 @@ type goEngine struct { resetMessage goMessage delayedAckTickNode goWheelNode reassemblyTickNode goWheelNode - sweepTickNode goWheelNode - reclaimTickNode goWheelNode + dispatchTickNode goWheelNode reassemblyEntries []goReassemblyEntry udpUserData goUDPUserData udpNat *UDPNat @@ -160,7 +161,7 @@ type goEngine struct { spliceDirtyList *GoConn spliceSegments [][]byte spliceIovecs []goIOVector - packetReceiveBuffers [goPacketBatchSize]*buf.Buffer + packetSlots goReadSlots packetReceiveBatch int packetMessages [goPacketBatchSize]goPacketMessage packetFrames [goPacketBatchSize]goUDPFrame @@ -193,11 +194,10 @@ type goEngine struct { rateSample goRateSample ackSample goAckSample - gsoReadOptions N.ReadWaitOptions - gsoBuffers [goGSOMaxSegments]*buf.Buffer - gsoSegments [goGSOMaxSegments][]byte - gsoSizes [goGSOMaxSegments]int - gsoFrame goFrame + gsoSlots goReadSlots + gsoSegments [goGSOMaxSegments][]byte + gsoSizes [goGSOMaxSegments]int + gsoFrame goFrame } func (e *goEngine) singleFrame(packet []byte) [][]byte { @@ -217,7 +217,6 @@ func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine flows: stack.directory.flows, flowCapacity: max(goFlowCapacity/engineCount, 1024), slabPool: newGoSlabPool(stack.memoryPressure, max(goSlabPoolLowWater/engineCount, 8)), - descriptorPool: goDescriptorPool{lowWater: max(goDescriptorPoolLowWater/engineCount, 4)}, sequenceSeed: maphash.MakeSeed(), controlSegments: make([][]byte, 0, 8), socketEvents: make([]goSocketEvent, goSocketEventBatch), @@ -242,12 +241,9 @@ func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine engine.packetReadOptions.Store(new(N.ReadWaitOptions)) engine.delayedAckTickNode.expire = func(now int64) { engine.drainAckList(now, false) } engine.reassemblyTickNode.expire = engine.expireReassemblyTick - engine.sweepTickNode.expire = engine.expireSweepTick - engine.reclaimTickNode.expire = engine.expireReclaimTick + engine.dispatchTickNode.expire = func(int64) { engine.dispatchStage.Flush() } engine.refreshCoarseTime() engine.wheel.currentTick = engine.coarseTime.Load() / goWheelTick - engine.wheel.schedule(&engine.sweepTickNode, engine.coarseTime.Load()+int64(goSweepInterval)) - engine.wheel.schedule(&engine.reclaimTickNode, engine.coarseTime.Load()+int64(goReclaimInterval)) return engine } @@ -296,6 +292,10 @@ func (e *goEngine) run() { e.drainAckList(e.coarseTime.Load(), true) if e.dispatchStage != nil { e.dispatchStage.Flush() + remaining, pending := e.dispatchStage.sweepDue() + if pending && e.dispatchTickNode.slot == nil { + e.wheel.schedule(&e.dispatchTickNode, e.coarseTime.Load()+int64(remaining)) + } } e.flushPacketUploads() e.flushPacketFrames() @@ -315,6 +315,8 @@ func (e *goEngine) run() { func (e *goEngine) exit() { e.shutdown() e.releaseReadBuffers() + e.packetSlots.release() + e.gsoSlots.release() e.engineState.Store(goEngineExited) e.releaseControlQueue() e.releaseInjected() @@ -406,10 +408,8 @@ func (e *goEngine) releaseReadBuffers() { clear(e.spliceSegments[:cap(e.spliceSegments)]) clear(e.spliceIovecs[:cap(e.spliceIovecs)]) clear(e.controlSegments[:cap(e.controlSegments)]) - buf.ReleaseMulti(e.packetReceiveBuffers[:]) - clear(e.packetReceiveBuffers[:]) - buf.ReleaseMulti(e.gsoBuffers[:]) - clear(e.gsoBuffers[:]) + e.packetSlots.sleep() + e.gsoSlots.sleep() clear(e.gsoSegments[:]) e.gsoFrame = goFrame{} clear(e.packetMessages[:]) @@ -417,13 +417,7 @@ func (e *goEngine) releaseReadBuffers() { clear(e.packetUploads[:]) e.packetIO.reset() e.platformIO.releaseReadBuffers() - for index := range e.reassemblyEntries { - entry := &e.reassemblyEntries[index] - if !entry.active { - entry.buffer.Release() - entry.buffer = nil - } - } + e.slabPool.purge() } func (e *goEngine) postMessage(message *goMessage) { @@ -495,6 +489,10 @@ func (e *goEngine) handleMessage(message *goMessage) { } case goMessageConnSplice: e.handleSpliceEngage(message.conn) + case goMessageConnKeepalive: + e.handleKeepaliveUpdate(message.conn) + case goMessageConnTransmitted: + e.handleTransmitted(message.conn) case goMessagePacketSplice: e.handlePacketSpliceEngage(message.packet) case goMessagePacketClose: @@ -570,7 +568,7 @@ func (e *goEngine) shutdown() { e.closeAllUDPSockets() e.closeAllListeners() for index := range e.reassemblyEntries { - e.reassemblyEntries[index].active = false + e.reassemblyEntries[index].discard() } e.releaseReadBuffers() unreset := e.closeAllFlows() @@ -846,6 +844,12 @@ type goReassemblyEntry struct { rangeCount int } +func (r *goReassemblyEntry) discard() { + r.active = false + r.buffer.Release() + r.buffer = nil +} + func (e *goEngine) reassembleIPv4(packet []byte, parsed *forwardPacket) { ipHdr := header.IPv4(parsed.network) headerLength := int(ipHdr.HeaderLength()) @@ -887,7 +891,7 @@ func (e *goEngine) reassemblyEntry(ipVersion uint8, protocol uint8, source netip for index := range e.reassemblyEntries { entry := &e.reassemblyEntries[index] if entry.active && entry.deadline <= now { - entry.active = false + entry.discard() } if !entry.active { if free == nil { @@ -939,23 +943,23 @@ func (e *goEngine) reassemblyAdd(entry *goReassemblyEntry, offset int, fragment } end := offset + len(fragment) if end > 65535 { - entry.active = false + entry.discard() return } if !more { if entry.totalLength >= 0 && entry.totalLength != end { - entry.active = false + entry.discard() return } entry.totalLength = end } if entry.totalLength >= 0 && end > entry.totalLength { - entry.active = false + entry.discard() return } copy(entry.buffer.Range(entry.headroom+offset, entry.headroom+end), fragment) if !reassemblyMergeRanges(entry, offset, end) { - entry.active = false + entry.discard() return } if entry.headerSeen && entry.totalLength >= 0 && entry.rangeCount == 1 && @@ -994,10 +998,12 @@ func reassemblyMergeRanges(entry *goReassemblyEntry, start int, end int) bool { func (e *goEngine) completeReassembly(entry *goReassemblyEntry) { entry.active = false + buffer := entry.buffer + entry.buffer = nil + defer buffer.Release() if entry.ipVersion == 4 && entry.headerLength+entry.totalLength > 65535 { return } - buffer := entry.buffer buffer.Resize(entry.headroom-entry.headerLength, entry.headerLength+entry.totalLength) packet := buffer.Bytes() if entry.ipVersion == 4 { @@ -1027,11 +1033,7 @@ func (e *goEngine) expireReassemblyTick(now int64) { continue } if entry.deadline <= now { - entry.active = false - if !e.readBuffersHeld { - entry.buffer.Release() - entry.buffer = nil - } + entry.discard() } else { next = min(next, entry.deadline) } diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go index 31d9e786..3272eccb 100644 --- a/stack_go_io_darwin.go +++ b/stack_go_io_darwin.go @@ -10,7 +10,6 @@ import ( "github.com/sagernet/sing-tun/gtcpip/header" rawfile "github.com/sagernet/sing-tun/internal/rawfile_darwin" - "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" N "github.com/sagernet/sing/common/network" @@ -55,8 +54,7 @@ type goDarwinIO struct { kqueueFd int pollAccess sync.RWMutex transmitAccess *sync.Mutex - receiveBuffers []*buf.Buffer - readWaitOptions N.ReadWaitOptions + receiveSlots goReadSlots receiveIovecs []unix.Iovec messageHeaders []rawfile.MsgHdrX events [goSocketEventBatch + 3]unix.Kevent_t @@ -313,21 +311,12 @@ func (o *goDarwinIO) unregisterSocket(socket *goSocket) { } func (o *goDarwinIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { - if o.receiveBuffers == nil || o.readWaitOptions != options { - o.releaseReadBuffers() - if o.receiveBuffers == nil { - o.receiveBuffers = make([]*buf.Buffer, goReadBatch) - } - o.readWaitOptions = options - } + o.receiveSlots.configure(goReadBatch, o.stack.mtu+PacketOffset+options.FrontHeadroom+options.RearHeadroom) // recvmsg_x walks every submitted msghdr slot, whether or not a packet is pending for it. count := min(o.receiveBatch, len(frames)) + o.receiveSlots.wake(count) for index := range count { - buffer := o.receiveBuffers[index] - if buffer == nil { - buffer = options.NewBufferSize(o.stack.mtu + PacketOffset) - o.receiveBuffers[index] = buffer - } + buffer := o.receiveSlots.slot(index) buffer.Reset() buffer.Resize(options.FrontHeadroom, 0) buffer.Reserve(options.RearHeadroom) @@ -353,10 +342,10 @@ func (o *goDarwinIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int frameCount := 0 for index := range received { dataLen := int(o.messageHeaders[index].DataLen) - if dataLen <= PacketOffset || dataLen > o.receiveBuffers[index].FreeLen() || o.messageHeaders[index].Msg.Flags&unix.MSG_TRUNC != 0 { + buffer := o.receiveSlots.slot(index) + if dataLen <= PacketOffset || dataLen > buffer.FreeLen() || o.messageHeaders[index].Msg.Flags&unix.MSG_TRUNC != 0 { continue } - buffer := o.receiveBuffers[index] buffer.Reset() buffer.Resize(options.FrontHeadroom+PacketOffset, dataLen-PacketOffset) frames[frameCount] = goFrame{buffer: buffer} @@ -368,8 +357,7 @@ func (o *goDarwinIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int func (o *goDarwinIO) releaseReadBuffers() { clear(o.receiveIovecs) clear(o.messageHeaders) - buf.ReleaseMulti(o.receiveBuffers) - clear(o.receiveBuffers) + o.receiveSlots.sleep() o.transmitAccess.Lock() clear(o.transmitIovecs) if o.batchCount == 0 { @@ -803,6 +791,7 @@ func (o *goDarwinIO) close() error { o.batchSpill = nil o.batchCount = 0 o.batchStart = 0 + o.receiveSlots.release() return unix.Close(o.kqueueFd) } diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go index 31819d09..8d23f36e 100644 --- a/stack_go_io_linux.go +++ b/stack_go_io_linux.go @@ -12,7 +12,6 @@ import ( "github.com/sagernet/sing-tun/gtcpip/header" "github.com/sagernet/sing/common" - "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" N "github.com/sagernet/sing/common/network" @@ -35,16 +34,15 @@ type goLinuxDevice struct { } type goLinuxIO struct { - device *goLinuxDevice - tun *NativeTun - tunFd int - ownsFd bool - epollFd int - eventFd int - fdAccess sync.RWMutex - receiveBuffers []*buf.Buffer - readWaitOptions N.ReadWaitOptions - events [goSocketEventBatch + 2]unix.EpollEvent + device *goLinuxDevice + tun *NativeTun + tunFd int + ownsFd bool + epollFd int + eventFd int + fdAccess sync.RWMutex + receiveSlots goReadSlots + events [goSocketEventBatch + 2]unix.EpollEvent } func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { @@ -272,22 +270,14 @@ func goEpollEvents(interest uint8) uint32 { } func (o *goLinuxIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { - if o.receiveBuffers == nil || o.readWaitOptions != options { - o.releaseReadBuffers() - if o.receiveBuffers == nil { - o.receiveBuffers = make([]*buf.Buffer, goPacketBatchSize) - } - o.readWaitOptions = options - } + // Linux hands pre-segmentation TSO aggregates to the TUN fd even with IFF_VNET_HDR off + // (observed on 6.x kernels). + o.receiveSlots.configure(goPacketBatchSize, virtioNetHdrLen+gsoMaxSize+options.FrontHeadroom+options.RearHeadroom) + limit := min(len(frames), goPacketBatchSize) + o.receiveSlots.wake(limit) count := 0 - for count < min(len(frames), len(o.receiveBuffers)) { - buffer := o.receiveBuffers[count] - if buffer == nil { - // Linux hands pre-segmentation TSO aggregates to the TUN fd even with IFF_VNET_HDR off - // (observed on 6.x kernels). - buffer = options.NewBufferSize(virtioNetHdrLen + gsoMaxSize) - o.receiveBuffers[count] = buffer - } + for count < limit { + buffer := o.receiveSlots.slot(count) buffer.Reset() buffer.Resize(options.FrontHeadroom, 0) buffer.Reserve(options.RearHeadroom) @@ -352,8 +342,7 @@ func (o *goLinuxIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, } func (o *goLinuxIO) releaseReadBuffers() { - buf.ReleaseMulti(o.receiveBuffers) - clear(o.receiveBuffers) + o.receiveSlots.sleep() } func (o *goLinuxIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { @@ -519,6 +508,7 @@ func (o *goLinuxIO) close() error { o.device.closing.Store(true) o.fdAccess.Lock() defer o.fdAccess.Unlock() + o.receiveSlots.release() err := E.Errors(unix.Close(o.epollFd), unix.Close(o.eventFd)) if o.ownsFd { err = E.Errors(err, unix.Close(o.tunFd)) diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index df782a78..1f8d1c56 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -8,7 +8,6 @@ import ( "unsafe" "github.com/sagernet/sing-tun/internal/afd" - "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" N "github.com/sagernet/sing/common/network" @@ -26,19 +25,18 @@ const ( const goAFDReadEvents = afd.POLL_RECEIVE | afd.POLL_DISCONNECT | afd.POLL_ABORT | afd.POLL_LOCAL_CLOSE | afd.POLL_CONNECT_FAIL type goWindowsIO struct { - stack *Go - tun *NativeTun - iocp windows.Handle - afd *afd.Device - waitPacket *afd.WaitCompletionPacket - waitArmed bool - bridgeArm windows.Handle - bridgeClose windows.Handle - bridgeDone chan struct{} - entries map[*goAFDEntry]struct{} - completions [goSocketEventBatch + 2]afd.OverlappedEntry - receiveBuffers []*buf.Buffer - readWaitOptions N.ReadWaitOptions + stack *Go + tun *NativeTun + iocp windows.Handle + afd *afd.Device + waitPacket *afd.WaitCompletionPacket + waitArmed bool + bridgeArm windows.Handle + bridgeClose windows.Handle + bridgeDone chan struct{} + entries map[*goAFDEntry]struct{} + completions [goSocketEventBatch + 2]afd.OverlappedEntry + receiveSlots goReadSlots // wintun's read-wait event is auto-reset and only set by the driver when it appends to // the ring; WintunReceivePacket neither re-signals nor resets it (wintun api/session.c: // CreateEventW(&SecurityAttributes, FALSE, FALSE, NULL)). @@ -284,21 +282,12 @@ func (o *goWindowsIO) unregisterSocket(socket *goSocket) { } func (o *goWindowsIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { - if o.receiveBuffers == nil || o.readWaitOptions != options { - o.releaseReadBuffers() - if o.receiveBuffers == nil { - o.receiveBuffers = make([]*buf.Buffer, goReadBatch) - } - o.readWaitOptions = options - } - limit := min(len(frames), len(o.receiveBuffers)) + o.receiveSlots.configure(goReadBatch, o.stack.mtu+options.FrontHeadroom+options.RearHeadroom) + limit := min(len(frames), goReadBatch) + o.receiveSlots.wake(limit) count := 0 for count < limit { - buffer := o.receiveBuffers[count] - if buffer == nil { - buffer = options.NewBufferSize(o.stack.mtu) - o.receiveBuffers[count] = buffer - } + buffer := o.receiveSlots.slot(count) buffer.Reset() buffer.Resize(options.FrontHeadroom, 0) buffer.Reserve(options.RearHeadroom) @@ -324,8 +313,7 @@ func goFatalReadError(err error) bool { } func (o *goWindowsIO) releaseReadBuffers() { - buf.ReleaseMulti(o.receiveBuffers) - clear(o.receiveBuffers) + o.receiveSlots.sleep() } func (o *goWindowsIO) writePacketBatch(frames []goUDPFrame) error { @@ -433,6 +421,7 @@ func (o *goWindowsIO) close() error { if closing { return nil } + o.receiveSlots.release() var err error if o.waitPacket != nil { err = E.Errors(o.waitPacket.Cancel(), o.waitPacket.Close()) diff --git a/stack_go_memory.go b/stack_go_memory.go new file mode 100644 index 00000000..5f361d93 --- /dev/null +++ b/stack_go_memory.go @@ -0,0 +1,113 @@ +package tun + +import ( + "os" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" +) + +var goPageSize = os.Getpagesize() + +type goSlabStore interface { + acquire() *goSlab + release(slab *goSlab, discard bool) + purge() + close() +} + +type goHeapSlabStore struct{} + +func (goHeapSlabStore) acquire() *goSlab { + return (*goSlab)(buf.Get(goSlabSize)) +} + +func (goHeapSlabStore) release(slab *goSlab, discard bool) { + if discard { + return + } + common.Must(buf.Put(slab[:])) +} + +func (goHeapSlabStore) purge() { +} + +func (goHeapSlabStore) close() { +} + +type goReadSlots struct { + buffers []*buf.Buffer + region []byte + stride int + woken int +} + +func (s *goReadSlots) configure(count int, stride int) { + if s.buffers != nil && len(s.buffers) == count && s.stride == stride { + return + } + s.release() + s.buffers = make([]*buf.Buffer, count) + s.stride = stride +} + +func (s *goReadSlots) wake(count int) { + if !goArenaEnabled() { + return + } + if s.region == nil { + s.region = goRegionMap(len(s.buffers) * s.stride) + } + if count <= s.woken { + return + } + start := s.woken * s.stride &^ (goPageSize - 1) + end := min((count*s.stride+goPageSize-1)&^(goPageSize-1), len(s.region)) + goRegionReuse(s.region[start:end]) + s.woken = count +} + +func (s *goReadSlots) slot(index int) *buf.Buffer { + if goArenaEnabled() && index >= s.woken { + s.wake(index + 1) + } + buffer := s.buffers[index] + if buffer != nil { + return buffer + } + if goArenaEnabled() { + buffer = buf.With(s.region[index*s.stride : (index+1)*s.stride]) + } else { + buffer = buf.NewSize(s.stride) + } + s.buffers[index] = buffer + return buffer +} + +func (s *goReadSlots) sleep() { + if !goArenaEnabled() { + buf.ReleaseMulti(s.buffers) + clear(s.buffers) + return + } + if s.woken == 0 { + return + } + end := min((s.woken*s.stride+goPageSize-1)&^(goPageSize-1), len(s.region)) + goRegionPark(s.region[:end]) + s.woken = 0 +} + +func (s *goReadSlots) release() { + if goArenaEnabled() { + if s.region != nil { + goRegionUnmap(s.region) + s.region = nil + } + } else { + buf.ReleaseMulti(s.buffers) + } + s.buffers = nil + s.stride = 0 + s.woken = 0 +} diff --git a/stack_go_memory_arena.go b/stack_go_memory_arena.go new file mode 100644 index 00000000..50d0d101 --- /dev/null +++ b/stack_go_memory_arena.go @@ -0,0 +1,218 @@ +//go:build darwin || linux + +package tun + +import ( + "cmp" + "slices" + "sync" + "unsafe" + + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +const ( + goSlabChunkSlabs = 32 + goSlabChunkSize = goSlabChunkSlabs * goSlabSize +) + +const ( + goSlabFresh uint8 = iota + goSlabLive + goSlabCached + goSlabReusable +) + +func newGoSlabStore(lowWater int) goSlabStore { + if !goArenaEnabled() { + return goHeapSlabStore{} + } + return &goArenaSlabStore{cached: make([]*goSlab, 0, lowWater), lowWater: lowWater} +} + +func goRegionMap(size int) []byte { + size = (size + goPageSize - 1) &^ (goPageSize - 1) + memory, err := unix.Mmap(-1, 0, size, unix.PROT_READ|unix.PROT_WRITE, goRegionMapFlags) + if err != nil { + panic(E.Cause(err, "go: map memory region")) + } + return memory +} + +func goRegionUnmap(memory []byte) { + unix.Munmap(memory) +} + +type goSlabChunk struct { + base uintptr + memory []byte + states [goSlabChunkSlabs]uint8 + live int + cached int + reusable int + fresh int +} + +func (c *goSlabChunk) slab(index int) *goSlab { + return (*goSlab)(unsafe.Pointer(&c.memory[index*goSlabSize])) +} + +func (c *goSlabChunk) span(index int) []byte { + return c.memory[index*goSlabSize : (index+1)*goSlabSize] +} + +func (c *goSlabChunk) idle() bool { + return c.live == 0 && c.cached == 0 +} + +func (c *goSlabChunk) take(state uint8) (*goSlab, bool) { + for index := range c.states { + if c.states[index] != state { + continue + } + switch state { + case goSlabReusable: + c.reusable-- + goRegionReuse(c.span(index)) + case goSlabFresh: + c.fresh-- + } + c.states[index] = goSlabLive + c.live++ + return c.slab(index), true + } + return nil, false +} + +type goArenaSlabStore struct { + access sync.Mutex + chunks []*goSlabChunk + cached []*goSlab + lowWater int + closed bool +} + +func (s *goArenaSlabStore) acquire() *goSlab { + s.access.Lock() + defer s.access.Unlock() + if count := len(s.cached); count > 0 { + slab := s.cached[count-1] + s.cached[count-1] = nil + s.cached = s.cached[:count-1] + chunk, index := s.locate(slab) + chunk.states[index] = goSlabLive + chunk.cached-- + chunk.live++ + return slab + } + for _, chunk := range s.chunks { + if chunk.reusable == 0 { + continue + } + slab, _ := chunk.take(goSlabReusable) + return slab + } + for _, chunk := range s.chunks { + if chunk.fresh == 0 { + continue + } + slab, _ := chunk.take(goSlabFresh) + return slab + } + memory := goRegionMap(goSlabChunkSize) + chunk := &goSlabChunk{base: uintptr(unsafe.Pointer(&memory[0])), memory: memory, fresh: goSlabChunkSlabs} + position, _ := slices.BinarySearchFunc(s.chunks, chunk.base, goCompareChunk) + s.chunks = slices.Insert(s.chunks, position, chunk) + slab, _ := chunk.take(goSlabFresh) + return slab +} + +func goCompareChunk(chunk *goSlabChunk, base uintptr) int { + return cmp.Compare(chunk.base, base) +} + +func (s *goArenaSlabStore) locate(slab *goSlab) (*goSlabChunk, int) { + address := uintptr(unsafe.Pointer(slab)) + position, found := slices.BinarySearchFunc(s.chunks, address, goCompareChunk) + if !found { + position-- + } + chunk := s.chunks[position] + return chunk, int((address - chunk.base) / goSlabSize) +} + +func (s *goArenaSlabStore) release(slab *goSlab, discard bool) { + s.access.Lock() + defer s.access.Unlock() + chunk, index := s.locate(slab) + chunk.live-- + if discard || s.closed { + s.park(chunk, index) + s.trim(chunk) + return + } + chunk.states[index] = goSlabCached + chunk.cached++ + s.cached = append(s.cached, slab) + if len(s.cached) <= s.lowWater { + return + } + oldest := s.cached[0] + remaining := copy(s.cached, s.cached[1:]) + s.cached[remaining] = nil + s.cached = s.cached[:remaining] + oldestChunk, oldestIndex := s.locate(oldest) + oldestChunk.cached-- + s.park(oldestChunk, oldestIndex) + s.trim(oldestChunk) +} + +func (s *goArenaSlabStore) park(chunk *goSlabChunk, index int) { + goRegionPark(chunk.span(index)) + chunk.states[index] = goSlabReusable + chunk.reusable++ +} + +func (s *goArenaSlabStore) trim(chunk *goSlabChunk) { + if !chunk.idle() { + return + } + if !s.closed { + idle := 0 + for _, candidate := range s.chunks { + if candidate.idle() { + idle++ + } + } + if idle <= 1 { + return + } + } + position, _ := slices.BinarySearchFunc(s.chunks, chunk.base, goCompareChunk) + s.chunks = slices.Delete(s.chunks, position, position+1) + goRegionUnmap(chunk.memory) +} + +func (s *goArenaSlabStore) purge() { + s.access.Lock() + defer s.access.Unlock() + for _, slab := range s.cached { + chunk, index := s.locate(slab) + chunk.cached-- + s.park(chunk, index) + } + clear(s.cached) + s.cached = s.cached[:0] + for index := len(s.chunks) - 1; index >= 0; index-- { + s.trim(s.chunks[index]) + } +} + +func (s *goArenaSlabStore) close() { + s.access.Lock() + s.closed = true + s.access.Unlock() + s.purge() +} diff --git a/stack_go_memory_darwin.go b/stack_go_memory_darwin.go new file mode 100644 index 00000000..95cf38e1 --- /dev/null +++ b/stack_go_memory_darwin.go @@ -0,0 +1,17 @@ +package tun + +import "golang.org/x/sys/unix" + +const goRegionMapFlags = unix.MAP_ANON | unix.MAP_PRIVATE + +func goArenaEnabled() bool { + return true +} + +func goRegionPark(memory []byte) { + unix.Madvise(memory, unix.MADV_FREE_REUSABLE) +} + +func goRegionReuse(memory []byte) { + unix.Madvise(memory, unix.MADV_FREE_REUSE) +} diff --git a/stack_go_memory_linux.go b/stack_go_memory_linux.go new file mode 100644 index 00000000..8118386b --- /dev/null +++ b/stack_go_memory_linux.go @@ -0,0 +1,26 @@ +package tun + +import ( + "os" + "sync" + + "github.com/sagernet/sing/common/memory" + + "golang.org/x/sys/unix" +) + +const ( + goLowMemoryLimit = 512 << 20 + goRegionMapFlags = unix.MAP_ANON | unix.MAP_PRIVATE | unix.MAP_NORESERVE +) + +var goArenaEnabled = sync.OnceValue(func() bool { + return goSlabSize%os.Getpagesize() == 0 && memory.LimitAvailable() && memory.Limit() < goLowMemoryLimit +}) + +func goRegionPark(memory []byte) { + unix.Madvise(memory, unix.MADV_DONTNEED) +} + +func goRegionReuse(memory []byte) { +} diff --git a/stack_go_memory_other.go b/stack_go_memory_other.go new file mode 100644 index 00000000..793446dc --- /dev/null +++ b/stack_go_memory_other.go @@ -0,0 +1,24 @@ +//go:build !darwin && !linux + +package tun + +func goArenaEnabled() bool { + return false +} + +func newGoSlabStore(lowWater int) goSlabStore { + return goHeapSlabStore{} +} + +func goRegionMap(size int) []byte { + return nil +} + +func goRegionUnmap(memory []byte) { +} + +func goRegionPark(memory []byte) { +} + +func goRegionReuse(memory []byte) { +} diff --git a/stack_go_profile_default.go b/stack_go_profile_default.go index e8cca441..3e14643a 100644 --- a/stack_go_profile_default.go +++ b/stack_go_profile_default.go @@ -16,5 +16,4 @@ const ( goDescriptorRingCapacity = 2048 goReassemblyEntries = 8 goSlabPoolLowWater = 128 - goDescriptorPoolLowWater = 64 ) diff --git a/stack_go_profile_ios.go b/stack_go_profile_ios.go index 4482bd3d..f083353c 100644 --- a/stack_go_profile_ios.go +++ b/stack_go_profile_ios.go @@ -14,5 +14,4 @@ const ( goDescriptorRingCapacity = 512 goReassemblyEntries = 4 goSlabPoolLowWater = 8 - goDescriptorPoolLowWater = 16 ) diff --git a/stack_go_ring.go b/stack_go_ring.go index e6344f28..5d4ac539 100644 --- a/stack_go_ring.go +++ b/stack_go_ring.go @@ -17,82 +17,33 @@ const ( type goSlab [goSlabSize]byte type goSlabPool struct { - access sync.Mutex - free []*goSlab + store goSlabStore inUse atomic.Int32 holders atomic.Int32 pressure func() MemoryPressure - lowWater int - closed bool } func newGoSlabPool(pressure func() MemoryPressure, lowWater int) *goSlabPool { - pool := &goSlabPool{ - free: make([]*goSlab, 0, lowWater), - pressure: pressure, - lowWater: lowWater, - } - for range lowWater { - pool.free = append(pool.free, goAllocateSlab()) - } - return pool + return &goSlabPool{store: newGoSlabStore(lowWater), pressure: pressure} } func (p *goSlabPool) acquire() *goSlab { - var slab *goSlab - p.access.Lock() - index := len(p.free) - 1 - if index >= 0 { - slab = p.free[index] - p.free[index] = nil - p.free = p.free[:index] - } - p.access.Unlock() - if slab == nil { - slab = goAllocateSlab() - } + slab := p.store.acquire() p.inUse.Add(1) return slab } func (p *goSlabPool) release(slab *goSlab) { - p.access.Lock() - if !p.closed && p.pressureLevel() == MemoryPressureNone { - p.free = append(p.free, slab) - p.access.Unlock() - } else { - p.access.Unlock() - goFreeSlab(slab) - } + p.store.release(slab, p.pressureLevel() != MemoryPressureNone) p.inUse.Add(-1) } func (p *goSlabPool) close() { - p.access.Lock() - p.closed = true - free := p.free - p.free = nil - p.access.Unlock() - for _, slab := range free { - goFreeSlab(slab) - } + p.store.close() } -func (p *goSlabPool) trim() { - p.access.Lock() - defer p.access.Unlock() - if len(p.free) <= p.lowWater { - return - } - target := p.lowWater - if p.pressureLevel() == MemoryPressureNone { - target = max(p.lowWater, len(p.free)/2) - } - for _, slab := range p.free[target:] { - goFreeSlab(slab) - } - clear(p.free[target:]) - p.free = p.free[:target] +func (p *goSlabPool) purge() { + p.store.purge() } func (p *goSlabPool) pressureLevel() MemoryPressure { @@ -464,40 +415,19 @@ type goDescriptorRing struct { } type goDescriptorPool struct { - access sync.Mutex - free []*goDescriptorBlock - lowWater int + blocks sync.Pool } func (p *goDescriptorPool) acquire() *goDescriptorBlock { - p.access.Lock() - index := len(p.free) - 1 - if index >= 0 { - block := p.free[index] - p.free[index] = nil - p.free = p.free[:index] - p.access.Unlock() - return block + block, _ := p.blocks.Get().(*goDescriptorBlock) + if block == nil { + block = new(goDescriptorBlock) } - p.access.Unlock() - return new(goDescriptorBlock) + return block } func (p *goDescriptorPool) release(block *goDescriptorBlock) { - p.access.Lock() - p.free = append(p.free, block) - p.access.Unlock() -} - -func (p *goDescriptorPool) trim() { - p.access.Lock() - defer p.access.Unlock() - if len(p.free) <= p.lowWater { - return - } - target := max(p.lowWater, len(p.free)/2) - clear(p.free[target:]) - p.free = p.free[:target] + p.blocks.Put(block) } // release requires exclusive access to both sides of the ring. For a live diff --git a/stack_go_slab_other.go b/stack_go_slab_other.go deleted file mode 100644 index e9b9c1a7..00000000 --- a/stack_go_slab_other.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !(darwin || linux) - -package tun - -func goAllocateSlab() *goSlab { - return new(goSlab) -} - -func goFreeSlab(slab *goSlab) { -} diff --git a/stack_go_slab_unix.go b/stack_go_slab_unix.go deleted file mode 100644 index aab241df..00000000 --- a/stack_go_slab_unix.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build darwin || linux - -package tun - -import ( - "unsafe" - - E "github.com/sagernet/sing/common/exceptions" - - "golang.org/x/sys/unix" -) - -func goAllocateSlab() *goSlab { - memory, err := unix.Mmap(-1, 0, goSlabSize, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_ANON|unix.MAP_PRIVATE) - if err != nil { - panic(E.Cause(err, "go: allocate slab")) - } - return (*goSlab)(unsafe.Pointer(&memory[0])) -} - -func goFreeSlab(slab *goSlab) { - unix.Munmap(unsafe.Slice(&slab[0], goSlabSize)) -} diff --git a/stack_go_splice.go b/stack_go_splice.go index 3602f43d..7c75c788 100644 --- a/stack_go_splice.go +++ b/stack_go_splice.go @@ -843,12 +843,10 @@ func (e *goEngine) packetSpliceRead(w *GoPacketConn) { splice := w.splice natConn := w.conn.Load() batchSize := e.packetReceiveBatch + e.packetSlots.configure(goPacketBatchSize, 65535) + e.packetSlots.wake(batchSize) for index := range batchSize { - buffer := e.packetReceiveBuffers[index] - if buffer == nil { - buffer = buf.NewSize(65535) - e.packetReceiveBuffers[index] = buffer - } + buffer := e.packetSlots.slot(index) buffer.Reset() e.packetMessages[index] = goPacketMessage{data: buffer.FreeBytes()} } @@ -870,7 +868,7 @@ func (e *goEngine) packetSpliceRead(w *GoPacketConn) { if message.truncated { continue } - buffer := e.packetReceiveBuffers[index] + buffer := e.packetSlots.slot(index) offset := 0 for { length := len(message.data) - offset diff --git a/stack_go_tcp_ack.go b/stack_go_tcp_ack.go index d8de7b51..cde9c6b4 100644 --- a/stack_go_tcp_ack.go +++ b/stack_go_tcp_ack.go @@ -398,8 +398,8 @@ func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFac } if newUnacked > priorUnacked { conn.scoreboard.advance(newUnacked) - conn.releaseTransmitted(newUnacked) conn.sendUnacked.Store(newUnacked) + conn.releaseTransmitted(newUnacked) if len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorSacked != 0 { flags |= goAckReneging } diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index b6088e5e..efa29d3a 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -35,8 +35,6 @@ const ( goFinWait2Timeout = 60 * time.Second goAckRampBytes = 256 << 10 goDelayedAckInterval = 25 * time.Millisecond - goSweepInterval = 500 * time.Millisecond - goReclaimInterval = 60 * time.Second goDyingLeakTimeout = 30 * time.Second goMaxCongestionSize = 4 << 20 goResetBurstLimit = 32 @@ -261,7 +259,11 @@ func (e *goEngine) inputTCP(conn *GoConn, parsed *forwardPacket) { return } conn.lastActivity = e.coarseTime.Load() - conn.keepaliveProbes = 0 + if conn.keepaliveProbes != 0 { + conn.keepaliveProbes = 0 + e.armKeepalive(conn) + e.rearmTimer(conn) + } if conn.state == goTCPSynSent { e.inputSynSent(conn, parsed, tcpHdr, dataOffset) return @@ -409,6 +411,8 @@ func (e *goEngine) establishConn(conn *GoConn) { conn.access.Lock() conn.phase = goPhaseEstablished conn.access.Unlock() + e.armKeepalive(conn) + e.rearmTimer(conn) close(conn.establishedSignal) if conn.listener != nil { conn.listener.deliver(conn) @@ -520,6 +524,7 @@ func (c *GoConn) releaseTransmitted(unacked uint64) { c.transmitStore.releaseBelow(edge) if edge == c.bufferedTail.Load() { c.transmitStore.releaseDrained(edge) + c.scoreboard.trim() } c.sendReleased.Store(edge) } @@ -654,7 +659,6 @@ func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { e.spliceDeliver(conn, data) return } - conn.receiveChain.reserve(conn.receiveNext, len(data)) direct := 0 taken := false if conn.receiveAvailable.Load() == conn.consumedTail.Load() { @@ -666,6 +670,7 @@ func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { } } if direct < len(data) { + conn.receiveChain.reserve(conn.receiveNext+uint64(direct), len(data)-direct) conn.receiveChain.writeAt(conn.receiveNext+uint64(direct), data[direct:]) } conn.receiveNext += uint64(len(data)) @@ -757,6 +762,8 @@ func (e *goEngine) advanceFinState(conn *GoConn) { } conn.state = goTCPFinWait2 conn.finWait2Since = e.coarseTime.Load() + e.armKeepalive(conn) + e.rearmTimer(conn) case goTCPClosing, goTCPLastAck: e.finishClose(conn) } @@ -995,6 +1002,11 @@ func (e *goEngine) handleCloseRequest(conn *GoConn) { } conn.finPending = true e.maybeSendFin(conn) + if conn.dead { + return + } + e.armKeepalive(conn) + e.rearmTimer(conn) } func (e *goEngine) handleReadShut(conn *GoConn) { @@ -1018,13 +1030,28 @@ func (e *goEngine) handleWindowUpdate(conn *GoConn) { if conn.dead { return } - conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + consumed := conn.consumedTail.Load() + windowUpdate := consumed >= conn.windowUpdateAt.Load() + conn.receiveChain.releaseBelow(consumed) + conn.releaseDrainedSlabs() + if !windowUpdate { + return + } conn.publishReceiveWindow() if conn.publishedEdge > conn.advertisedEdge() || conn.ackPending > 0 { e.markAck(conn, true) } } +func (e *goEngine) handleTransmitted(conn *GoConn) { + if conn.dead { + return + } + conn.releaseTransmitted(conn.sendUnacked.Load()) + conn.releaseIdleDescriptors() + conn.wakeWriter() +} + func (e *goEngine) handleRetransmitArm(conn *GoConn) { if conn.dead || conn.retransmitDeadline != 0 { return @@ -1157,7 +1184,7 @@ func (e *goEngine) updatePersist(conn *GoConn) { func (e *goEngine) rearmTimer(conn *GoConn) { next := int64(0) - for _, deadline := range [8]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline, conn.idleDeadline, conn.pacingDeadline, conn.reorderDeadline} { + for _, deadline := range [9]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline, conn.idleDeadline, conn.pacingDeadline, conn.reorderDeadline, conn.keepaliveDeadline} { if deadline == 0 { continue } @@ -1206,6 +1233,10 @@ func (e *goEngine) fireConnTimer(conn *GoConn, now int64) { e.expireRetransmit(conn, now) return } + if conn.keepaliveDeadline != 0 && conn.keepaliveDeadline <= now { + e.expireKeepalive(conn, now) + return + } e.rearmTimer(conn) } @@ -1290,6 +1321,7 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { conn.idleDeadline = 0 conn.pacingDeadline = 0 conn.reorderDeadline = 0 + conn.keepaliveDeadline = 0 conn.finPending = false switch class { case goDeathFinLinger: @@ -1411,35 +1443,32 @@ func (e *goEngine) closeAllFlows() []*GoConn { return unreset } -func (e *goEngine) expireSweepTick(now int64) { - for _, conn := range e.flows { - if conn.dead { - continue - } - sent := conn.sentTail.Load() - if sent != conn.sweepSentTail { - conn.sweepSentTail = sent - conn.lastActivity = now - } - e.keepalive(conn, now) - if conn.dead { - continue - } - conn.receiveChain.releaseBelow(conn.consumedTail.Load()) - conn.releaseTransmitted(conn.sendUnacked.Load()) - conn.releaseDrainedSlabs() - conn.releaseIdleDescriptors() - conn.wakeWriter() - e.maybeSendFin(conn) - e.updatePersist(conn) - if conn.hasOutstanding() && conn.retransmitDeadline == 0 { - e.rearmRetransmit(conn) - } +func (e *goEngine) handleKeepaliveUpdate(conn *GoConn) { + if conn.dead { + return + } + e.armKeepalive(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) armKeepalive(conn *GoConn) { + conn.access.Lock() + userClosed := conn.userClosed + enabled := conn.keepaliveEnabled + idleThreshold := conn.keepaliveIdle + conn.access.Unlock() + switch { + case conn.state == goTCPFinWait2 && userClosed: + conn.keepaliveDeadline = conn.finWait2Since + int64(goFinWait2Timeout) + case enabled && !conn.handshaking(): + conn.keepaliveProbes = 0 + conn.keepaliveDeadline = conn.lastActivity + int64(idleThreshold) + default: + conn.keepaliveDeadline = 0 } - e.wheel.schedule(&e.sweepTickNode, now+int64(goSweepInterval)) } -func (e *goEngine) keepalive(conn *GoConn, now int64) { +func (e *goEngine) expireKeepalive(conn *GoConn, now int64) { conn.access.Lock() userClosed := conn.userClosed enabled := conn.keepaliveEnabled @@ -1447,22 +1476,32 @@ func (e *goEngine) keepalive(conn *GoConn, now int64) { interval := conn.keepaliveInterval count := conn.keepaliveCount conn.access.Unlock() - if conn.state == goTCPFinWait2 && userClosed && now-conn.finWait2Since > int64(goFinWait2Timeout) { - e.sendReset(conn) - e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: orphaned FIN_WAIT_2"), goDeathImmediate) + if conn.state == goTCPFinWait2 && userClosed { + if now-conn.finWait2Since >= int64(goFinWait2Timeout) { + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: orphaned FIN_WAIT_2"), goDeathImmediate) + return + } + conn.keepaliveDeadline = conn.finWait2Since + int64(goFinWait2Timeout) + e.rearmTimer(conn) return } - idle := now - conn.lastActivity - if !enabled || idle < int64(idleThreshold) { + if !enabled { conn.keepaliveProbes = 0 + conn.keepaliveDeadline = 0 + e.rearmTimer(conn) return } - if conn.handshaking() || conn.hasOutstanding() { + idle := now - conn.lastActivity + if idle < int64(idleThreshold) { + conn.keepaliveProbes = 0 + conn.keepaliveDeadline = conn.lastActivity + int64(idleThreshold) + e.rearmTimer(conn) return } - sinceIdle := idle - int64(idleThreshold) - due := int64(conn.keepaliveProbes) * int64(interval) - if sinceIdle < due { + if conn.hasOutstanding() { + conn.keepaliveDeadline = now + int64(idleThreshold) + e.rearmTimer(conn) return } if conn.keepaliveProbes >= count { @@ -1473,24 +1512,13 @@ func (e *goEngine) keepalive(conn *GoConn, now int64) { conn.keepaliveProbes++ segment := goSegment{offset: conn.sendNext() - 1, flags: header.TCPFlagAck} e.writeConnControl(conn, &segment) -} - -func (e *goEngine) expireReclaimTick(now int64) { - e.reclaim() - e.wheel.schedule(&e.reclaimTickNode, now+int64(goReclaimInterval)) + conn.keepaliveDeadline = now + int64(interval) + e.rearmTimer(conn) } func (e *goEngine) reclaim() { - e.slabPool.trim() - e.descriptorPool.trim() + e.slabPool.purge() for _, conn := range e.flows { conn.scoreboard.trim() } - for index := range e.reassemblyEntries { - entry := &e.reassemblyEntries[index] - if !entry.active { - entry.buffer.Release() - entry.buffer = nil - } - } } diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go index 022f7b1c..8fc4afe2 100644 --- a/stack_go_tcp_output.go +++ b/stack_go_tcp_output.go @@ -314,7 +314,7 @@ func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { conn.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) conn.blockedValid = false conn.blockedFrame = nil - conn.transmittedTail.Store(end) + conn.transmitted(end) conn.transmitAccess.Unlock() e.handleDroppedFrames(conn) return @@ -328,7 +328,7 @@ func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { } conn.blockedValid = false conn.blockedFrame = nil - conn.transmittedTail.Store(end) + conn.transmitted(end) conn.transmitAccess.Unlock() if conn.splice != nil { e.spliceResume(conn) @@ -583,6 +583,9 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu } c.pushDescriptors(sent, length, mss, c.stamp(now), flags, flightEmpty) c.sentTail.Store(sent + uint64(length)) + if !armed && c.sendUnacked.Load() == sent { + armed = true + } creditBase := c.packetCreditBase.Load() flight := uint64(c.dataSegmentsOut.Load() - creditBase) if flight > c.peakFlight.Load() { @@ -599,7 +602,7 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu } switch err { case nil: - c.transmittedTail.Store(sent + uint64(length)) + c.transmitted(sent + uint64(length)) case errGoTransmitBlocked: c.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) c.blockedSegment = segment @@ -612,7 +615,7 @@ func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResu } case errGoFrameDropped: c.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) - c.transmittedTail.Store(sent + uint64(length)) + c.transmitted(sent + uint64(length)) c.engine.postMessage(&c.droppedMessage) default: if !c.closed() { @@ -649,7 +652,7 @@ func (c *GoConn) retryBlocked() bool { c.advancePacing(int64(time.Since(c.engine.epoch)), c.blockedSegment.length) } c.blockedValid = false - c.transmittedTail.Store(end) + c.transmitted(end) if err == errGoFrameDropped { c.engine.postMessage(&c.droppedMessage) } diff --git a/stack_go_udp.go b/stack_go_udp.go index 3c933c37..dd1ca0f8 100644 --- a/stack_go_udp.go +++ b/stack_go_udp.go @@ -63,17 +63,10 @@ func (e *goEngine) processUDPSegments(frame *goFrame) { return } options := *e.packetReadOptions.Load() - if e.gsoReadOptions != options { - buf.ReleaseMulti(e.gsoBuffers[:]) - clear(e.gsoBuffers[:]) - e.gsoReadOptions = options - } + e.gsoSlots.configure(goGSOMaxSegments, mtu+options.FrontHeadroom+options.RearHeadroom) + e.gsoSlots.wake(count) for index := range count { - buffer := e.gsoBuffers[index] - if buffer == nil { - buffer = options.NewBufferSize(mtu) - e.gsoBuffers[index] = buffer - } + buffer := e.gsoSlots.slot(index) buffer.Reset() buffer.Resize(options.FrontHeadroom, 0) buffer.Reserve(options.RearHeadroom) @@ -88,7 +81,7 @@ func (e *goEngine) processUDPSegments(frame *goFrame) { return } for index := range count { - buffer := e.gsoBuffers[index] + buffer := e.gsoSlots.slot(index) buffer.Truncate(e.gsoSizes[index]) options.PostReturn(buffer) e.gsoFrame = goFrame{buffer: buffer} diff --git a/stack_go_wheel.go b/stack_go_wheel.go index 4ff7b157..6bdcb130 100644 --- a/stack_go_wheel.go +++ b/stack_go_wheel.go @@ -13,11 +13,18 @@ const ( goWheelRotationTicks = goWheelLevel1Slots * goWheelLevel1Ticks ) +const ( + goWheelLevel0 uint8 = iota + goWheelLevel1 + goWheelOverflow +) + type goWheelNode struct { prev *goWheelNode next *goWheelNode slot **goWheelNode deadline int64 + level uint8 expire func(now int64) } @@ -26,7 +33,10 @@ type goWheel struct { level1 [goWheelLevel1Slots]*goWheelNode overflow *goWheelNode currentTick int64 + overflowMin int64 scheduled int + level0Count int + level1Count int } func (w *goWheel) schedule(node *goWheelNode, deadline int64) { @@ -52,10 +62,18 @@ func (w *goWheel) file(node *goWheelNode) { var slot **goWheelNode if tick-w.currentTick < goWheelLevel0Slots { slot = &w.level0[tick%goWheelLevel0Slots] + node.level = goWheelLevel0 + w.level0Count++ } else if tick/goWheelLevel1Ticks-w.currentTick/goWheelLevel1Ticks < goWheelLevel1Slots { slot = &w.level1[(tick/goWheelLevel1Ticks)%goWheelLevel1Slots] + node.level = goWheelLevel1 + w.level1Count++ } else { slot = &w.overflow + node.level = goWheelOverflow + if w.overflow == nil || tick < w.overflowMin { + w.overflowMin = tick + } } node.prev = nil node.next = *slot @@ -78,6 +96,12 @@ func (w *goWheel) unlink(node *goWheelNode) { node.prev = nil node.next = nil node.slot = nil + switch node.level { + case goWheelLevel0: + w.level0Count-- + case goWheelLevel1: + w.level1Count-- + } } func (w *goWheel) advance(now int64) { @@ -87,7 +111,19 @@ func (w *goWheel) advance(now int64) { w.currentTick = targetTick return } - w.currentTick++ + if w.level0Count == 0 { + boundary := (w.currentTick/goWheelLevel1Ticks + 1) * goWheelLevel1Ticks + if w.level1Count == 0 { + boundary = (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks + } + if boundary > targetTick { + w.currentTick = targetTick + return + } + w.currentTick = boundary + } else { + w.currentTick++ + } if w.currentTick%goWheelLevel1Ticks == 0 { if w.currentTick%goWheelRotationTicks == 0 { w.refile(&w.overflow) @@ -107,6 +143,9 @@ func (w *goWheel) refile(slot **goWheelNode) { node.prev = nil node.next = nil node.slot = nil + if node.level == goWheelLevel1 { + w.level1Count-- + } w.file(node) } } @@ -125,24 +164,29 @@ func (w *goWheel) nextDeadline() (int64, bool) { return 0, false } deadline := int64(math.MaxInt64) - for offset := int64(1); offset < goWheelLevel0Slots; offset++ { - tick := w.currentTick + offset - if w.level0[tick%goWheelLevel0Slots] != nil { - deadline = tick * goWheelTick - break + if w.level0Count > 0 { + for offset := int64(1); offset < goWheelLevel0Slots; offset++ { + tick := w.currentTick + offset + if w.level0[tick%goWheelLevel0Slots] != nil { + deadline = tick * goWheelTick + break + } } } - level1Index := w.currentTick / goWheelLevel1Ticks - for offset := int64(1); offset < goWheelLevel1Slots; offset++ { - index := level1Index + offset - if w.level1[index%goWheelLevel1Slots] != nil { - deadline = min(deadline, index*goWheelLevel1Ticks*goWheelTick) - break + if w.level1Count > 0 { + level1Index := w.currentTick / goWheelLevel1Ticks + for offset := int64(1); offset < goWheelLevel1Slots; offset++ { + index := level1Index + offset + if w.level1[index%goWheelLevel1Slots] != nil { + deadline = min(deadline, index*goWheelLevel1Ticks*goWheelTick) + break + } } } if w.overflow != nil { - rotation := (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks * goWheelTick - deadline = min(deadline, rotation) + rotation := (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks + landing := (w.overflowMin / goWheelRotationTicks) * goWheelRotationTicks + deadline = min(deadline, max(rotation, landing)*goWheelTick) } return deadline, true } From ce3df19cde8967c27f5d67ef2b9c86c7b0381ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 19 Sep 2026 18:04:09 +0800 Subject: [PATCH 14/25] Fix Android VPNService kernel bypass --- monitor_linux.go | 6 +- redirect_ingress_discard_android.go | 177 +++++++++++++++++++++++++++ redirect_iptables.go | 8 ++ redirect_linux.go | 1 + redirect_route_vpnservice_android.go | 14 ++- 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 redirect_ingress_discard_android.go diff --git a/monitor_linux.go b/monitor_linux.go index 5a13a7f7..b57a1f29 100644 --- a/monitor_linux.go +++ b/monitor_linux.go @@ -67,9 +67,13 @@ func (m *networkUpdateMonitor) Start() error { if err != nil { return E.Cause(err, "create netlink socket") } + groups := uint32(netlinkGroups) + if runtime.GOOS == "android" { + groups |= unix.RTMGRP_IPV4_RULE | 1<<(unix.RTNLGRP_IPV6_RULE-1) + } err = unix.Bind(netlinkSocket, &unix.SockaddrNetlink{ Family: unix.AF_NETLINK, - Groups: netlinkGroups, + Groups: groups, }) if err != nil { unix.Close(netlinkSocket) diff --git a/redirect_ingress_discard_android.go b/redirect_ingress_discard_android.go new file mode 100644 index 00000000..291a6771 --- /dev/null +++ b/redirect_ingress_discard_android.go @@ -0,0 +1,177 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "os" + "runtime" + "unsafe" + + "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +// ConnectivityService.updateIngressToVpnAddressFiltering keys this map by the +// VPN address (IPv4-mapped) with the tun ifindex in both value slots, and the +// cgroup skb ingress program in netd.c drops packets for app uids to that +// address from any other interface, which includes the de-NATed replies of +// bypassed flows. ConnectivityService updates entries from LinkProperties diffs. +const androidIngressDiscardMapPath = "/sys/fs/bpf/netd_shared/map_netd_ingress_discard_map" + +type bpfObjectAttribute struct { + pathname uint64 + fileDescriptor uint32 + fileFlags uint32 +} + +type bpfMapAttribute struct { + mapDescriptor uint32 + _ uint32 + key uint64 + value uint64 + flags uint64 +} + +type bpfObjectInfoAttribute struct { + fileDescriptor uint32 + length uint32 + info uint64 +} + +func bpfObjectGet(path string) (int, error) { + pathBytes, err := unix.BytePtrFromString(path) + if err != nil { + return 0, err + } + var pinner runtime.Pinner + pinner.Pin(pathBytes) + defer pinner.Unpin() + attribute := bpfObjectAttribute{pathname: uint64(uintptr(unsafe.Pointer(pathBytes)))} + descriptor, _, errno := unix.Syscall(unix.SYS_BPF, unix.BPF_OBJ_GET, uintptr(unsafe.Pointer(&attribute)), unsafe.Sizeof(attribute)) + if errno != 0 { + return 0, errno + } + var info struct { + mapType uint32 + id uint32 + keySize uint32 + valueSize uint32 + } + pinner.Pin(&info) + infoAttribute := bpfObjectInfoAttribute{ + fileDescriptor: uint32(descriptor), + length: uint32(unsafe.Sizeof(info)), + info: uint64(uintptr(unsafe.Pointer(&info))), + } + _, _, errno = unix.Syscall(unix.SYS_BPF, unix.BPF_OBJ_GET_INFO_BY_FD, uintptr(unsafe.Pointer(&infoAttribute)), unsafe.Sizeof(infoAttribute)) + if errno != 0 { + unix.Close(int(descriptor)) + return 0, errno + } + if info.mapType != unix.BPF_MAP_TYPE_HASH || info.keySize != 16 || info.valueSize != 8 { + unix.Close(int(descriptor)) + return 0, E.New("unexpected ingress discard map layout: type ", info.mapType, ", key ", info.keySize, ", value ", info.valueSize) + } + return int(descriptor), nil +} + +func bpfMapCall(command uintptr, mapDescriptor int, key []byte, value []byte, flags uint64) error { + var pinner runtime.Pinner + pinner.Pin(&key[0]) + defer pinner.Unpin() + attribute := bpfMapAttribute{ + mapDescriptor: uint32(mapDescriptor), + key: uint64(uintptr(unsafe.Pointer(&key[0]))), + flags: flags, + } + if value != nil { + pinner.Pin(&value[0]) + attribute.value = uint64(uintptr(unsafe.Pointer(&value[0]))) + } + _, _, errno := unix.Syscall(unix.SYS_BPF, command, uintptr(unsafe.Pointer(&attribute)), unsafe.Sizeof(attribute)) + if errno != 0 { + return errno + } + return nil +} + +func (r *autoRedirect) removeAndroidIngressDiscardRulesLocked() error { + if !r.enableIPv4 { + return nil + } + _, err := os.Stat(androidIngressDiscardMapPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return E.Cause(err, "stat ingress discard map") + } + mapDescriptor, err := bpfObjectGet(androidIngressDiscardMapPath) + if err != nil { + if err == unix.ENOENT { + return nil + } + return E.Cause(err, "open ingress discard map") + } + defer unix.Close(mapDescriptor) + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return E.Cause(err, "find tun interface") + } + for _, prefix := range r.tunOptions.Inet4Address { + address := prefix.Addr() + key := address.As16() + var value [8]byte + err = bpfMapCall(unix.BPF_MAP_LOOKUP_ELEM, mapDescriptor, key[:], value[:], 0) + if err == unix.ENOENT { + continue + } + if err != nil { + return E.Cause(err, "read ingress discard rule for ", address) + } + if int(binary.NativeEndian.Uint32(value[:4])) != tunLink.Attrs().Index || + int(binary.NativeEndian.Uint32(value[4:])) != tunLink.Attrs().Index { + continue + } + err = bpfMapCall(unix.BPF_MAP_DELETE_ELEM, mapDescriptor, key[:], nil, 0) + if err != nil && err != unix.ENOENT { + return E.Cause(err, "remove ingress discard rule for ", address) + } + if r.androidIngressDiscardValues == nil { + r.androidIngressDiscardValues = make(map[netip.Addr][8]byte) + } + r.androidIngressDiscardValues[address] = value + r.logger.Debug("removed ingress discard rule for ", address) + } + return nil +} + +func (r *autoRedirect) restoreAndroidIngressDiscardRulesLocked() { + values := r.androidIngressDiscardValues + r.androidIngressDiscardValues = nil + if len(values) == 0 { + return + } + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return + } + mapDescriptor, err := bpfObjectGet(androidIngressDiscardMapPath) + if err != nil { + r.logger.Error("restore ingress discard rules: open map: ", err) + return + } + defer unix.Close(mapDescriptor) + for address, value := range values { + if int(binary.NativeEndian.Uint32(value[:4])) != tunLink.Attrs().Index { + continue + } + key := address.As16() + err = bpfMapCall(unix.BPF_MAP_UPDATE_ELEM, mapDescriptor, key[:], value[:], unix.BPF_NOEXIST) + if err != nil && err != unix.EEXIST { + r.logger.Error("restore ingress discard rule for ", address, ": ", err) + } + } +} diff --git a/redirect_iptables.go b/redirect_iptables.go index 93f85478..ba7d7773 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -589,6 +589,14 @@ func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { return postroutingNAT.err } inserts = append(inserts, iptablesInsert{iptablesTableNAT, "POSTROUTING", []string{"-j", postroutingNAT.name}}) + postroutingMangle := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-postrouting") + for _, prefix := range options.Inet4Address { + postroutingMangle.add("-s", prefix.Masked().String(), "!", "-o", options.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-m", "mark", "--mark", outputMark, "-j", "TCPMSS", "--clamp-mss-to-pmtu") + } + if postroutingMangle.err != nil { + return postroutingMangle.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "POSTROUTING", []string{"-j", postroutingMangle.name}}) } } diff --git a/redirect_linux.go b/redirect_linux.go index 66e7ebc5..798b63f1 100644 --- a/redirect_linux.go +++ b/redirect_linux.go @@ -59,6 +59,7 @@ type autoRedirect struct { androidVPNServiceRuleAccess sync.Mutex androidVPNServiceRules []*netlink.Rule androidVPNServiceRulesActive bool + androidIngressDiscardValues map[netip.Addr][8]byte dockerFirewallMonitor *nftables.Monitor dockerFirewallDone chan struct{} } diff --git a/redirect_route_vpnservice_android.go b/redirect_route_vpnservice_android.go index 952882cf..af2803a6 100644 --- a/redirect_route_vpnservice_android.go +++ b/redirect_route_vpnservice_android.go @@ -174,7 +174,7 @@ func (r *autoRedirect) setupAndroidVPNServiceRules() error { } r.androidVPNServiceRules = append(r.androidVPNServiceRules, rule) } - return nil + return r.removeAndroidIngressDiscardRulesLocked() } func (r *autoRedirect) updateAndroidVPNServiceRules() error { @@ -191,6 +191,10 @@ func (r *autoRedirect) updateAndroidVPNServiceRules() error { } return err } + currentRules, err := netlink.RuleListFiltered(netlink.FAMILY_ALL, &netlink.Rule{Priority: r.androidVPNServiceRulePriority()}, netlink.RT_FILTER_PRIORITY) + if err != nil { + return E.Cause(err, "list current vpn service rules") + } var errorList []error var installedRules []*netlink.Rule for _, installedRule := range r.androidVPNServiceRules { @@ -206,6 +210,12 @@ func (r *autoRedirect) updateAndroidVPNServiceRules() error { } } for _, rule := range rules { + if slices.ContainsFunc(currentRules, func(it netlink.Rule) bool { + return androidVPNServiceRuleEquals(&it, rule) + }) { + installedRules = append(installedRules, rule) + continue + } addErr := netlink.RuleAdd(rule) if addErr != nil && !E.IsMulti(addErr, unix.EEXIST, os.ErrExist) { errorList = append(errorList, E.Cause(addErr, "add rule ", rule)) @@ -214,6 +224,7 @@ func (r *autoRedirect) updateAndroidVPNServiceRules() error { installedRules = append(installedRules, rule) } r.androidVPNServiceRules = installedRules + errorList = append(errorList, r.removeAndroidIngressDiscardRulesLocked()) return E.Errors(errorList...) } @@ -221,6 +232,7 @@ func (r *autoRedirect) cleanupAndroidVPNServiceRules() { r.androidVPNServiceRuleAccess.Lock() defer r.androidVPNServiceRuleAccess.Unlock() r.androidVPNServiceRulesActive = false + r.restoreAndroidIngressDiscardRulesLocked() for _, rule := range r.androidVPNServiceRules { _ = netlink.RuleDel(rule) } From 775ddacdf2e3ebeadb919d2b895ed150364068ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 19 Sep 2026 19:44:24 +0800 Subject: [PATCH 15/25] Exit go stack engine on unrecoverable read errors --- stack_go_engine.go | 9 +++++++-- stack_go_io_other.go | 4 ---- stack_go_io_windows.go | 4 ---- stack_go_socket_unix.go | 4 ---- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/stack_go_engine.go b/stack_go_engine.go index 0294dab1..f017eb8d 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -251,6 +251,7 @@ func (e *goEngine) run() { defer close(e.exitSignal) defer e.exit() e.loadWindowStart = e.coarseTime.Load() + var readRetry ReadRetry for { parkStart := int64(time.Since(e.epoch)) tunReadable, eventCount, err := e.park() @@ -280,10 +281,14 @@ func (e *goEngine) run() { if e.stack.closed.Load() || E.IsClosedOrCanceled(err) { return } - e.stack.logger.Error(E.Cause(err, "go: engine read")) - if goFatalReadError(err) { + if !IsRecoverableReadError(err) { + e.stack.logger.Error(E.Cause(err, "go: engine read")) return } + e.stack.logger.Debug(E.Cause(err, "go: engine read")) + readRetry.Wait(err) + } else { + readRetry.Reset() } } e.dispatchSocketEvents(eventCount) diff --git a/stack_go_io_other.go b/stack_go_io_other.go index 2f3f1363..5dc00117 100644 --- a/stack_go_io_other.go +++ b/stack_go_io_other.go @@ -15,10 +15,6 @@ func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { return nil, E.New("go: unsupported platform") } -func goFatalReadError(err error) bool { - return true -} - const goSpliceDuplicatesSocket = false type goIOVector struct{} diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index 1f8d1c56..134e3bc0 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -308,10 +308,6 @@ func (o *goWindowsIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (in return count, false, nil } -func goFatalReadError(err error) bool { - return true -} - func (o *goWindowsIO) releaseReadBuffers() { o.receiveSlots.sleep() } diff --git a/stack_go_socket_unix.go b/stack_go_socket_unix.go index 69799ca6..5ee3ff37 100644 --- a/stack_go_socket_unix.go +++ b/stack_go_socket_unix.go @@ -104,10 +104,6 @@ func goSocketDropped(errno syscall.Errno) bool { return errno == unix.EMSGSIZE || errno == unix.EAFNOSUPPORT || errno == unix.ENOBUFS } -func goFatalReadError(err error) bool { - return E.IsMulti(err, unix.EBADF, unix.ENODEV, unix.ENXIO) -} - func (s *goSocket) readVector(iovecs []unix.Iovec) (int, syscall.Errno) { //nolint:staticcheck n, _, errno := unix.RawSyscall(unix.SYS_READV, uintptr(s.fd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) From 279285b2d809bf38734738dee6448bc31847018a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 22 Sep 2026 09:42:21 +0800 Subject: [PATCH 16/25] Add ICMP error builder and IP transport protocol helper --- flow_dispatch.go | 10 +- flow_mtu.go | 65 +-------- flow_reject.go | 134 ++---------------- icmp_error.go | 203 ++++++++++++++++++++++++++++ stack_go_integration_memory_test.go | 2 +- stack_go_udp.go | 4 +- 6 files changed, 219 insertions(+), 199 deletions(-) create mode 100644 icmp_error.go diff --git a/flow_dispatch.go b/flow_dispatch.go index fdc61974..97221d38 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -645,16 +645,8 @@ func (s *ForwardStage) forwardToPort(flow *forwardFlow, packet *forwardPacket, r } return } - reply, ok := buildFragmentationNeeded(ipHdr, flow.effectiveMTU, s.writeback.ReturnHeadroom()) - if ok { - s.writebackBatch = append(s.writebackBatch, reply) - } - return - } - reply, ok := buildPacketTooBig(header.IPv6(packet.network), flow.effectiveMTU, s.writeback.ReturnHeadroom()) - if ok { - s.writebackBatch = append(s.writebackBatch, reply) } + s.writebackBatch = append(s.writebackBatch, buildICMPError(packet, ICMPErrorPacketTooBig, flow.effectiveMTU, s.writeback.ReturnHeadroom())) return } if flow.tracker != nil { diff --git a/flow_mtu.go b/flow_mtu.go index bfbf9c9e..a998774d 100644 --- a/flow_mtu.go +++ b/flow_mtu.go @@ -19,10 +19,7 @@ func (s *ForwardStage) resegmentTCP(flow *forwardFlow, packet *forwardPacket, ra } headerLength := len(raw) - len(packet.transport) if packet.ipVersion == 6 && headerLength != header.IPv6MinimumSize { - reply, ok := buildPacketTooBig(header.IPv6(packet.network), effectiveMTU, s.writeback.ReturnHeadroom()) - if ok { - s.writebackBatch = append(s.writebackBatch, reply) - } + s.writebackBatch = append(s.writebackBatch, buildICMPError(packet, ICMPErrorPacketTooBig, effectiveMTU, s.writeback.ReturnHeadroom())) return } tcpHeaderLength := int(header.TCP(packet.transport).DataOffset()) @@ -142,63 +139,3 @@ func encodeIPv6FragmentHeader(target []byte, nextHeader uint8, offset int, more binary.BigEndian.PutUint16(target[2:], value) binary.BigEndian.PutUint32(target[4:], ident) } - -func buildFragmentationNeeded(packet header.IPv4, effectiveMTU uint32, headroom int) ([]byte, bool) { - advertised := max(effectiveMTU, header.IPv4MinimumMTU) - originalLength := min(int(packet.TotalLength()), len(packet)) - minPayloadLength := int(packet.HeaderLength()) + header.ICMPv4MinimumErrorPayloadSize - if originalLength < minPayloadLength { - return nil, false - } - maxPayloadLength := header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - header.ICMPv4MinimumSize - payloadLength := min(originalLength, maxPayloadLength) - size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + payloadLength - buffer := make([]byte, headroom+size) - response := header.IPv4(buffer[headroom:]) - response.Encode(&header.IPv4Fields{ - TotalLength: uint16(size), - TTL: synthesizedTTL, - Protocol: uint8(header.ICMPv4ProtocolNumber), - SrcAddr: packet.DestinationAddr(), - DstAddr: packet.SourceAddr(), - }) - response.SetChecksum(^response.CalculateChecksum()) - icmpHdr := header.ICMPv4(response.Payload()) - icmpHdr.SetType(header.ICMPv4DstUnreachable) - icmpHdr.SetCode(header.ICMPv4FragmentationNeeded) - icmpHdr.SetMTU(uint16(min(advertised, uint32(0xffff)))) - copy(icmpHdr.Payload(), packet[:payloadLength]) - icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) - return buffer, true -} - -func buildPacketTooBig(packet header.IPv6, effectiveMTU uint32, headroom int) ([]byte, bool) { - advertised := max(effectiveMTU, header.IPv6MinimumMTU) - originalLength := min(header.IPv6MinimumSize+int(packet.PayloadLength()), len(packet)) - if originalLength < header.IPv6MinimumSize { - return nil, false - } - maxPayloadLength := header.IPv6MinimumMTU - header.IPv6MinimumSize - header.ICMPv6PacketTooBigMinimumSize - payloadLength := min(originalLength, maxPayloadLength) - size := header.IPv6MinimumSize + header.ICMPv6PacketTooBigMinimumSize + payloadLength - buffer := make([]byte, headroom+size) - response := header.IPv6(buffer[headroom:]) - response.Encode(&header.IPv6Fields{ - PayloadLength: uint16(header.ICMPv6PacketTooBigMinimumSize + payloadLength), - TransportProtocol: header.ICMPv6ProtocolNumber, - HopLimit: synthesizedTTL, - SrcAddr: packet.DestinationAddr(), - DstAddr: packet.SourceAddr(), - }) - icmpHdr := header.ICMPv6(response.Payload()) - icmpHdr.SetType(header.ICMPv6PacketTooBig) - icmpHdr.SetCode(header.ICMPv6UnusedCode) - icmpHdr.SetMTU(advertised) - copy(icmpHdr.Payload(), packet[:payloadLength]) - icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ - Header: icmpHdr, - Src: response.SourceAddressSlice(), - Dst: response.DestinationAddressSlice(), - })) - return buffer, true -} diff --git a/flow_reject.go b/flow_reject.go index 5610531d..a6a623a8 100644 --- a/flow_reject.go +++ b/flow_reject.go @@ -1,9 +1,6 @@ package tun import ( - "net/netip" - - "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" ) @@ -19,22 +16,21 @@ func buildReject(packet *forwardPacket, headroom int) ([]byte, bool) { } return buildResetIPv6(header.IPv6(packet.network), tcpHdr, headroom), true case uint8(header.UDPProtocolNumber): - if packet.ipVersion == 4 { - ipHdr := header.IPv4(packet.network) - return buildRejectICMPv4(ipHdr, header.ICMPv4PortUnreachable, ipHdr.DestinationAddr(), headroom) - } - ipHdr := header.IPv6(packet.network) - return buildRejectICMPv6(ipHdr, header.ICMPv6PortUnreachable, ipHdr.DestinationAddr(), headroom) + return buildICMPError(packet, ICMPErrorPortUnreachable, 0, headroom), true default: - if packet.ipVersion == 4 { - ipHdr := header.IPv4(packet.network) - return buildRejectICMPv4(ipHdr, header.ICMPv4HostUnreachable, ipHdr.DestinationAddr(), headroom) - } - ipHdr := header.IPv6(packet.network) - return buildRejectICMPv6(ipHdr, header.ICMPv6AddressUnreachable, ipHdr.DestinationAddr(), headroom) + return buildICMPError(packet, ICMPErrorAddressUnreachable, 0, headroom), true } } +func buildICMPError(packet *forwardPacket, errorType ICMPError, mtu uint32, headroom int) []byte { + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) + return buildICMPv4Error(ipHdr, errorType, ipHdr.DestinationAddr(), mtu, headroom) + } + ipHdr := header.IPv6(packet.network) + return buildICMPv6Error(ipHdr, errorType, ipHdr.DestinationAddr(), mtu, headroom) +} + func buildResetIPv4(origIPHdr header.IPv4, origTCPHdr header.TCP, headroom int) []byte { size := header.IPv4MinimumSize + header.TCPMinimumSize buffer := make([]byte, headroom+size) @@ -92,111 +88,3 @@ func encodeResetTCP(tcpHdr header.TCP, origTCPHdr header.TCP) { } tcpHdr.Encode(&fields) } - -func buildRejectICMPv4(ipHdr header.IPv4, code header.ICMPv4Code, source netip.Addr, headroom int) ([]byte, bool) { - const maxIPData = header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - available := maxIPData - header.ICMPv4MinimumSize - if len(ipHdr) < header.ICMPv4MinimumErrorPayloadSize { - return nil, false - } - payload := []byte(ipHdr) - if len(payload) > available { - payload = payload[:available] - } - size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + len(payload) - buffer := make([]byte, headroom+size) - newIPHdr := header.IPv4(buffer[headroom:]) - newIPHdr.Encode(&header.IPv4Fields{ - TotalLength: uint16(size), - TTL: synthesizedTTL, - Protocol: uint8(header.ICMPv4ProtocolNumber), - SrcAddr: source, - DstAddr: ipHdr.SourceAddr(), - }) - newIPHdr.SetChecksum(^newIPHdr.CalculateChecksum()) - icmpHdr := header.ICMPv4(newIPHdr.Payload()) - icmpHdr.SetType(header.ICMPv4DstUnreachable) - icmpHdr.SetCode(code) - copy(icmpHdr.Payload(), payload) - icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr[:header.ICMPv4MinimumSize], checksum.Checksum(payload, 0))) - return buffer, true -} - -func buildRejectICMPv6(ipHdr header.IPv6, code header.ICMPv6Code, source netip.Addr, headroom int) ([]byte, bool) { - const maxIPv6Data = header.IPv6MinimumMTU - header.IPv6FixedHeaderSize - available := maxIPv6Data - header.ICMPv6ErrorHeaderSize - if available < header.IPv6MinimumSize { - return nil, false - } - payload := []byte(ipHdr) - if len(payload) > available { - payload = payload[:available] - } - size := header.IPv6MinimumSize + header.ICMPv6DstUnreachableMinimumSize + len(payload) - buffer := make([]byte, headroom+size) - newIPHdr := header.IPv6(buffer[headroom:]) - newIPHdr.Encode(&header.IPv6Fields{ - PayloadLength: uint16(header.ICMPv6DstUnreachableMinimumSize + len(payload)), - TransportProtocol: header.ICMPv6ProtocolNumber, - HopLimit: synthesizedTTL, - SrcAddr: source, - DstAddr: ipHdr.SourceAddr(), - }) - icmpHdr := header.ICMPv6(newIPHdr.Payload()) - icmpHdr.SetType(header.ICMPv6DstUnreachable) - icmpHdr.SetCode(code) - icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ - Header: icmpHdr[:header.ICMPv6DstUnreachableMinimumSize], - Src: newIPHdr.SourceAddressSlice(), - Dst: newIPHdr.DestinationAddressSlice(), - PayloadCsum: checksum.Checksum(payload, 0), - PayloadLen: len(payload), - })) - copy(icmpHdr.Payload(), payload) - return buffer, true -} - -func BuildUnreachable(packet []byte, source netip.Addr, headroom int) ([]byte, bool) { - switch header.IPVersion(packet) { - case header.IPv4Version: - ipHdr := header.IPv4(packet) - if !ipHdr.IsValid(len(packet)) || ipHdr.FragmentOffset() != 0 { - return nil, false - } - sourceAddr := ipHdr.SourceAddr() - if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { - return nil, false - } - if ipHdr.TransportProtocol() == header.ICMPv4ProtocolNumber { - if len(ipHdr.Payload()) < header.ICMPv4MinimumSize || header.ICMPv4(ipHdr.Payload()).Type() != header.ICMPv4Echo { - return nil, false - } - } - replySource := ipHdr.DestinationAddr() - if source.Is4() { - replySource = source - } - return buildRejectICMPv4(ipHdr, header.ICMPv4HostUnreachable, replySource, headroom) - case header.IPv6Version: - ipHdr := header.IPv6(packet) - if !ipHdr.IsValid(len(packet)) { - return nil, false - } - sourceAddr := ipHdr.SourceAddr() - if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { - return nil, false - } - if ipHdr.TransportProtocol() == header.ICMPv6ProtocolNumber { - if len(ipHdr.Payload()) < header.ICMPv6MinimumSize || header.ICMPv6(ipHdr.Payload()).Type() != header.ICMPv6EchoRequest { - return nil, false - } - } - replySource := ipHdr.DestinationAddr() - if source.Is6() { - replySource = source - } - return buildRejectICMPv6(ipHdr, header.ICMPv6NetworkUnreachable, replySource, headroom) - default: - return nil, false - } -} diff --git a/icmp_error.go b/icmp_error.go new file mode 100644 index 00000000..88a41365 --- /dev/null +++ b/icmp_error.go @@ -0,0 +1,203 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + + "github.com/sagernet/sing-tun/gtcpip/header" +) + +type ICMPError uint8 + +const ( + ICMPErrorNoRoute ICMPError = iota + ICMPErrorAddressUnreachable + ICMPErrorPortUnreachable + ICMPErrorSourcePolicy + ICMPErrorHopLimitExceeded + ICMPErrorPacketTooBig +) + +func BuildICMPError(packet []byte, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) ([]byte, bool) { + switch header.IPVersion(packet) { + case header.IPv4Version: + ipHdr := header.IPv4(packet) + if !ipHdr.IsValid(len(packet)) || ipHdr.FragmentOffset() != 0 { + return nil, false + } + sourceAddr := ipHdr.SourceAddr() + if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { + return nil, false + } + destinationAddr := ipHdr.DestinationAddr() + if destinationAddr.IsMulticast() || destinationAddr == netip.AddrFrom4([4]byte{0xff, 0xff, 0xff, 0xff}) { + return nil, false + } + if ipHdr.TransportProtocol() == header.ICMPv4ProtocolNumber { + if len(ipHdr.Payload()) < header.ICMPv4MinimumSize { + return nil, false + } + switch header.ICMPv4(ipHdr.Payload()).Type() { + case header.ICMPv4DstUnreachable, header.ICMPv4SrcQuench, header.ICMPv4Redirect, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: + return nil, false + } + } + if !source.Is4() { + source = destinationAddr + } + return buildICMPv4Error(ipHdr, errorType, source, mtu, headroom), true + case header.IPv6Version: + ipHdr := header.IPv6(packet) + if !ipHdr.IsValid(len(packet)) { + return nil, false + } + sourceAddr := ipHdr.SourceAddr() + if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { + return nil, false + } + destinationAddr := ipHdr.DestinationAddr() + if destinationAddr.IsMulticast() && errorType != ICMPErrorPacketTooBig { + return nil, false + } + if ipHdr.TransportProtocol() == header.ICMPv6ProtocolNumber { + if len(ipHdr.Payload()) < header.ICMPv6MinimumSize { + return nil, false + } + icmpType := header.ICMPv6(ipHdr.Payload()).Type() + if icmpType < header.ICMPv6EchoRequest || icmpType == header.ICMPv6RedirectMsg { + return nil, false + } + } + if !source.Is6() || source.Is4In6() { + source = destinationAddr + } + return buildICMPv6Error(ipHdr, errorType, source, mtu, headroom), true + default: + return nil, false + } +} + +func buildICMPv4Error(packet header.IPv4, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) []byte { + icmpType := header.ICMPv4DstUnreachable + var icmpCode header.ICMPv4Code + switch errorType { + case ICMPErrorNoRoute, ICMPErrorAddressUnreachable: + icmpCode = header.ICMPv4HostUnreachable + case ICMPErrorPortUnreachable: + icmpCode = header.ICMPv4PortUnreachable + case ICMPErrorSourcePolicy: + icmpCode = header.ICMPv4AdminProhibited + case ICMPErrorHopLimitExceeded: + icmpType = header.ICMPv4TimeExceeded + icmpCode = header.ICMPv4TTLExceeded + case ICMPErrorPacketTooBig: + icmpCode = header.ICMPv4FragmentationNeeded + } + const maxPayloadLength = header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - header.ICMPv4MinimumSize + payloadLength := min(int(packet.TotalLength()), len(packet), maxPayloadLength) + size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + payloadLength + buffer := make([]byte, headroom+size) + response := header.IPv4(buffer[headroom:]) + response.Encode(&header.IPv4Fields{ + TotalLength: uint16(size), + TTL: synthesizedTTL, + Protocol: uint8(header.ICMPv4ProtocolNumber), + SrcAddr: source, + DstAddr: packet.SourceAddr(), + }) + response.SetChecksum(^response.CalculateChecksum()) + icmpHdr := header.ICMPv4(response.Payload()) + icmpHdr.SetType(icmpType) + icmpHdr.SetCode(icmpCode) + if errorType == ICMPErrorPacketTooBig { + icmpHdr.SetMTU(uint16(min(max(mtu, header.IPv4MinimumMTU), 0xffff))) + } + copy(icmpHdr.Payload(), packet[:payloadLength]) + icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) + return buffer +} + +func buildICMPv6Error(packet header.IPv6, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) []byte { + icmpType := header.ICMPv6DstUnreachable + var icmpCode header.ICMPv6Code + switch errorType { + case ICMPErrorNoRoute: + icmpCode = header.ICMPv6NetworkUnreachable + case ICMPErrorAddressUnreachable: + icmpCode = header.ICMPv6AddressUnreachable + case ICMPErrorPortUnreachable: + icmpCode = header.ICMPv6PortUnreachable + case ICMPErrorSourcePolicy: + icmpCode = header.ICMPv6Policy + case ICMPErrorHopLimitExceeded: + icmpType = header.ICMPv6TimeExceeded + icmpCode = header.ICMPv6HopLimitExceeded + case ICMPErrorPacketTooBig: + icmpType = header.ICMPv6PacketTooBig + icmpCode = header.ICMPv6UnusedCode + } + const maxPayloadLength = header.IPv6MinimumMTU - header.IPv6MinimumSize - header.ICMPv6ErrorHeaderSize + payloadLength := min(header.IPv6MinimumSize+int(packet.PayloadLength()), len(packet), maxPayloadLength) + size := header.IPv6MinimumSize + header.ICMPv6ErrorHeaderSize + payloadLength + buffer := make([]byte, headroom+size) + response := header.IPv6(buffer[headroom:]) + response.Encode(&header.IPv6Fields{ + PayloadLength: uint16(header.ICMPv6ErrorHeaderSize + payloadLength), + TransportProtocol: header.ICMPv6ProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: source, + DstAddr: packet.SourceAddr(), + }) + icmpHdr := header.ICMPv6(response.Payload()) + icmpHdr.SetType(icmpType) + icmpHdr.SetCode(icmpCode) + if errorType == ICMPErrorPacketTooBig { + icmpHdr.SetMTU(max(mtu, header.IPv6MinimumMTU)) + } + copy(icmpHdr[header.ICMPv6ErrorHeaderSize:], packet[:payloadLength]) + icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmpHdr, + Src: response.SourceAddressSlice(), + Dst: response.DestinationAddressSlice(), + })) + return buffer +} + +func IPTransportProtocol(packet []byte) (uint8, bool) { + switch header.IPVersion(packet) { + case header.IPv4Version: + ipHdr := header.IPv4(packet) + if !ipHdr.IsValid(len(packet)) { + return 0, false + } + return ipHdr.Protocol(), true + case header.IPv6Version: + ipHdr := header.IPv6(packet) + if !ipHdr.IsValid(len(packet)) { + return 0, false + } + protocol := ipHdr.NextHeader() + payload := ipHdr.Payload() + for { + var fragment, transportPresent bool + protocol, payload, fragment, transportPresent = skipIPv6ExtensionHeaders(protocol, payload) + if transportPresent { + return protocol, true + } + if !fragment || len(payload) < header.IPv6FragmentHeaderSize { + return 0, false + } + protocol = payload[0] + if binary.BigEndian.Uint16(payload[2:])>>3 != 0 { + switch header.IPv6ExtensionHeaderIdentifier(protocol) { + case header.IPv6HopByHopOptionsExtHdrIdentifier, header.IPv6RoutingExtHdrIdentifier, header.IPv6DestinationOptionsExtHdrIdentifier, header.IPv6FragmentExtHdrIdentifier: + return 0, false + } + return protocol, true + } + payload = payload[header.IPv6FragmentHeaderSize:] + } + default: + return 0, false + } +} diff --git a/stack_go_integration_memory_test.go b/stack_go_integration_memory_test.go index 0bce082c..b33bd365 100644 --- a/stack_go_integration_memory_test.go +++ b/stack_go_integration_memory_test.go @@ -525,7 +525,7 @@ func TestGoMemoryPathMTU(t *testing.T) { if header.IPVersion(packet) != header.IPv6Version || len(packet) <= header.IPv6MinimumMTU { return true } - reply, built := buildPacketTooBig(header.IPv6(packet), header.IPv6MinimumMTU, 0) + reply, built := BuildICMPError(packet, ICMPErrorPacketTooBig, netip.Addr{}, header.IPv6MinimumMTU, 0) if built { tooBig.Add(1) fixture.memoryTun.WritePackets([][]byte{reply}) diff --git a/stack_go_udp.go b/stack_go_udp.go index dd1ca0f8..86760cd1 100644 --- a/stack_go_udp.go +++ b/stack_go_udp.go @@ -109,7 +109,7 @@ func (e *goEngine) demuxUDP(packetBuffer *buf.Buffer, meta ForwardFrameMeta, par switch parsed.verdict.Action { case ActionReject: meta.completeChecksum(packet) - reply, ok := BuildUnreachable(packet, netip.Addr{}, 0) + reply, ok := BuildICMPError(packet, ICMPErrorNoRoute, netip.Addr{}, 0, 0) if ok { err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) if err != nil { @@ -470,7 +470,7 @@ func (w *GoPacketConn) HandshakeFailure(err error) error { return os.ErrInvalid } snapshotMeta.completeChecksum(snapshot) - reply, ok := BuildUnreachable(snapshot, netip.Addr{}, 0) + reply, ok := BuildICMPError(snapshot, ICMPErrorNoRoute, netip.Addr{}, 0, 0) if !ok { return nil } From a1811985451529c9bd1837e3c7116ab250d1585d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 22 Sep 2026 18:52:47 +0800 Subject: [PATCH 17/25] Allow IPv6 neighbor discovery under strict route on Windows --- internal/winsys/constants.go | 22 ++++++++++++++- internal/winsys/types_windows.go | 4 +++ tun_windows.go | 46 +++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/internal/winsys/constants.go b/internal/winsys/constants.go index 1173ea9a..03e6e65e 100644 --- a/internal/winsys/constants.go +++ b/internal/winsys/constants.go @@ -103,6 +103,25 @@ var FWPM_CONDITION_IP_REMOTE_PORT = windows.GUID{ Data4: [8]byte{0x91, 0xb4, 0x68, 0xf6, 0x74, 0xee, 0x67, 0x4b}, } +var FWPM_CONDITION_IP_LOCAL_PORT = windows.GUID{ + Data1: 0x0c1ba1af, + Data2: 0x5765, + Data3: 0x453f, + Data4: [8]byte{0xaf, 0x22, 0xa8, 0xf7, 0x91, 0xac, 0x77, 0x5b}, +} + +var FWPM_CONDITION_IP_REMOTE_ADDRESS = windows.GUID{ + Data1: 0xb235ae9a, + Data2: 0x1d64, + Data3: 0x49b8, + Data4: [8]byte{0xa4, 0x4c, 0x5f, 0xf3, 0xd9, 0x09, 0x50, 0x45}, +} + +var ( + FWPM_CONDITION_ICMP_TYPE = FWPM_CONDITION_IP_LOCAL_PORT + FWPM_CONDITION_ICMP_CODE = FWPM_CONDITION_IP_REMOTE_PORT +) + var FWPM_LAYER_ALE_AUTH_CONNECT_V4 = windows.GUID{ Data1: 0xc38d57d1, Data2: 0x05a7, @@ -132,7 +151,8 @@ var FWPM_CONDITION_ALE_APP_ID = windows.GUID{ } const ( - IPPROTO_UDP uint32 = 17 + IPPROTO_UDP uint32 = 17 + IPPROTO_ICMPV6 uint32 = 58 ) const ( diff --git a/internal/winsys/types_windows.go b/internal/winsys/types_windows.go index af6f3c04..01ca6f3e 100644 --- a/internal/winsys/types_windows.go +++ b/internal/winsys/types_windows.go @@ -68,6 +68,10 @@ type FWP_VALUE0 struct { type FWP_CONDITION_VALUE0 FWP_VALUE0 +type FWP_BYTE_ARRAY16 struct { + ByteArray16 [16]uint8 +} + type FWPM_FILTER_CONDITION0 struct { FieldKey windows.GUID // Windows type: GUID MatchType uint32 diff --git a/tun_windows.go b/tun_windows.go index 82cd29e6..c612bd16 100644 --- a/tun_windows.go +++ b/tun_windows.go @@ -22,7 +22,10 @@ import ( "golang.org/x/sys/windows" ) -var TunnelType = "sing-tun" +var ( + TunnelType = "sing-tun" + ipv6AllRoutersMulticast = winsys.FWP_BYTE_ARRAY16{ByteArray16: [16]uint8{0xff, 0x02, 15: 0x02}} +) type NativeTun struct { adapter *wintun.Adapter @@ -273,6 +276,47 @@ func (t *NativeTun) Start() error { }*/ if len(t.options.Inet6Address) == 0 { + ndpCondition := make([]winsys.FWPM_FILTER_CONDITION0, 4) + ndpCondition[0].FieldKey = winsys.FWPM_CONDITION_IP_PROTOCOL + ndpCondition[0].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[0].ConditionValue.Type = winsys.FWP_UINT8 + ndpCondition[0].ConditionValue.Value = uintptr(winsys.IPPROTO_ICMPV6) + ndpCondition[1].FieldKey = winsys.FWPM_CONDITION_ICMP_TYPE + ndpCondition[1].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[1].ConditionValue.Type = winsys.FWP_UINT16 + ndpCondition[2].FieldKey = winsys.FWPM_CONDITION_ICMP_CODE + ndpCondition[2].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[2].ConditionValue.Type = winsys.FWP_UINT16 + ndpCondition[2].ConditionValue.Value = 0 + ndpCondition[3].FieldKey = winsys.FWPM_CONDITION_IP_REMOTE_ADDRESS + ndpCondition[3].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[3].ConditionValue.Type = winsys.FWP_BYTE_ARRAY16_TYPE + ndpCondition[3].ConditionValue.Value = uintptr(unsafe.Pointer(&ipv6AllRoutersMulticast)) + for _, ndpMessage := range []struct { + name string + icmpType uint16 + numConditions uint32 + }{ + {"router solicitation", 133, 4}, + {"neighbor solicitation", 135, 3}, + {"neighbor advertisement", 136, 3}, + } { + ndpCondition[1].ConditionValue.Value = uintptr(ndpMessage.icmpType) + ndpFilter := winsys.FWPM_FILTER0{} + ndpFilter.FilterCondition = &ndpCondition[0] + ndpFilter.NumFilterConditions = ndpMessage.numConditions + ndpFilter.DisplayData = winsys.CreateDisplayData(TunnelType, "allow ipv6 "+ndpMessage.name) + ndpFilter.SubLayerKey = subLayerKey + ndpFilter.LayerKey = winsys.FWPM_LAYER_ALE_AUTH_CONNECT_V6 + ndpFilter.Action.Type = winsys.FWP_ACTION_PERMIT + ndpFilter.Weight.Type = winsys.FWP_UINT8 + ndpFilter.Weight.Value = uintptr(13) + err = winsys.FwpmFilterAdd0(engine, &ndpFilter, 0, &filterId) + if err != nil { + return os.NewSyscallError("FwpmFilterAdd0", err) + } + } + blockFilter := winsys.FWPM_FILTER0{} blockFilter.DisplayData = winsys.CreateDisplayData(TunnelType, "block ipv6") blockFilter.SubLayerKey = subLayerKey From ad6b3366ac650e671b1849247d664e184fd082db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Thu, 24 Sep 2026 15:34:34 +0800 Subject: [PATCH 18/25] Fix auto-redirect interfering with bridge --- redirect_iptables.go | 48 +++++++++++++++++++++++++++++++------- redirect_nftables.go | 16 +++++++++++++ redirect_nftables_rules.go | 1 + tun.go | 1 + 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/redirect_iptables.go b/redirect_iptables.go index ba7d7773..05b7a43d 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -287,6 +287,13 @@ func (r *autoRedirect) updateIPTablesLocalAddresses() error { return nil } +func (r *autoRedirect) iptablesAddOwnIngressReturn(builder *iptablesBuilder) { + builder.add("-i", r.tunOptions.Name, "-j", "RETURN") + for _, name := range r.tunOptions.BridgeInterface { + builder.add("-i", name, "-j", "RETURN") + } +} + func (r *autoRedirect) iptablesAddExcludeRules(builder *iptablesBuilder, hook iptablesHook, kind iptablesChainKind, dnsHijack bool) { options := r.tunOptions mask := r.effectiveMarkMask() @@ -302,7 +309,7 @@ func (r *autoRedirect) iptablesAddExcludeRules(builder *iptablesBuilder, hook ip builder.add("-p", "tcp", "-m", "connmark", "--mark", iptablesMark(options.AutoRedirectInputMark, mask), "-j", "RETURN") } if hook == iptablesHookPrerouting { - builder.add("-i", options.Name, "-j", "RETURN") + r.iptablesAddOwnIngressReturn(builder) builder.includeOnly(common.Map(options.IncludeInterface, func(it string) iptablesMatch { return iptablesMatch{positive: []string{"-i", it}, negative: []string{"!", "-i", it}} })) @@ -357,7 +364,7 @@ func (r *autoRedirect) iptablesAddPreMatchRules(builder *iptablesBuilder, hook i } } else { builder.add("-i", "lo", "-j", "RETURN") - builder.add("-i", options.Name, "-j", "RETURN") + r.iptablesAddOwnIngressReturn(builder) } builder.add("-m", "conntrack", "--ctdir", "REPLY", "-j", "RETURN") if builder.family.tproxy { @@ -516,7 +523,7 @@ func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { } } preroutingMarkChain := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-mark") - preroutingMarkChain.add("-i", options.Name, "-j", "RETURN") + r.iptablesAddOwnIngressReturn(preroutingMarkChain) if family.isIPv6 && !family.tproxy { preroutingMarkChain.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") } @@ -543,7 +550,7 @@ func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { // through tcp_fwmark_accept, and netd's VPN rule then routes its replies // into the tun. preroutingLocal := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-local") - preroutingLocal.add("-i", options.Name, "-j", "RETURN") + r.iptablesAddOwnIngressReturn(preroutingLocal) preroutingLocal.add("-m", "connmark", "!", "--mark", iptablesMark(0, mask), "-j", "RETURN") preroutingLocal.add("-j", "MARK", "--set-xmark", outputMark) localInclude := preroutingLocal.branch("include") @@ -777,8 +784,33 @@ func (r *autoRedirect) cleanupIPTables() { } } +var ( + iptablesChainSuffixes = []string{ + "forward", "input", "output", "output-loopback", "output-mark", "output-prematch", + "output-protect", "output-tproxy", "postrouting", "prerouting", "prerouting-local", + "prerouting-loopback", "prerouting-mark", "prerouting-prematch", "prerouting-tproxy", + "reset", "unreachable-forward", "unreachable-output", + } + iptablesBranchPrefixes = []string{"cont", "include", "local", "redirect"} +) + +func (r *autoRedirect) ownsIPTablesChain(name string) bool { + suffix, found := strings.CutPrefix(name, r.tableName+"-") + if !found { + return false + } + if slices.Contains(iptablesChainSuffixes, suffix) { + return true + } + prefix, counter, found := strings.Cut(suffix, "-") + if !found || !slices.Contains(iptablesBranchPrefixes, prefix) { + return false + } + _, err := strconv.ParseUint(counter, 10, 32) + return err == nil +} + func (r *autoRedirect) cleanupIPTablesForFamily(iptablesPath string) { - prefix := r.tableName + "-" for _, table := range []string{iptablesTableNAT, iptablesTableMangle, iptablesTableFilter} { output, err := iptablesRunOutput(iptablesPath, "-t", table, "-S") if err != nil { @@ -793,15 +825,15 @@ func (r *autoRedirect) cleanupIPTablesForFamily(iptablesPath string) { } switch fields[0] { case "-N": - if strings.HasPrefix(fields[1], prefix) { + if r.ownsIPTablesChain(fields[1]) { chains = append(chains, fields[1]) } case "-A": - if strings.HasPrefix(fields[1], prefix) { + if r.ownsIPTablesChain(fields[1]) { continue } for index := 2; index < len(fields)-1; index++ { - if fields[index] == "-j" && strings.HasPrefix(fields[index+1], prefix) { + if fields[index] == "-j" && r.ownsIPTablesChain(fields[index+1]) { references = append(references, fields[1:]) break } diff --git a/redirect_nftables.go b/redirect_nftables.go index d3e00c90..b491a903 100644 --- a/redirect_nftables.go +++ b/redirect_nftables.go @@ -217,6 +217,7 @@ func (r *autoRedirect) setupNFTables() error { }, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chainPreRoutingUDP) nft.AddRule(&nftables.Rule{ Table: table, Chain: chainPreRoutingUDP, @@ -424,6 +425,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Verdict{Kind: expr.VerdictReturn}, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chain) } nft.AddRule(&nftables.Rule{ Table: table, @@ -607,3 +609,17 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab } return nil } + +func (r *autoRedirect) nftablesAddBridgeIngressReturn(nft *nftables.Conn, table *nftables.Table, chain *nftables.Chain) { + for _, name := range r.tunOptions.BridgeInterface { + nft.AddRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: nftablesIfname(name)}, + &expr.Verdict{Kind: expr.VerdictReturn}, + }, + }) + } +} diff --git a/redirect_nftables_rules.go b/redirect_nftables_rules.go index e74c43df..69fce8af 100644 --- a/redirect_nftables_rules.go +++ b/redirect_nftables_rules.go @@ -201,6 +201,7 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chain) if len(r.tunOptions.IncludeInterface) > 0 { if len(r.tunOptions.IncludeInterface) > 1 { includeInterface := &nftables.Set{ diff --git a/tun.go b/tun.go index 12f70651..3e1239cf 100644 --- a/tun.go +++ b/tun.go @@ -99,6 +99,7 @@ type Options struct { Inet6RouteExcludeAddress []netip.Prefix IncludeInterface []string ExcludeInterface []string + BridgeInterface []string IncludeUID []ranges.Range[uint32] ExcludeUID []ranges.Range[uint32] IncludeAndroidUser []int From 43861a8c2b56eb9232d5635e82bbbf39246892f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Fri, 25 Sep 2026 19:24:05 +0800 Subject: [PATCH 19/25] Add per-destination outbound queues to memory tun --- stack_go.go | 2 +- stack_go_conn.go | 4 +- stack_go_engine.go | 27 +-- stack_go_integration_memory_test.go | 10 +- stack_go_integration_test.go | 48 +----- stack_go_io.go | 6 +- stack_go_io_darwin.go | 20 ++- stack_go_io_linux.go | 10 +- stack_go_io_memory.go | 180 ++++++++------------ stack_go_io_windows.go | 10 +- stack_go_tcp_input.go | 93 +++++++---- stack_go_tcp_output.go | 2 +- stack_mixed.go | 2 +- stack_system.go | 2 +- tun.go | 2 +- tun_darwin.go | 16 +- tun_memory.go | 208 ++++++----------------- tun_memory_queue.go | 248 ++++++++++++++++++++++++++++ 18 files changed, 500 insertions(+), 390 deletions(-) create mode 100644 tun_memory_queue.go diff --git a/stack_go.go b/stack_go.go index 982f7d8c..6c6bf19f 100644 --- a/stack_go.go +++ b/stack_go.go @@ -76,7 +76,7 @@ func NewGo(options StackOptions) (*Go, error) { if isMemoryTun { stack.validateChecksum = true stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { - return []goPlatformIO{&goMemoryIO{stack: current, tun: memoryTun, handed: make([]*buf.Buffer, 0, goReadBatch)}}, nil + return []goPlatformIO{&goMemoryIO{stack: current, tun: memoryTun, handed: make([]*buf.Buffer, 0, goReadBatch), blocked: make(map[*OutboundQueue]*goBlockedWriters)}}, nil } } return stack, nil diff --git a/stack_go_conn.go b/stack_go_conn.go index 1a09fbcc..1c2251e1 100644 --- a/stack_go_conn.go +++ b/stack_go_conn.go @@ -181,13 +181,14 @@ type GoConn struct { finAcked bool ackForced bool ackDirty bool - onBlockedList bool onDyingList bool clientISN uint32 sendISN uint32 finOffset uint64 dyingSince int64 ackNext *GoConn + blockedOn *goBlockedWriters + blockedPrev *GoConn blockedNext *GoConn dyingNext *GoConn splice *goSpliceStream @@ -225,6 +226,7 @@ type GoConn struct { dataSegmentsOut atomic.Uint32 ident atomic.Uint32 windowLimitedSince atomic.Bool + outboundQueue atomic.Pointer[OutboundQueue] transmitStore goTransmitStore descriptors goDescriptorRing transmitScratch [goHeaderScratchSize]byte diff --git a/stack_go_engine.go b/stack_go_engine.go index f017eb8d..fe7ce1ff 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -148,7 +148,7 @@ type goEngine struct { exitSignal chan struct{} wokeHandlerThisBurst bool tunPending bool - transmitReady bool + transmitWritable []*goBlockedWriters readBuffersHeld bool idleSince int64 parkedNanos int64 @@ -181,8 +181,7 @@ type goEngine struct { descriptorPool goDescriptorPool sequenceSeed maphash.Seed ackList *GoConn - blockedList *GoConn - blockedTail *GoConn + releasing goBlockedWriters dyingList *GoConn reclaimPending bool resetBurst int @@ -261,11 +260,12 @@ func (e *goEngine) run() { if e.stack.closed.Load() { return } - transmitWritable := e.platformIO.takeTransmitWritable() - if e.transmitReady || transmitWritable { - e.transmitReady = false - e.releaseBlockedWriters() + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + for _, waiters := range e.transmitWritable { + e.releaseBlockedWriters(waiters) } + clear(e.transmitWritable) + e.transmitWritable = e.transmitWritable[:0] if err != nil { if !errors.Is(err, os.ErrClosed) { e.stack.logger.Error(E.Cause(err, "go: engine wait")) @@ -371,8 +371,8 @@ func (e *goEngine) park() (bool, int, error) { timeout = 0 } tunReadable, eventCount, err := e.platformIO.wait(0, e.socketEvents) - e.transmitReady = e.platformIO.takeTransmitWritable() - if timeout != 0 && !tunReadable && eventCount == 0 && !e.transmitReady && err == nil && e.controlStack.head.Load() == nil { + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + if timeout != 0 && !tunReadable && eventCount == 0 && len(e.transmitWritable) == 0 && err == nil && e.controlStack.head.Load() == nil { tunReadable, eventCount, err = e.platformIO.wait(e.idleTimeout(timeout), e.socketEvents) } if tunReadable || eventCount > 0 { @@ -590,14 +590,17 @@ func (e *goEngine) shutdown() { if remaining <= 0 { break } - armed, err := e.platformIO.armTransmitWritable() - if err != nil || !armed { + waiters, err := e.platformIO.armTransmitWritable(nil) + if err != nil || waiters == nil { time.Sleep(time.Millisecond) } else { e.platformIO.wait(min(time.Duration(remaining), 10*time.Millisecond), nil) - if !e.platformIO.takeTransmitWritable() { + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + if len(e.transmitWritable) == 0 { time.Sleep(time.Millisecond) } + clear(e.transmitWritable) + e.transmitWritable = e.transmitWritable[:0] } kept := unreset[:0] for _, conn := range unreset { diff --git a/stack_go_integration_memory_test.go b/stack_go_integration_memory_test.go index b33bd365..305c2eb7 100644 --- a/stack_go_integration_memory_test.go +++ b/stack_go_integration_memory_test.go @@ -397,7 +397,7 @@ func TestGoMemoryNoHandler(t *testing.T) { } func TestGoMemoryOutboundReply(t *testing.T) { - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) for _, ipv6 := range []bool{false, true} { t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { client, conn, destination := fixture.packetPair(test, ipv6) @@ -427,7 +427,7 @@ func TestGoMemoryOutboundReply(t *testing.T) { } func TestGoMemoryOutboundBound(t *testing.T) { - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) client, conn, destination := fixture.packetPair(t, false) err := client.SetReadBuffer(4 << 20) if err != nil { @@ -442,7 +442,7 @@ func TestGoMemoryOutboundBound(t *testing.T) { options.PostReturn(packet) return conn.WritePacket(packet, destination) } - capacity := len(fixture.memoryTun.datagram.packets) + capacity := fixture.memoryTun.outbound.datagram.capacity for range 4 * capacity { err = sendReply() if err != nil { @@ -479,13 +479,13 @@ func TestGoMemoryOutboundBound(t *testing.T) { } func TestGoMemoryControlPriority(t *testing.T) { - fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, memoryOutbound: true}) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) _, conn, destination := fixture.packetPair(t, false) listener, address := kernelListen(t, fixture, false) fixture.bridge.pauseOutbound(true) options := N.NewReadWaitOptions(nil, conn) payload := kernelPayload(100, 113) - for range 4 * len(fixture.memoryTun.datagram.packets) { + for range 4 * fixture.memoryTun.outbound.datagram.capacity { packet := options.NewBufferSize(len(payload)) packet.Write(payload) options.PostReturn(packet) diff --git a/stack_go_integration_test.go b/stack_go_integration_test.go index 1d54efea..cf31f6d2 100644 --- a/stack_go_integration_test.go +++ b/stack_go_integration_test.go @@ -10,7 +10,6 @@ import ( "io" "net" "net/netip" - "os" "runtime" "sync" "sync/atomic" @@ -35,7 +34,6 @@ type kernelStackConfig struct { gso bool multiQueue bool memoryLink bool - memoryOutbound bool noHandler bool prepare func(*testing.T, *Options) configure func(*testing.T, Options) @@ -123,7 +121,7 @@ func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackF var stackTun Tun stackTun = device if config.memoryLink { - fixture.bridge = newMemoryBridge(t, device, int(fixture.options.MTU), config.memoryOutbound) + fixture.bridge = newMemoryBridge(t, device, int(fixture.options.MTU)) fixture.memoryTun = fixture.bridge.memoryTun stackTun = fixture.memoryTun } @@ -200,20 +198,13 @@ func (g *bridgeGate) pause(paused bool) { g.resume.Broadcast() } -func newMemoryBridge(t *testing.T, device Tun, mtu int, callback bool) *memoryBridge { +func newMemoryBridge(t *testing.T, device Tun, mtu int) *memoryBridge { bridge := &memoryBridge{device: device, done: make(chan struct{})} bridge.inbound.resume = sync.NewCond(&bridge.inbound.access) bridge.outbound.resume = sync.NewCond(&bridge.outbound.access) - memoryOptions := MemoryTunOptions{MTU: mtu} - if callback { - memoryOptions.Outbound = bridge.writeOutbound - } - bridge.memoryTun = NewMemoryTun(memoryOptions) + bridge.memoryTun = NewMemoryTun(MemoryTunOptions{MTU: mtu, Outbound: bridge.writeOutbound}) t.Cleanup(func() { bridge.memoryTun.Close() }) go bridge.pumpInbound() - if !callback { - go bridge.pumpOutbound() - } t.Cleanup(func() { close(bridge.done) bridge.inbound.pause(false) @@ -251,49 +242,22 @@ func (b *memoryBridge) pumpInbound() { } } -func (b *memoryBridge) pumpOutbound() { - storage := make([][]byte, 64) - for index := range storage { - storage[index] = make([]byte, 65536+PacketOffset) - } - sizes := make([]int, len(storage)) - for { - count, err := b.memoryTun.ReadPackets(storage, sizes, PacketOffset) - if err != nil { - return - } - b.outbound.pass() - select { - case <-b.done: - return - default: - } - for index := range count { - err = b.forwardOutbound(storage[index][:PacketOffset+sizes[index]]) - if err != nil { - return - } - } - } -} - -func (b *memoryBridge) writeOutbound(packets []*buf.Buffer) error { +func (b *memoryBridge) writeOutbound(packets []*buf.Buffer) { defer buf.ReleaseMulti(packets) for _, packet := range packets { b.outbound.pass() select { case <-b.done: - return os.ErrClosed + return default: } frame := make([]byte, PacketOffset+packet.Len()) copy(frame[PacketOffset:], packet.Bytes()) err := b.forwardOutbound(frame) if err != nil { - return err + return } } - return nil } func (b *memoryBridge) forwardOutbound(frame []byte) error { diff --git a/stack_go_io.go b/stack_go_io.go index 1ef46e7c..9e28f88f 100644 --- a/stack_go_io.go +++ b/stack_go_io.go @@ -49,7 +49,7 @@ type goPlatformIO interface { writeDatagram(packet []byte, meta ForwardFrameMeta) error writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error writePacketBatch(frames []goUDPFrame) error - transmitBacklogBelowBatch() bool + transmitBacklogBelowBatch(conn *GoConn) bool releaseReadBuffers() flush() mtu() int @@ -57,8 +57,8 @@ type goPlatformIO interface { transmitPrefix() int transmitChecksumOffload() bool transmitSegmentOffload() bool - armTransmitWritable() (bool, error) - takeTransmitWritable() bool + armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) + takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters wake() close() error } diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go index 3272eccb..1d13e702 100644 --- a/stack_go_io_darwin.go +++ b/stack_go_io_darwin.go @@ -62,6 +62,7 @@ type goDarwinIO struct { wakeEvent [1]unix.Kevent_t transmitWritableEvent [1]unix.Kevent_t transmitWritable atomic.Bool + blocked goBlockedWriters receiveBatch int transmitIovecs []unix.Iovec socketTokens []uint32 @@ -456,7 +457,7 @@ func (o *goDarwinIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { return o.writePacket(packet, meta) } -func (o *goDarwinIO) transmitBacklogBelowBatch() bool { +func (o *goDarwinIO) transmitBacklogBelowBatch(conn *GoConn) bool { return true } @@ -751,24 +752,27 @@ func (o *goDarwinIO) transmitSegmentOffload() bool { return false } -func (o *goDarwinIO) armTransmitWritable() (bool, error) { +func (o *goDarwinIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { o.pollAccess.RLock() defer o.pollAccess.RUnlock() if o.closing.Load() { - return false, os.ErrClosed + return nil, os.ErrClosed } if o.gateBlocked.Load() { - return true, nil + return &o.blocked, nil } _, err := unix.Kevent(o.kqueueFd, o.transmitWritableEvent[:], nil, nil) if err != nil { - return false, E.Cause(err, "go: arm EVFILT_WRITE") + return nil, E.Cause(err, "go: arm EVFILT_WRITE") } - return true, nil + return &o.blocked, nil } -func (o *goDarwinIO) takeTransmitWritable() bool { - return o.transmitWritable.Swap(false) +func (o *goDarwinIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + if !o.transmitWritable.Swap(false) { + return writable + } + return append(writable, &o.blocked) } func (o *goDarwinIO) wake() { diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go index 8d23f36e..6cd412cf 100644 --- a/stack_go_io_linux.go +++ b/stack_go_io_linux.go @@ -485,12 +485,12 @@ func (o *goLinuxIO) transmitSegmentOffload() bool { return o.device.transmitOffload.Load() } -func (o *goLinuxIO) armTransmitWritable() (bool, error) { - return false, nil +func (o *goLinuxIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + return nil, nil } -func (o *goLinuxIO) takeTransmitWritable() bool { - return false +func (o *goLinuxIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + return writable } func (o *goLinuxIO) wake() { @@ -520,7 +520,7 @@ func (o *goLinuxIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { return o.writePacket(packet, meta) } -func (o *goLinuxIO) transmitBacklogBelowBatch() bool { +func (o *goLinuxIO) transmitBacklogBelowBatch(conn *GoConn) bool { return true } diff --git a/stack_go_io_memory.go b/stack_go_io_memory.go index 86cb036c..c319aee7 100644 --- a/stack_go_io_memory.go +++ b/stack_go_io_memory.go @@ -4,26 +4,23 @@ import ( "os" "time" + "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" N "github.com/sagernet/sing/common/network" ) type goMemoryIO struct { - stack *Go - tun *MemoryTun - timer *time.Timer - handed []*buf.Buffer - dropped goDropCounter - stop chan struct{} + stack *Go + tun *MemoryTun + timer *time.Timer + handed []*buf.Buffer + blocked map[*OutboundQueue]*goBlockedWriters + writableSpare []*OutboundQueue } func (o *goMemoryIO) start() error { o.tun.attached.Store(true) - if o.tun.outboundHandler != nil { - o.stop = make(chan struct{}) - go o.deliverOutbound() - } return nil } @@ -112,103 +109,38 @@ const ( ) func (o *goMemoryIO) enqueue(segments [][]byte, class goOutboundClass, owner *GoConn, segmentEnd uint64) error { - length := 0 - for _, segment := range segments { - length += len(segment) - } - o.tun.outboundAccess.Lock() - defer o.tun.outboundAccess.Unlock() if o.tun.closed.Load() || !o.tun.attached.Load() { return os.ErrClosed } - ring := &o.tun.control - switch class { - case goOutboundDatagram: - ring = &o.tun.datagram - case goOutboundData: - ring = &o.tun.data - } - if ring.full() { - if class == goOutboundData { - return errGoTransmitBlocked - } - return errGoFrameDropped - } - if o.tun.control.empty() && o.tun.datagram.empty() && o.tun.data.empty() { - defer o.tun.readerWake.notify() + length := 0 + for _, segment := range segments { + length += len(segment) } packet := o.tun.newOutboundPacket(length) for _, segment := range segments { - packet.Write(segment) - } - mtu := o.tun.MTU() - if length <= mtu { - o.pushOutboundLocked(ring, packet, owner, segmentEnd) - return nil + common.Must1(packet.Write(segment)) } - defer packet.Release() - fragments, fragmentsBuilt := goFragmentPacket(&o.stack.fragmentIdentification, packet.Bytes(), mtu) - if !fragmentsBuilt { - return errGoFrameDropped - } - for _, fragment := range fragments { - if ring.full() { - return errGoFrameDropped + queue := o.tun.outbound + if o.tun.route != nil { + routed := o.tun.route(packet.Bytes()) + if routed != nil { + queue = routed } - fragmentPacket := o.tun.newOutboundPacket(len(fragment)) - fragmentPacket.Write(fragment) - o.pushOutboundLocked(ring, fragmentPacket, owner, segmentEnd) } - return nil -} - -func (o *goMemoryIO) pushOutboundLocked(ring *goPacketRing, packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { if owner != nil { - owner.frameEnqueued(packet.Len()) - o.tun.dataQueued.Add(1) + owner.outboundQueue.Store(queue) } - ring.push(packet, owner, segmentEnd) -} - -func (o *goMemoryIO) deliverOutbound() { - batch := make([]*buf.Buffer, o.tun.batchSize) - for { - o.tun.outboundAccess.Lock() - count := 0 - for count < len(batch) { - buffer, owner, segmentEnd := o.tun.popOutboundLocked() - if buffer == nil { - break - } - if owner != nil { - owner.frameDequeued(buffer.Len(), segmentEnd) - } - batch[count] = buffer - count++ - } - wakeEngine := o.tun.takeTransmitWritableLocked() - o.tun.outboundAccess.Unlock() - if wakeEngine { - o.tun.engineWake.notify() - } - if count > 0 { - err := o.tun.outboundHandler(batch[:count]) - if err != nil { - for range count { - o.dropped.record(o.stack.logger, "outbound frames") - } - } - clear(batch[:count]) - continue - } - select { - case <-o.tun.readerWake: - case <-o.tun.closeSignal: - return - case <-o.stop: - return + var fragments [][]byte + mtu := o.tun.MTU() + if length > mtu { + var fragmentsBuilt bool + fragments, fragmentsBuilt = goFragmentPacket(&o.stack.fragmentIdentification, packet.Bytes(), mtu) + if !fragmentsBuilt { + packet.Release() + return errGoFrameDropped } } + return queue.enqueue(packet, fragments, class, owner, segmentEnd) } func (o *goMemoryIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { @@ -239,8 +171,12 @@ func (o *goMemoryIO) writePacketBatch(frames []goUDPFrame) error { return writeErr } -func (o *goMemoryIO) transmitBacklogBelowBatch() bool { - return int(o.tun.dataQueued.Load()) < o.tun.batchSize +func (o *goMemoryIO) transmitBacklogBelowBatch(conn *GoConn) bool { + queue := conn.outboundQueue.Load() + if queue == nil { + queue = o.tun.outbound + } + return int(queue.dataQueued.Load()) < o.tun.batchSize } func (o *goMemoryIO) flush() { @@ -266,21 +202,45 @@ func (o *goMemoryIO) transmitSegmentOffload() bool { return false } -func (o *goMemoryIO) armTransmitWritable() (bool, error) { - o.tun.outboundAccess.Lock() - defer o.tun.outboundAccess.Unlock() - if !o.tun.data.full() { - return false, nil +func (o *goMemoryIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + queue := o.tun.outbound + if conn != nil { + routed := conn.outboundQueue.Load() + if routed != nil { + queue = routed + } + } + queue.access.Lock() + armed := !queue.closed.Load() && queue.data.full() + if armed { + queue.transmitInterest = true + } + queue.access.Unlock() + if !armed { + return nil, nil } - o.tun.transmitInterest = true - return true, nil + waiters := o.blocked[queue] + if waiters == nil { + waiters = new(goBlockedWriters) + o.blocked[queue] = waiters + } + return waiters, nil } -func (o *goMemoryIO) takeTransmitWritable() bool { - o.tun.outboundAccess.Lock() - defer o.tun.outboundAccess.Unlock() - writable := o.tun.transmitWritable - o.tun.transmitWritable = false +func (o *goMemoryIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + queues := o.tun.takeWritable(o.writableSpare) + for _, queue := range queues { + waiters := o.blocked[queue] + if waiters == nil { + continue + } + if queue.closed.Load() { + delete(o.blocked, queue) + } + writable = append(writable, waiters) + } + clear(queues) + o.writableSpare = queues[:0] return writable } @@ -291,14 +251,12 @@ func (o *goMemoryIO) wake() { func (o *goMemoryIO) close() error { o.releaseReadBuffers() o.tun.attached.Store(false) - if o.stop != nil { - close(o.stop) - } o.tun.inboundAccess.Lock() o.tun.inbound.releaseAll() o.tun.inboundAccess.Unlock() if o.timer != nil { o.timer.Stop() } + clear(o.blocked) return nil } diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go index 134e3bc0..7627542c 100644 --- a/stack_go_io_windows.go +++ b/stack_go_io_windows.go @@ -393,12 +393,12 @@ func (o *goWindowsIO) transmitSegmentOffload() bool { return false } -func (o *goWindowsIO) armTransmitWritable() (bool, error) { - return false, nil +func (o *goWindowsIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + return nil, nil } -func (o *goWindowsIO) takeTransmitWritable() bool { - return false +func (o *goWindowsIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + return writable } func (o *goWindowsIO) wake() { @@ -498,7 +498,7 @@ func (o *goWindowsIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error return o.writePacket(packet, meta) } -func (o *goWindowsIO) transmitBacklogBelowBatch() bool { +func (o *goWindowsIO) transmitBacklogBelowBatch(conn *GoConn) bool { return true } diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index efa29d3a..89d6c6e2 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -1084,44 +1084,80 @@ func (e *goEngine) handleDroppedFrames(conn *GoConn) { conn.publishPermit(conn.sendUnacked.Load()) } +type goBlockedWriters struct { + head *GoConn + tail *GoConn +} + +func (l *goBlockedWriters) push(conn *GoConn) { + conn.blockedOn = l + conn.blockedPrev = l.tail + conn.blockedNext = nil + if l.tail == nil { + l.head = conn + } else { + l.tail.blockedNext = conn + } + l.tail = conn +} + +func (l *goBlockedWriters) remove(conn *GoConn) { + if conn.blockedPrev == nil { + l.head = conn.blockedNext + } else { + conn.blockedPrev.blockedNext = conn.blockedNext + } + if conn.blockedNext == nil { + l.tail = conn.blockedPrev + } else { + conn.blockedNext.blockedPrev = conn.blockedPrev + } + conn.blockedOn = nil + conn.blockedPrev = nil + conn.blockedNext = nil +} + func (e *goEngine) handleTransmitBlocked(conn *GoConn) { if conn.dead { conn.transmitSignal.notify() return } - if !conn.onBlockedList { - conn.onBlockedList = true - if e.blockedTail == nil { - e.blockedList = conn - } else { - e.blockedTail.blockedNext = conn - } - e.blockedTail = conn - } - armed, err := e.platformIO.armTransmitWritable() + waiters, err := e.platformIO.armTransmitWritable(conn) if err != nil { e.stack.logger.Trace(E.Cause(err, "go: arm transmit writable")) } - if !armed { - e.releaseBlockedWriters() + if conn.blockedOn == waiters && waiters != nil { + return + } + if conn.blockedOn != nil { + conn.blockedOn.remove(conn) } + if waiters == nil { + e.releaseBlockedWriter(conn) + return + } + waiters.push(conn) } -func (e *goEngine) releaseBlockedWriters() { - blocked := e.blockedList - e.blockedList = nil - e.blockedTail = nil - for conn := blocked; conn != nil; { - next := conn.blockedNext - conn.blockedNext = nil - conn.onBlockedList = false - if conn.splice != nil || goEngineTransmits { - e.retryBlockedOnEngine(conn) - } else { - conn.transmitSignal.notify() - e.wokeHandlerThisBurst = true - } - conn = next +func (e *goEngine) releaseBlockedWriters(waiters *goBlockedWriters) { + e.releasing = *waiters + *waiters = goBlockedWriters{} + for conn := e.releasing.head; conn != nil; conn = conn.blockedNext { + conn.blockedOn = &e.releasing + } + for e.releasing.head != nil { + conn := e.releasing.head + e.releasing.remove(conn) + e.releaseBlockedWriter(conn) + } +} + +func (e *goEngine) releaseBlockedWriter(conn *GoConn) { + if conn.splice != nil || goEngineTransmits { + e.retryBlockedOnEngine(conn) + } else { + conn.transmitSignal.notify() + e.wokeHandlerThisBurst = true } } @@ -1341,7 +1377,8 @@ func (e *goEngine) detachConn(conn *GoConn, err error, class uint8) { conn.dyingNext = e.dyingList e.dyingList = conn } - if conn.onBlockedList { + if conn.blockedOn != nil { + conn.blockedOn.remove(conn) conn.transmitSignal.notify() } conn.wakeUser() diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go index 8fc4afe2..e7af6dab 100644 --- a/stack_go_tcp_output.go +++ b/stack_go_tcp_output.go @@ -721,7 +721,7 @@ const goQueueLimitShift = 8 func (c *GoConn) queueThrottle(length int) bool { limit := max(2*int64(length), int64(c.pacingRate.Load()>>goQueueLimitShift)) - if c.queuedBytes.Load()+int64(length) <= limit || c.engine.platformIO.transmitBacklogBelowBatch() { + if c.queuedBytes.Load()+int64(length) <= limit || c.engine.platformIO.transmitBacklogBelowBatch(c) { return false } if c.sentTail.Load() == c.sendUnacked.Load() { diff --git a/stack_mixed.go b/stack_mixed.go index 50b53603..ba1b9b7b 100644 --- a/stack_mixed.go +++ b/stack_mixed.go @@ -204,7 +204,7 @@ func (m *Mixed) batchLoopDarwin(darwinTUN DarwinTUN) { var releaseBuffers []*buf.Buffer var readRetry ReadRetry for { - buffers, err := darwinTUN.BatchRead() + buffers, err := darwinTUN.BatchRead(0, 0) if err != nil { if !IsRecoverableReadError(err) { if !E.IsClosed(err) && !errors.Is(err, syscall.EBADF) { diff --git a/stack_system.go b/stack_system.go index 8a01174a..5b34d0af 100644 --- a/stack_system.go +++ b/stack_system.go @@ -326,7 +326,7 @@ func (s *System) batchLoopDarwin(darwinTUN DarwinTUN) { var releaseBuffers []*buf.Buffer var readRetry ReadRetry for { - buffers, err := darwinTUN.BatchRead() + buffers, err := darwinTUN.BatchRead(0, 0) if err != nil { if !IsRecoverableReadError(err) { if !E.IsClosed(err) && !errors.Is(err, syscall.EBADF) { diff --git a/tun.go b/tun.go index 3e1239cf..e7013e37 100644 --- a/tun.go +++ b/tun.go @@ -50,7 +50,7 @@ type LinuxTUN interface { type DarwinTUN interface { Tun - BatchRead() ([]*buf.Buffer, error) + BatchRead(frontHeadroom int, rearHeadroom int) ([]*buf.Buffer, error) BatchWrite(buffers []*buf.Buffer) error } diff --git a/tun_darwin.go b/tun_darwin.go index 2c5ff863..5ede7565 100644 --- a/tun_darwin.go +++ b/tun_darwin.go @@ -59,15 +59,21 @@ func newIovecBuffer(mtu int) iovecBuffer { } } -func (b *iovecBuffer) nextIovecs() []unix.Iovec { +func (b *iovecBuffer) nextIovecs(frontHeadroom int, rearHeadroom int) []unix.Iovec { if b.iovecs[0].Len == 0 { headBuffer := make([]byte, PacketOffset) b.iovecs[0].Base = &headBuffer[0] b.iovecs[0].SetLen(PacketOffset) } + bufferSize := frontHeadroom + b.mtu + rearHeadroom + if b.buffer != nil && (b.buffer.Start() != frontHeadroom || b.buffer.Cap() != bufferSize) { + b.buffer.Release() + b.buffer = nil + } if b.buffer == nil { - b.buffer = buf.NewSize(b.mtu) - b.iovecs[1] = b.buffer.Iovec(b.buffer.Cap()) + b.buffer = buf.NewSize(bufferSize) + b.buffer.Resize(frontHeadroom, 0) + b.iovecs[1] = b.buffer.Iovec(b.mtu) } return b.iovecs } @@ -411,9 +417,9 @@ func (t *NativeTun) enableMaxPendingPackets() error { return configure(t.tunFd, true, int(t.options.MTU)) } -func (t *NativeTun) BatchRead() ([]*buf.Buffer, error) { +func (t *NativeTun) BatchRead(frontHeadroom int, rearHeadroom int) ([]*buf.Buffer, error) { for i := 0; i < t.batchSize; i++ { - iovecs := t.iovecs[i].nextIovecs() + iovecs := t.iovecs[i].nextIovecs(frontHeadroom, rearHeadroom) // Cannot clear only the length field. Older versions of the darwin kernel will check whether other data is empty. // https://github.com/Darm64/XNU/blob/xnu-2782.40.9/bsd/kern/uipc_syscalls.c#L2026-L2048 t.msgHdrs[i] = rawfile.MsgHdrX{} diff --git a/tun_memory.go b/tun_memory.go index 1ba12c45..aff3a1fa 100644 --- a/tun_memory.go +++ b/tun_memory.go @@ -18,7 +18,8 @@ type MemoryTunOptions struct { Headroom int RearSpace int BatchSize int - Outbound func(packets []*buf.Buffer) error + Outbound func(packets []*buf.Buffer) + Route func(packet []byte) *OutboundQueue } type MemoryTun struct { @@ -26,23 +27,18 @@ type MemoryTun struct { headroom int rearSpace int batchSize int - outboundHandler func(packets []*buf.Buffer) error + route func(packet []byte) *OutboundQueue attached atomic.Bool closed atomic.Bool inboundAccess sync.Mutex inbound goPacketRing inboundHeadroom atomic.Int32 inboundRearSpace atomic.Int32 - outboundAccess sync.Mutex - control goPacketRing - datagram goPacketRing - data goPacketRing - dataTurn bool - dataQueued atomic.Int32 - transmitInterest bool - transmitWritable bool + outbound *OutboundQueue + writableAccess sync.Mutex + writable []*OutboundQueue + writablePending atomic.Bool engineWake goSignal - readerWake goSignal closeSignal chan struct{} } @@ -52,18 +48,19 @@ func NewMemoryTun(options MemoryTunOptions) *MemoryTun { batchSize = memoryTunDefaultBatchSize } memoryTun := &MemoryTun{ - headroom: options.Headroom, - rearSpace: options.RearSpace, - batchSize: batchSize, - outboundHandler: options.Outbound, - inbound: newGoPacketRing(4 * batchSize), - control: newGoPacketRing(4 * batchSize), - datagram: newGoPacketRing(4 * batchSize), - data: newGoPacketRing(2 * batchSize), - engineWake: make(goSignal, 1), - readerWake: make(goSignal, 1), - closeSignal: make(chan struct{}), - } + headroom: options.Headroom, + rearSpace: options.RearSpace, + batchSize: batchSize, + route: options.Route, + inbound: newGoPacketRing(4 * batchSize), + engineWake: make(goSignal, 1), + closeSignal: make(chan struct{}), + } + outboundHandler := options.Outbound + if outboundHandler == nil { + outboundHandler = buf.ReleaseMulti + } + memoryTun.outbound = memoryTun.NewOutboundQueue(outboundHandler) memoryTun.mtu.Store(int32(options.MTU)) return memoryTun } @@ -89,13 +86,7 @@ func (t *MemoryTun) UpdateMTU(mtu int) { } func (t *MemoryTun) Read(p []byte) (int, error) { - buffers := [1][]byte{p} - var sizes [1]int - _, err := t.ReadPackets(buffers[:], sizes[:], 0) - if err != nil { - return 0, err - } - return sizes[0], nil + return 0, os.ErrInvalid } func (t *MemoryTun) Write(p []byte) (int, error) { @@ -141,77 +132,6 @@ func (t *MemoryTun) WritePackets(packets [][]byte) (int, error) { return accepted, nil } -func (t *MemoryTun) ReadPackets(buffers [][]byte, sizes []int, offset int) (int, error) { - if len(buffers) == 0 { - return 0, nil - } - if t.outboundHandler != nil { - return 0, os.ErrInvalid - } - for { - t.outboundAccess.Lock() - count := 0 - for count < len(buffers) { - buffer, owner, segmentEnd := t.popOutboundLocked() - if buffer == nil { - break - } - packet := buffer.Bytes() - if owner != nil { - owner.frameDequeued(len(packet), segmentEnd) - } - target := buffers[count][offset:] - if len(packet) > len(target) { - buffer.Release() - continue - } - sizes[count] = copy(target, packet) - buffer.Release() - count++ - } - wakeEngine := t.takeTransmitWritableLocked() - t.outboundAccess.Unlock() - if wakeEngine { - t.engineWake.notify() - } - if count > 0 { - return count, nil - } - if t.closed.Load() { - return 0, os.ErrClosed - } - select { - case <-t.readerWake: - case <-t.closeSignal: - } - } -} - -func (t *MemoryTun) popOutboundLocked() (*buf.Buffer, *GoConn, uint64) { - if !t.control.empty() { - return t.control.pop() - } - if !t.datagram.empty() && (t.data.empty() || !t.dataTurn) { - t.dataTurn = true - return t.datagram.pop() - } - if !t.data.empty() { - t.dataTurn = false - t.dataQueued.Add(-1) - return t.data.pop() - } - return nil, nil, 0 -} - -func (t *MemoryTun) takeTransmitWritableLocked() bool { - if !t.transmitInterest || t.data.full() { - return false - } - t.transmitInterest = false - t.transmitWritable = true - return true -} - func (t *MemoryTun) Close() error { if t.closed.Swap(true) { return nil @@ -219,72 +139,40 @@ func (t *MemoryTun) Close() error { t.inboundAccess.Lock() t.inbound.releaseAll() t.inboundAccess.Unlock() - t.outboundAccess.Lock() - t.control.releaseAll() - t.datagram.releaseAll() - t.data.releaseAll() - t.dataQueued.Store(0) - t.outboundAccess.Unlock() + t.outbound.Close() close(t.closeSignal) return nil } -func (t *MemoryTun) newOutboundPacket(length int) *buf.Buffer { - packet := buf.NewSize(t.headroom + length + t.rearSpace) - packet.Resize(t.headroom, 0) - return packet -} - -type goPacketRing struct { - packets []*buf.Buffer - owners []*GoConn - segmentEnds []uint64 - head int - count int -} - -func newGoPacketRing(capacity int) goPacketRing { - return goPacketRing{ - packets: make([]*buf.Buffer, capacity), - owners: make([]*GoConn, capacity), - segmentEnds: make([]uint64, capacity), +func (t *MemoryTun) queueWritable(queue *OutboundQueue) { + t.writableAccess.Lock() + if queue.writableQueued { + t.writableAccess.Unlock() + return } + queue.writableQueued = true + t.writable = append(t.writable, queue) + t.writablePending.Store(true) + t.writableAccess.Unlock() + t.engineWake.notify() } -func (r *goPacketRing) empty() bool { - return r.count == 0 -} - -func (r *goPacketRing) full() bool { - return r.count == len(r.packets) -} - -func (r *goPacketRing) push(packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { - index := (r.head + r.count) % len(r.packets) - r.packets[index] = packet - r.owners[index] = owner - r.segmentEnds[index] = segmentEnd - r.count++ -} - -func (r *goPacketRing) pop() (*buf.Buffer, *GoConn, uint64) { - packet := r.packets[r.head] - owner := r.owners[r.head] - segmentEnd := r.segmentEnds[r.head] - r.packets[r.head] = nil - r.owners[r.head] = nil - r.segmentEnds[r.head] = 0 - r.head = (r.head + 1) % len(r.packets) - r.count-- - return packet, owner, segmentEnd +func (t *MemoryTun) takeWritable(spare []*OutboundQueue) []*OutboundQueue { + if !t.writablePending.Swap(false) { + return spare + } + t.writableAccess.Lock() + queues := t.writable + t.writable = spare + for _, queue := range queues { + queue.writableQueued = false + } + t.writableAccess.Unlock() + return queues } -func (r *goPacketRing) releaseAll() { - for !r.empty() { - packet, owner, segmentEnd := r.pop() - if owner != nil { - owner.frameDequeued(packet.Len(), segmentEnd) - } - packet.Release() - } +func (t *MemoryTun) newOutboundPacket(length int) *buf.Buffer { + packet := buf.NewSize(t.headroom + length + t.rearSpace) + packet.Resize(t.headroom, 0) + return packet } diff --git a/tun_memory_queue.go b/tun_memory_queue.go new file mode 100644 index 00000000..023982ea --- /dev/null +++ b/tun_memory_queue.go @@ -0,0 +1,248 @@ +package tun + +import ( + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" +) + +type OutboundQueue struct { + tun *MemoryTun + handler func(packets []*buf.Buffer) + access sync.Mutex + control goPacketRing + datagram goPacketRing + data goPacketRing + dataTurn bool + dataQueued atomic.Int32 + transmitInterest bool + writableQueued bool + closed atomic.Bool + readerWake goSignal + closeSignal chan struct{} +} + +func (t *MemoryTun) NewOutboundQueue(handler func(packets []*buf.Buffer)) *OutboundQueue { + queue := &OutboundQueue{ + tun: t, + handler: handler, + control: newGoPacketRing(2 * t.batchSize), + datagram: newGoPacketRing(2 * t.batchSize), + data: newGoPacketRing(2 * t.batchSize), + readerWake: make(goSignal, 1), + closeSignal: make(chan struct{}), + } + go queue.loopHandle() + return queue +} + +func (q *OutboundQueue) WriteBuffers(buffers []*buf.Buffer) { + q.access.Lock() + if q.closed.Load() { + q.access.Unlock() + buf.ReleaseMulti(buffers) + return + } + wasEmpty := q.emptyLocked() + for _, buffer := range buffers { + if buffer.IsEmpty() || q.datagram.full() { + buffer.Release() + continue + } + q.pushLocked(&q.datagram, buffer, nil, 0) + } + notify := wasEmpty && !q.emptyLocked() + q.access.Unlock() + if notify { + q.readerWake.notify() + } +} + +func (q *OutboundQueue) Close() error { + q.access.Lock() + if q.closed.Swap(true) { + q.access.Unlock() + return nil + } + q.control.releaseAll() + q.datagram.releaseAll() + q.data.releaseAll() + q.dataQueued.Store(0) + q.transmitInterest = false + q.access.Unlock() + close(q.closeSignal) + q.tun.queueWritable(q) + return nil +} + +func (q *OutboundQueue) loopHandle() { + packets := make([]*buf.Buffer, q.tun.batchSize) + for { + count := q.drain(packets) + if count > 0 { + q.handler(packets[:count]) + clear(packets[:count]) + continue + } + select { + case <-q.readerWake: + case <-q.closeSignal: + return + } + } +} + +func (q *OutboundQueue) drain(buffers []*buf.Buffer) int { + q.access.Lock() + count := 0 + for count < len(buffers) { + buffer, owner, segmentEnd := q.popLocked() + if buffer == nil { + break + } + if owner != nil { + owner.frameDequeued(buffer.Len(), segmentEnd) + } + buffers[count] = buffer + count++ + } + writable := q.transmitInterest && !q.data.full() + if writable { + q.transmitInterest = false + } + q.access.Unlock() + if writable { + q.tun.queueWritable(q) + } + return count +} + +func (q *OutboundQueue) emptyLocked() bool { + return q.control.empty() && q.datagram.empty() && q.data.empty() +} + +func (q *OutboundQueue) popLocked() (*buf.Buffer, *GoConn, uint64) { + if !q.control.empty() { + return q.control.pop() + } + if !q.datagram.empty() && (q.data.empty() || !q.dataTurn) { + q.dataTurn = true + return q.datagram.pop() + } + if !q.data.empty() { + q.dataTurn = false + q.dataQueued.Add(-1) + return q.data.pop() + } + return nil, nil, 0 +} + +func (q *OutboundQueue) enqueue(packet *buf.Buffer, fragments [][]byte, class goOutboundClass, owner *GoConn, segmentEnd uint64) error { + q.access.Lock() + ring := &q.control + switch class { + case goOutboundDatagram: + ring = &q.datagram + case goOutboundData: + ring = &q.data + } + var err error + switch { + case q.closed.Load(): + err = errGoFrameDropped + case ring.full() && class == goOutboundData: + err = errGoTransmitBlocked + case ring.capacity-ring.count < max(len(fragments), 1): + err = errGoFrameDropped + } + if err != nil { + q.access.Unlock() + packet.Release() + return err + } + wasEmpty := q.emptyLocked() + if fragments == nil { + q.pushLocked(ring, packet, owner, segmentEnd) + } else { + for _, fragment := range fragments { + fragmentPacket := q.tun.newOutboundPacket(len(fragment)) + common.Must1(fragmentPacket.Write(fragment)) + q.pushLocked(ring, fragmentPacket, owner, segmentEnd) + } + packet.Release() + } + q.access.Unlock() + if wasEmpty { + q.readerWake.notify() + } + return nil +} + +func (q *OutboundQueue) pushLocked(ring *goPacketRing, packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + if owner != nil { + owner.frameEnqueued(packet.Len()) + } + if ring == &q.data { + q.dataQueued.Add(1) + } + ring.push(packet, owner, segmentEnd) +} + +const goPacketRingInitialSize = 16 + +type goPacketRing struct { + entries []goPacketEntry + capacity int + head int + count int +} + +type goPacketEntry struct { + packet *buf.Buffer + owner *GoConn + segmentEnd uint64 +} + +func newGoPacketRing(capacity int) goPacketRing { + return goPacketRing{capacity: capacity} +} + +func (r *goPacketRing) empty() bool { + return r.count == 0 +} + +func (r *goPacketRing) full() bool { + return r.count == r.capacity +} + +func (r *goPacketRing) push(packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + if r.count == len(r.entries) { + entries := make([]goPacketEntry, min(max(2*len(r.entries), goPacketRingInitialSize), r.capacity)) + copied := copy(entries, r.entries[r.head:]) + copy(entries[copied:], r.entries[:r.head]) + r.entries = entries + r.head = 0 + } + r.entries[(r.head+r.count)%len(r.entries)] = goPacketEntry{packet: packet, owner: owner, segmentEnd: segmentEnd} + r.count++ +} + +func (r *goPacketRing) pop() (*buf.Buffer, *GoConn, uint64) { + entry := r.entries[r.head] + r.entries[r.head] = goPacketEntry{} + r.head = (r.head + 1) % len(r.entries) + r.count-- + return entry.packet, entry.owner, entry.segmentEnd +} + +func (r *goPacketRing) releaseAll() { + for !r.empty() { + packet, owner, segmentEnd := r.pop() + if owner != nil { + owner.frameDequeued(packet.Len(), segmentEnd) + } + packet.Release() + } +} From a19caf7b5839d368957d5a9fb06ba87b9d76437a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 28 Sep 2026 16:09:45 +0800 Subject: [PATCH 20/25] Fix zero UDP checksum in checksum completion --- flow_dispatch.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flow_dispatch.go b/flow_dispatch.go index 97221d38..0d5101bb 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -58,7 +58,11 @@ func (m *ForwardFrameMeta) completeChecksum(raw []byte) { } initial := binary.BigEndian.Uint16(raw[checksumAt:]) raw[checksumAt], raw[checksumAt+1] = 0, 0 - binary.BigEndian.PutUint16(raw[checksumAt:], ^checksum.Checksum(raw[m.checksumStart:], initial)) + transportChecksum := ^checksum.Checksum(raw[m.checksumStart:], initial) + if transportChecksum == 0 { + transportChecksum = 0xffff + } + binary.BigEndian.PutUint16(raw[checksumAt:], transportChecksum) } type flowEntry struct { From 603f85a45a87c2d98d34b33cbe936859275615f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 28 Sep 2026 16:14:16 +0800 Subject: [PATCH 21/25] Avoid allocations in checksum paths --- flow_dispatch.go | 8 ++++---- flow_parse.go | 8 +++++--- flow_rewrite.go | 16 ++++++++-------- internal/tschecksum/checksum_amd64.go | 20 +++++++++----------- stack_go_engine.go | 3 ++- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/flow_dispatch.go b/flow_dispatch.go index 0d5101bb..08cc7d58 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -986,10 +986,10 @@ func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed if flow.dnatAddress || flow.dnatPort { rewriteEmbeddedDestination(&embedded, addrToTCPIP(flow.clientDestinationAddress), flow.clientDestinationPort, flow.dnatPort) } - networkHeader := parsed.networkHeader() - networkHeader.SetDestinationAddr(flow.clientAddress) - if networkHeader.SourceAddr() == flow.serverAddress { - networkHeader.SetSourceAddr(flow.clientDestinationAddress) + sourceAddress, destinationAddress := parsed.addressSlices() + copy(destinationAddress, flow.clientAddress.AsSlice()) + if parsed.source.Addr() == flow.serverAddress { + copy(sourceAddress, flow.clientDestinationAddress.AsSlice()) } recomputeChecksums(parsed) return flow diff --git a/flow_parse.go b/flow_parse.go index a5faa265..a8977176 100644 --- a/flow_parse.go +++ b/flow_parse.go @@ -35,11 +35,13 @@ func (p *forwardPacket) flowKey() flowKey { return flowKey{protocol: p.protocol, source: p.source, destination: p.destination} } -func (p *forwardPacket) networkHeader() header.Network { +func (p *forwardPacket) addressSlices() ([]byte, []byte) { if p.ipVersion == 4 { - return header.IPv4(p.network) + ipHdr := header.IPv4(p.network) + return ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice() } - return header.IPv6(p.network) + ipHdr := header.IPv6(p.network) + return ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice() } func (p *forwardPacket) isTCPSyn() bool { diff --git a/flow_rewrite.go b/flow_rewrite.go index 12559190..c96c3695 100644 --- a/flow_rewrite.go +++ b/flow_rewrite.go @@ -131,12 +131,12 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { - networkHeader := packet.networkHeader() + sourceAddress, destinationAddress := packet.addressSlices() if rule.sourceAddress.Len() > 0 { - networkHeader.SetSourceAddress(rule.sourceAddress) + copy(sourceAddress, rule.sourceAddress.AsSlice()) } if rule.destinationAddress.Len() > 0 { - networkHeader.SetDestinationAddress(rule.destinationAddress) + copy(destinationAddress, rule.destinationAddress.AsSlice()) } transport := packet.transport switch packet.protocol { @@ -186,7 +186,7 @@ func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { } func recomputeChecksums(packet *forwardPacket) { - networkHeader := packet.networkHeader() + sourceAddress, destinationAddress := packet.addressSlices() if packet.ipVersion == 4 { ipHdr := header.IPv4(packet.network) ipHdr.SetChecksum(0) @@ -201,7 +201,7 @@ func recomputeChecksums(packet *forwardPacket) { tcpHdr := header.TCP(transport) tcpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(tcpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, networkHeader.SourceAddressSlice(), networkHeader.DestinationAddressSlice(), uint16(len(transport))) + pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, sourceAddress, destinationAddress, uint16(len(transport))) tcpHdr.SetChecksum(^tcpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum))) case uint8(header.UDPProtocolNumber): if len(transport) < header.UDPMinimumSize { @@ -213,7 +213,7 @@ func recomputeChecksums(packet *forwardPacket) { } udpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(udpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, networkHeader.SourceAddressSlice(), networkHeader.DestinationAddressSlice(), udpHdr.Length()) + pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpHdr.Length()) udpChecksum := ^udpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum)) if udpChecksum == 0 { udpChecksum = 0xffff @@ -234,8 +234,8 @@ func recomputeChecksums(packet *forwardPacket) { icmpHdr.SetChecksum(0) icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ Header: icmpHdr, - Src: networkHeader.SourceAddressSlice(), - Dst: networkHeader.DestinationAddressSlice(), + Src: sourceAddress, + Dst: destinationAddress, })) } } diff --git a/internal/tschecksum/checksum_amd64.go b/internal/tschecksum/checksum_amd64.go index 85b925a0..954512f9 100644 --- a/internal/tschecksum/checksum_amd64.go +++ b/internal/tschecksum/checksum_amd64.go @@ -2,22 +2,20 @@ package tschecksum import "golang.org/x/sys/cpu" -var checksum = checksumAMD64 +var ( + useAVX2 = cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 + useSSE2 = cpu.X86.HasSSE2 +) // Checksum computes an IP checksum starting with the provided initial value. // The length of data should be at least 128 bytes for best performance. Smaller // buffers will still compute a correct result. func Checksum(data []byte, initial uint16) uint16 { - return checksum(data, initial) -} - -func init() { - if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { - checksum = checksumAVX2 - return + if useAVX2 { + return checksumAVX2(data, initial) } - if cpu.X86.HasSSE2 { - checksum = checksumSSE2 - return + if useSSE2 { + return checksumSSE2(data, initial) } + return checksumAMD64(data, initial) } diff --git a/stack_go_engine.go b/stack_go_engine.go index fe7ce1ff..20472ffd 100644 --- a/stack_go_engine.go +++ b/stack_go_engine.go @@ -759,7 +759,8 @@ func goValidateTransportChecksum(parsed *forwardPacket) bool { default: return true } - partial := header.PseudoHeaderChecksum(tcpip.TransportProtocolNumber(parsed.protocol), parsed.source.Addr().AsSlice(), parsed.destination.Addr().AsSlice(), uint16(len(transport))) + sourceAddress, destinationAddress := parsed.addressSlices() + partial := header.PseudoHeaderChecksum(tcpip.TransportProtocolNumber(parsed.protocol), sourceAddress, destinationAddress, uint16(len(transport))) return checksum.Checksum(transport, partial) == 0xffff } From afec069a9954dc97d7e4420421eb37619e334373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 28 Sep 2026 16:17:36 +0800 Subject: [PATCH 22/25] Optimize header and NAT checksums --- flow_rewrite.go | 116 +++++++++++++---------------- gtcpip/checksum/checksum_unsafe.go | 6 +- gtcpip/header/ipv4.go | 17 +++-- gtcpip/header/udp.go | 8 +- 4 files changed, 70 insertions(+), 77 deletions(-) diff --git a/flow_rewrite.go b/flow_rewrite.go index c96c3695..e99deaa6 100644 --- a/flow_rewrite.go +++ b/flow_rewrite.go @@ -1,6 +1,7 @@ package tun import ( + "bytes" "encoding/binary" "github.com/sagernet/sing-tun/gtcpip" @@ -18,40 +19,17 @@ type rewriteRule struct { } func applyRewrite(packet *forwardPacket, rule *rewriteRule) { - var oldSource, oldDestination tcpip.Address - if packet.ipVersion == 4 { - ipHdr := header.IPv4(packet.network) - oldSource = ipHdr.SourceAddress() - oldDestination = ipHdr.DestinationAddress() - } else { - ipHdr := header.IPv6(packet.network) - oldSource = ipHdr.SourceAddress() - oldDestination = ipHdr.DestinationAddress() - } - newSource := oldSource - newDestination := oldDestination + sourceAddress, destinationAddress := packet.addressSlices() + var addressDelta uint64 if rule.sourceAddress.Len() > 0 { - newSource = rule.sourceAddress + addressDelta += rewriteAddress(sourceAddress, rule.sourceAddress.AsSlice()) } if rule.destinationAddress.Len() > 0 { - newDestination = rule.destinationAddress + addressDelta += rewriteAddress(destinationAddress, rule.destinationAddress.AsSlice()) } - if packet.ipVersion == 4 { + if packet.ipVersion == 4 && addressDelta != 0 { ipHdr := header.IPv4(packet.network) - if newSource != oldSource { - ipHdr.SetSourceAddressWithChecksumUpdate(newSource) - } - if newDestination != oldDestination { - ipHdr.SetDestinationAddressWithChecksumUpdate(newDestination) - } - } else { - ipHdr := header.IPv6(packet.network) - if newSource != oldSource { - ipHdr.SetSourceAddress(newSource) - } - if newDestination != oldDestination { - ipHdr.SetDestinationAddress(newDestination) - } + ipHdr.SetChecksum(updateChecksum(ipHdr.Checksum(), addressDelta)) } transport := packet.transport switch packet.protocol { @@ -59,18 +37,10 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { if len(transport) < header.TCPMinimumSize { return } - tcpHdr := header.TCP(transport) - if newSource != oldSource { - tcpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource, true) - } - if newDestination != oldDestination { - tcpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination, true) - } - if rule.rewriteSourcePort { - tcpHdr.SetSourcePortWithChecksumUpdate(rule.sourcePort) - } - if rule.rewriteDestinationPort { - tcpHdr.SetDestinationPortWithChecksumUpdate(rule.destinationPort) + delta := addressDelta + rewritePorts(transport, rule) + if delta != 0 { + tcpHdr := header.TCP(transport) + tcpHdr.SetChecksum(updateChecksum(tcpHdr.Checksum(), delta)) } case uint8(header.UDPProtocolNumber): if len(transport) < header.UDPMinimumSize { @@ -78,25 +48,12 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } udpHdr := header.UDP(transport) if packet.ipVersion == 4 && udpHdr.Checksum() == 0 { - if rule.rewriteSourcePort { - udpHdr.SetSourcePort(rule.sourcePort) - } - if rule.rewriteDestinationPort { - udpHdr.SetDestinationPort(rule.destinationPort) - } + rewritePorts(transport, rule) return } - if newSource != oldSource { - udpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource, true) - } - if newDestination != oldDestination { - udpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination, true) - } - if rule.rewriteSourcePort { - udpHdr.SetSourcePortWithChecksumUpdate(rule.sourcePort) - } - if rule.rewriteDestinationPort { - udpHdr.SetDestinationPortWithChecksumUpdate(rule.destinationPort) + delta := addressDelta + rewritePorts(transport, rule) + if delta != 0 { + udpHdr.SetChecksum(updateChecksum(udpHdr.Checksum(), delta)) } if udpHdr.Checksum() == 0 { udpHdr.SetChecksum(0xffff) @@ -116,11 +73,8 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { return } icmpHdr := header.ICMPv6(transport) - if newSource != oldSource { - icmpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource) - } - if newDestination != oldDestination { - icmpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination) + if addressDelta != 0 { + icmpHdr.SetChecksum(updateChecksum(icmpHdr.Checksum(), addressDelta)) } if rule.rewriteSourcePort { icmpHdr.SetIdentWithChecksumUpdate(rule.sourcePort) @@ -130,6 +84,42 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } } +func rewriteAddress(field []byte, address []byte) uint64 { + if bytes.Equal(field, address) { + return 0 + } + var delta uint64 + for offset := 0; offset < len(field); offset += 4 { + delta += uint64(^binary.BigEndian.Uint32(field[offset:])) + uint64(binary.BigEndian.Uint32(address[offset:])) + } + copy(field, address) + return delta +} + +func rewritePorts(transport []byte, rule *rewriteRule) uint64 { + var delta uint64 + if rule.rewriteSourcePort { + delta += checksumDelta(binary.BigEndian.Uint16(transport), rule.sourcePort) + binary.BigEndian.PutUint16(transport, rule.sourcePort) + } + if rule.rewriteDestinationPort { + delta += checksumDelta(binary.BigEndian.Uint16(transport[2:]), rule.destinationPort) + binary.BigEndian.PutUint16(transport[2:], rule.destinationPort) + } + return delta +} + +func checksumDelta(previous uint16, next uint16) uint64 { + if previous == next { + return 0 + } + return uint64(^previous) + uint64(next) +} + +func updateChecksum(current uint16, delta uint64) uint16 { + return ^checksum.Fold(uint64(^current) + delta) +} + func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { sourceAddress, destinationAddress := packet.addressSlices() if rule.sourceAddress.Len() > 0 { diff --git a/gtcpip/checksum/checksum_unsafe.go b/gtcpip/checksum/checksum_unsafe.go index 66b7ab67..558dc6db 100644 --- a/gtcpip/checksum/checksum_unsafe.go +++ b/gtcpip/checksum/checksum_unsafe.go @@ -52,7 +52,7 @@ func calculateChecksum(buf []byte, odd bool, initial uint16) (uint16, bool) { // buf = buf[1:] is skipped because it's unused and nogo will // complain. } - return reduce(acc), odd + return Fold(acc), odd } // On little-endian architectures, multi-byte loads from buf will load @@ -147,14 +147,14 @@ func calculateChecksum(buf []byte, odd bool, initial uint16) (uint16, bool) { } // Reduce the checksum to 16 bits and undo byte swaps before returning. - acc16 := bswapIfLittleEndian16(reduce(acc)) + acc16 := bswapIfLittleEndian16(Fold(acc)) if bswapped { acc16 = bits.ReverseBytes16(acc16) } return acc16, odd } -func reduce(acc uint64) uint16 { +func Fold(acc uint64) uint16 { // Ideally we would do: // return uint16(acc>>48) +' uint16(acc>>32) +' uint16(acc>>16) +' uint16(acc) // for more instruction-level parallelism; however, there is no diff --git a/gtcpip/header/ipv4.go b/gtcpip/header/ipv4.go index 3041eae9..65b7ca17 100644 --- a/gtcpip/header/ipv4.go +++ b/gtcpip/header/ipv4.go @@ -478,10 +478,16 @@ func (b IPv4) SetDestinationAddress(addr tcpip.Address) { // CalculateChecksum calculates the checksum of the IPv4 header. func (b IPv4) CalculateChecksum() uint16 { - // return checksum.Checksum(b[:b.HeaderLength()], 0) - xsum0 := checksum.Checksum(b[:xsum], 0) - xsum0 = checksum.Checksum(b[xsum+2:b.HeaderLength()], xsum0) - return xsum0 + _ = b[IPv4MinimumSize-1] + sum := uint64(binary.BigEndian.Uint32(b)) + + uint64(binary.BigEndian.Uint32(b[id:])) + + uint64(binary.BigEndian.Uint16(b[ttl:])) + + uint64(binary.BigEndian.Uint32(b[srcAddr:])) + + uint64(binary.BigEndian.Uint32(b[dstAddr:])) + for offset := IPv4MinimumSize; offset < int(b.HeaderLength()); offset += 4 { + sum += uint64(binary.BigEndian.Uint32(b[offset:])) + } + return checksum.Fold(sum) } // Encode encodes all the fields of the IPv4 header. @@ -573,8 +579,7 @@ func (b IPv4) IsChecksumValid() bool { // same set of octets, including the checksum field. If the result // is all 1 bits (-0 in 1's complement arithmetic), the check // succeeds. - //return b.CalculateChecksum() == 0xffff - return checksum.Checksum(b[:b.HeaderLength()], 0) == 0xffff + return checksum.Combine(b.CalculateChecksum(), b.Checksum()) == 0xffff } // IsV4MulticastAddress determines if the provided address is an IPv4 multicast diff --git a/gtcpip/header/udp.go b/gtcpip/header/udp.go index ce7708e1..531d1d77 100644 --- a/gtcpip/header/udp.go +++ b/gtcpip/header/udp.go @@ -113,11 +113,9 @@ func (b UDP) SetLength(length uint16) { // CalculateChecksum calculates the checksum of the UDP packet, given the // checksum of the network-layer pseudo-header and the checksum of the payload. func (b UDP) CalculateChecksum(partialChecksum uint16) uint16 { - // Calculate the rest of the checksum.\ - // return checksum.Checksum(b[:UDPMinimumSize], partialChecksum) - xsum := checksum.Checksum(b[:udpChecksum], partialChecksum) - xsum = checksum.Checksum(b[udpChecksum+2:UDPMinimumSize], xsum) - return xsum + return checksum.Fold(uint64(partialChecksum) + + uint64(binary.BigEndian.Uint32(b[udpSrcPort:])) + + uint64(binary.BigEndian.Uint16(b[udpLength:]))) } // IsChecksumValid returns true iff the UDP header's checksum is valid. From 99fa20d23de25d90c5dd949d8e831905ebe2ec5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 29 Sep 2026 22:57:49 +0800 Subject: [PATCH 23/25] Fix packet splice reads on darwin --- stack_go_splice.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stack_go_splice.go b/stack_go_splice.go index 7c75c788..31d1de1c 100644 --- a/stack_go_splice.go +++ b/stack_go_splice.go @@ -859,7 +859,12 @@ func (e *goEngine) packetSpliceRead(w *GoPacketConn) { defer func() { e.flushPacketFrames() clear(e.packetMessages[:]) - if errno != 0 && !goSocketWouldBlock(errno) && w.splice != nil { + if w.splice == nil { + return + } + if count == 0 && errno == 0 { + e.packetSpliceClose(w, io.EOF) + } else if errno != 0 && !goSocketWouldBlock(errno) { e.packetSpliceClose(w, E.Cause(errno, "go: receive peer batch")) } }() From 837976228ca2d772647a215c46384094311b5aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 29 Sep 2026 23:22:01 +0800 Subject: [PATCH 24/25] Fix protocol input validation --- stack_go_tcp_input.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go index 89d6c6e2..46c5c161 100644 --- a/stack_go_tcp_input.go +++ b/stack_go_tcp_input.go @@ -201,11 +201,8 @@ func (e *goEngine) initializeSession(conn *GoConn, now int64, peerWindow uint16) } func (e *goEngine) localMSS(ipVersion uint8) uint16 { - mtu := e.platformIO.mtu() - if ipVersion == 4 { - return uint16(mtu - header.IPv4MinimumSize - header.TCPMinimumSize) - } - return uint16(mtu - header.IPv6MinimumSize - header.TCPMinimumSize) + mss := e.platformIO.mtu() - goNetworkHeaderLength(ipVersion) - header.TCPMinimumSize + return uint16(min(max(mss, header.TCPMinimumSendMSS), header.TCPMaximumMSS)) } func (e *goEngine) evictFlow() bool { @@ -594,6 +591,10 @@ func (c *GoConn) signalReader() { } func (e *goEngine) deliverSegment(conn *GoConn, segOffset int64, payload []byte, fin bool) { + if conn.finReceived { + e.markAck(conn, true) + return + } conn.receiveChain.releaseBelow(conn.consumedTail.Load()) receiveNext := int64(conn.receiveNext) edge := int64(conn.publishedEdge) From 1d2b8b61d717200092e2018c4416dcc26b7deaaf Mon Sep 17 00:00:00 2001 From: chasylexus <13235797+chasylexus@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:49:44 +0300 Subject: [PATCH 25/25] Fix forwarding packets on flows closed on attach A FlowTracker may close its flow through the FlowHandle from AttachFlow. The forward dispatcher and the gVisor ICMP forwarder still wrote the packet that created the flow to the port after AttachFlow returned, so it went out on a flow that was already closed. For ICMP this happened to every echo request, because a closed cached flow is dropped and the next request installs a new one. Check the flow's closed state before that first write. --- flow_dispatch.go | 4 ++- flow_dispatch_test.go | 56 +++++++++++++++++++++++++++++++++++++++ stack_gvisor_icmp.go | 4 ++- stack_gvisor_icmp_test.go | 27 +++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 flow_dispatch_test.go create mode 100644 stack_gvisor_icmp_test.go diff --git a/flow_dispatch.go b/flow_dispatch.go index 08cc7d58..477f2ebc 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -402,7 +402,9 @@ func (s *ForwardStage) judgeAndInstall(key flowKey, packet *forwardPacket, raw [ entry := &flowEntry{action: ActionFlow, flow: flow, idle: d.flowIdle(flow)} entry.deadline = now + int64(entry.idle) s.insertEntry(key, entry, now) - s.forwardToPort(flow, packet, raw, meta) + if !flow.closed.Load() { + s.forwardToPort(flow, packet, raw, meta) + } return true } if result == createFlowExhausted { diff --git a/flow_dispatch_test.go b/flow_dispatch_test.go new file mode 100644 index 00000000..c8f54f9b --- /dev/null +++ b/flow_dispatch_test.go @@ -0,0 +1,56 @@ +package tun + +import ( + "net/netip" + "testing" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/logger" +) + +type closeOnAttachTracker struct{} + +func (closeOnAttachTracker) AttachFlow(handle FlowHandle) { handle.CloseFlow() } +func (closeOnAttachTracker) CountForward(int) {} +func (closeOnAttachTracker) CountReverse(int) {} +func (closeOnAttachTracker) FlowEstablished() {} +func (closeOnAttachTracker) CloseFlow(FlowCloseReason) {} + +type recordingPort struct { + address netip.Addr + written int +} + +func (p *recordingPort) PortAddresses() (netip.Addr, netip.Addr) { return p.address, netip.Addr{} } +func (*recordingPort) PortMTU() uint32 { return 0 } +func (*recordingPort) AttachReturn(Return) error { return nil } +func (*recordingPort) DetachReturn(Return) error { return nil } +func (p *recordingPort) WritePackets(packets [][]byte) error { + p.written += len(packets) + return nil +} + +type closeOnAttachHandler struct { + Handler + port Port +} + +func (h closeOnAttachHandler) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionFlow, Port: h.port, NewTracker: func() FlowTracker { return closeOnAttachTracker{} }} +} + +func TestForwardFlowClosedOnAttach(t *testing.T) { + port := &recordingPort{address: netip.MustParseAddr("192.0.2.1")} + dispatcher := NewForwardDispatcher(closeOnAttachHandler{port: port}, nil, logger.NOP(), 0, 0) + t.Cleanup(dispatcher.Close) + stage := dispatcher.NewStage(nil) + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(packet, 4, netip.MustParseAddrPort("198.18.0.2:12345"), netip.MustParseAddrPort("203.0.113.1:443"), []byte{1}, 1) + if !stage.Dispatch(packet) { + t.Fatal("packet was not handled") + } + stage.Flush() + if port.written != 0 { + t.Fatal("packet forwarded on a flow closed by its tracker") + } +} diff --git a/stack_gvisor_icmp.go b/stack_gvisor_icmp.go index 360f600b..2f0646b5 100644 --- a/stack_gvisor_icmp.go +++ b/stack_gvisor_icmp.go @@ -289,7 +289,9 @@ func (f *ICMPForwarder) installFlow(key icmpFlowKey, verdict FlowVerdict, pkt *s if flow.tracker != nil { flow.tracker.AttachFlow(flow) } - f.writeToPort(flow, pkt) + if !flow.closed.Load() { + f.writeToPort(flow, pkt) + } return true } diff --git a/stack_gvisor_icmp_test.go b/stack_gvisor_icmp_test.go new file mode 100644 index 00000000..19f40751 --- /dev/null +++ b/stack_gvisor_icmp_test.go @@ -0,0 +1,27 @@ +//go:build with_gvisor + +package tun + +import ( + "net/netip" + "testing" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/sing/common/logger" +) + +func TestICMPForwarderClosedOnAttach(t *testing.T) { + port := &recordingPort{address: netip.IPv4Unspecified()} + forwarder := NewICMPForwarder(nil, nil, logger.NOP()) + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData([]byte{1})}) + defer pkt.DecRef() + key := icmpFlowKey{source: netip.MustParseAddr("198.18.0.2"), destination: netip.MustParseAddr("203.0.113.1"), identifier: 1} + verdict := FlowVerdict{Action: ActionFlow, Port: port, NewTracker: func() FlowTracker { return closeOnAttachTracker{} }} + if !forwarder.installFlow(key, verdict, pkt) { + t.Fatal("flow was not installed") + } + if port.written != 0 { + t.Fatal("packet forwarded on a flow closed by its tracker") + } +}