diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be40dea8..3ab6ab14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,15 +9,28 @@ on: - '**.md' - '.github/**' - '!.github/workflows/debug.yml' + - '!.github/workflows/test.yml' pull_request: branches: - main - dev jobs: - build: - name: Linux - runs-on: ubuntu-latest + test: + name: ${{ matrix.os }} (Go ${{ matrix.go }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + go: ['1.25.x', '1.26.x', stable] + include: + - os: windows-latest + go: stable + - os: macos-latest + go: stable + env: + GOTOOLCHAIN: local steps: - name: Checkout uses: actions/checkout@v4 @@ -26,81 +39,21 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: ^1.25.0 - - name: Build + go-version: ${{ matrix.go }} + - name: Install network test tools + if: runner.os == 'Linux' run: | - make test - go test -c -o ping_test ./ping - sudo ./ping_test -test.v - build_go124: - name: Linux (Go 1.24) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ~1.24 - continue-on-error: true - - name: Build - run: | - make test - go test -c -o ping_test ./ping - sudo ./ping_test -test.v - build_go123: - name: Linux (Go 1.23) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ~1.23 - continue-on-error: true - - name: Build + sudo apt-get update + sudo apt-get install -y iproute2 + - name: Test + env: + TEST_FLAGS: ${{ runner.os != 'Windows' && '-exec sudo' || '' }} + run: make test + - name: Test unprivileged ping + if: runner.os != 'Linux' + run: go test -v ./ping + - name: Test privileged ping + if: runner.os != 'Windows' run: | - make test go test -c -o ping_test ./ping sudo ./ping_test -test.v - build_windows: - name: Windows - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ^1.23 - continue-on-error: true - - name: Build - run: | - make test - go test -v ./ping - build_darwin: - name: macOS - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ^1.23 - continue-on-error: true - - name: Build - run: | - make test - go test -v ./ping - go test -c -o ping_test ./ping - sudo ./ping_test -test.v \ No newline at end of file diff --git a/Makefile b/Makefile index ed36c792..3b836dde 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,7 @@ build: GOOS=windows GOARCH=amd64 go build -v -tags with_gvisor . fmt: - @gofumpt -l -w . - @gofmt -s -w . - @gci write --custom-order -s standard -s "prefix(github.com/sagernet/)" -s "default" . - -fmt_install: - go install -v mvdan.cc/gofumpt@latest - go install -v github.com/daixiang0/gci@latest + @golangci-lint fmt lint: GOOS=linux golangci-lint --max-same-issues=0 --max-issues-per-linter=0 run ./... @@ -29,4 +23,4 @@ lint_install: test: go build -v . #go test -bench=. ./internal/checksum_test - go test -v . + go test -v $(TEST_FLAGS) . diff --git a/flow_dispatch.go b/flow_dispatch.go index c8562a61..477f2ebc 100644 --- a/flow_dispatch.go +++ b/flow_dispatch.go @@ -1,13 +1,16 @@ package tun import ( + "encoding/binary" "maps" "net/netip" + "slices" "sync" "sync/atomic" "time" "github.com/sagernet/sing-tun/gtcpip" + "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/logger" @@ -25,6 +28,7 @@ const ( flowTombstoneTimeout = 4 * time.Minute flowTableCapacity = 16384 + flowStageCapacity = 1024 flowSweepInterval = 30 * time.Second flowSweepLimit = flowTableCapacity / int(flowTombstoneTimeout/flowSweepInterval) @@ -35,15 +39,43 @@ type ForwardWriteback interface { WriteReturnPackets(packets [][]byte) error } +type ForwardFrameMeta struct { + needsChecksum bool + checksumStart uint16 + checksumOffset uint16 + gsoType uint8 + gsoSize uint16 +} + +func (m *ForwardFrameMeta) completeChecksum(raw []byte) { + if !m.needsChecksum { + return + } + m.needsChecksum = false + checksumAt := int(m.checksumStart) + int(m.checksumOffset) + if int(m.checksumStart) >= len(raw) || checksumAt+2 > len(raw) { + return + } + initial := binary.BigEndian.Uint16(raw[checksumAt:]) + raw[checksumAt], raw[checksumAt+1] = 0, 0 + transportChecksum := ^checksum.Checksum(raw[m.checksumStart:], initial) + if transportChecksum == 0 { + transportChecksum = 0xffff + } + binary.BigEndian.PutUint16(raw[checksumAt:], transportChecksum) +} + type flowEntry struct { action FlowAction deadline int64 idle time.Duration flow *forwardFlow + verdict FlowVerdict } type forwardFlow struct { nat *portNAT + owner *ForwardStage reverseKey flowKey forwardRule rewriteRule reverseRule rewriteRule @@ -114,20 +146,34 @@ type ForwardDispatcher struct { logger logger.Logger udpTimeout time.Duration icmpTimeout time.Duration - access sync.RWMutex - table map[flowKey]*flowEntry - lastSweep int64 - resetPending atomic.Bool - ports map[Port]*portNAT - natList atomic.Pointer[[]*portNAT] - revNAT atomic.Pointer[map[netip.Addr]*portNAT] + portsAccess sync.Mutex + ports map[Port]*portNAT + natList atomic.Pointer[[]*portNAT] + revNAT atomic.Pointer[map[netip.Addr]*portNAT] + returnPath forwardReturn + stagesAccess sync.Mutex + stages []*ForwardStage + stageCount atomic.Int32 + resetGeneration atomic.Uint32 +} - activeNATs []*portNAT - writebackBatch [][]byte - returnPath forwardReturn - exhaustedLogAt int64 +type stagedPort struct { + nat *portNAT + packets [][]byte +} +type ForwardStage struct { + dispatcher *ForwardDispatcher + writeback ForwardWriteback + access sync.Mutex + table map[flowKey]*flowEntry + lastSweep int64 + resetSeen uint32 + + exhaustedLogAt int64 + ports []stagedPort + writebackBatch [][]byte segmentBuffers [][]byte segmentSizes []int segmentUsed int @@ -148,7 +194,6 @@ func NewForwardDispatcher(handler Handler, writeback ForwardWriteback, logger lo logger: logger, udpTimeout: udpTimeout, icmpTimeout: icmpTimeout, - table: make(map[flowKey]*flowEntry), ports: make(map[Port]*portNAT), } if dispatcher.udpTimeout <= 0 { @@ -161,6 +206,26 @@ func NewForwardDispatcher(handler Handler, writeback ForwardWriteback, logger lo return dispatcher } +func (d *ForwardDispatcher) NewStage(writeback ForwardWriteback) *ForwardStage { + if d == nil { + return nil + } + if writeback == nil { + writeback = d.writeback + } + stage := &ForwardStage{ + dispatcher: d, + writeback: writeback, + table: make(map[flowKey]*flowEntry), + resetSeen: d.resetGeneration.Load(), + } + d.stagesAccess.Lock() + d.stages = append(d.stages, stage) + d.stageCount.Store(int32(len(d.stages))) + d.stagesAccess.Unlock() + return stage +} + func (d *ForwardDispatcher) now() int64 { return int64(time.Since(d.epoch)) } @@ -170,76 +235,100 @@ func (d *ForwardDispatcher) Close() { return } d.returnPath.closed.Store(true) - d.access.Lock() - flows := make([]*forwardFlow, 0, len(d.table)) - for _, entry := range d.table { - if entry.flow != nil { - flows = append(flows, entry.flow) + d.stagesAccess.Lock() + stages := d.stages + d.stagesAccess.Unlock() + for _, stage := range stages { + stage.access.Lock() + for key, entry := range stage.table { + stage.removeEntry(key, entry, FlowCloseReset) } + stage.access.Unlock() } + d.portsAccess.Lock() ports := make([]Port, 0, len(d.ports)) - for port, nat := range d.ports { - if nat != nil { - ports = append(ports, port) - } - } - d.access.Unlock() - for _, flow := range flows { - flow.close(FlowCloseReset) + for port := range d.ports { + ports = append(ports, port) } + d.portsAccess.Unlock() for _, port := range ports { port.DetachReturn(&d.returnPath) } } -func (d *ForwardDispatcher) Dispatch(packet []byte) bool { - if d == nil || d.returnPath.closed.Load() { +func (s *ForwardStage) Dispatch(packet []byte) bool { + if s == nil || s.dispatcher.returnPath.closed.Load() { return false } parsed, ok := parseForwardPacket(packet) - if !ok || parsed.fragment || !parsed.hasFlow { + if !ok { return false } - d.access.RLock() - if d.returnPath.closed.Load() { - d.access.RUnlock() + return s.dispatch(packet, &parsed, nil) +} + +func (s *ForwardStage) DispatchParsed(packet []byte, meta ForwardFrameMeta, parsed *forwardPacket) bool { + if s == nil || s.dispatcher.returnPath.closed.Load() { return false } - key := parsed.flowKey() - now := d.now() - entry, loaded := d.table[key] - if loaded && d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) - loaded = false + return s.dispatch(packet, parsed, &meta) +} + +func (s *ForwardStage) dispatch(packet []byte, parsed *forwardPacket, meta *ForwardFrameMeta) bool { + if parsed.fragment || !parsed.hasFlow { + return false } + key := parsed.flowKey() + now := s.dispatcher.now() + s.access.Lock() + entry, loaded := s.table[key] if loaded { - handled := d.handleHit(key, entry, &parsed, packet, now) - d.access.RUnlock() - return handled + if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) + } else { + handled, remove := s.handleHit(entry, parsed, packet, now, meta) + if remove { + s.removeEntry(key, entry, FlowCloseReset) + } + s.access.Unlock() + return handled + } } - d.access.RUnlock() + s.access.Unlock() if parsed.protocol == uint8(header.TCPProtocolNumber) && (parsed.tcpFlags&header.TCPFlagSyn == 0 || parsed.tcpFlags&header.TCPFlagAck != 0) { return false } - return d.judgeAndInstall(key, &parsed, packet) + return s.judgeAndInstall(key, parsed, packet, meta) } -func (d *ForwardDispatcher) handleHit(key flowKey, entry *flowEntry, packet *forwardPacket, raw []byte, now int64) bool { +func (s *ForwardStage) teardownFlow(key flowKey, reason FlowCloseReason) { + if s == nil || s.dispatcher.returnPath.closed.Load() { + return + } + s.access.Lock() + entry, loaded := s.table[key] + if loaded { + s.removeEntry(key, entry, reason) + } + s.access.Unlock() +} + +func (s *ForwardStage) handleHit(entry *flowEntry, packet *forwardPacket, raw []byte, now int64, meta *ForwardFrameMeta) (handled bool, remove bool) { switch entry.action { case ActionFlow: flow := entry.flow if flow.closed.Load() { - d.tombstoneEntry(entry, now) - return true + tombstoneEntry(entry, now) + return true, false } var flowFinished bool if packet.protocol == uint8(header.TCPProtocolNumber) { if packet.tcpFlags&header.TCPFlagRst != 0 { - d.forwardToPort(flow, packet, raw) + s.forwardToPort(flow, packet, raw, meta) flow.close(FlowCloseReset) - d.tombstoneEntry(entry, now) - return true + tombstoneEntry(entry, now) + return true, false } if packet.tcpFlags&header.TCPFlagFin != 0 { flow.finForward.Store(true) @@ -247,36 +336,38 @@ func (d *ForwardDispatcher) handleHit(key flowKey, entry *flowEntry, packet *for flowFinished = true } } - entry.idle = d.flowIdle(flow) + entry.idle = s.dispatcher.flowIdle(flow) entry.deadline = now + int64(entry.idle) - d.forwardToPort(flow, packet, raw) + s.forwardToPort(flow, packet, raw, meta) if flowFinished { flow.report(FlowCloseFinished) } - return true + return true, false case ActionAccept: if packet.protocol == uint8(header.TCPProtocolNumber) { if packet.tcpFlags&header.TCPFlagRst != 0 { - d.removeEntry(key, entry, FlowCloseReset) - return false + return false, true } if packet.tcpFlags&header.TCPFlagSyn == 0 { entry.idle = tcpEstablishedTimeout } + } else if packet.protocol == uint8(header.UDPProtocolNumber) { + packet.verdict = entry.verdict } entry.deadline = now + int64(entry.idle) - return false + return false, false case ActionReject: entry.deadline = now + int64(entry.idle) - d.stageReject(packet) - return true + s.stageReject(packet, raw, meta) + return true, false default: entry.deadline = now + int64(entry.idle) - return true + return true, false } } -func (d *ForwardDispatcher) judgeAndInstall(key flowKey, packet *forwardPacket, raw []byte) bool { +func (s *ForwardStage) judgeAndInstall(key flowKey, packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) bool { + d := s.dispatcher var firstPacket []byte if packet.protocol == uint8(header.UDPProtocolNumber) { firstPacket = header.UDP(packet.transport).Payload() @@ -285,59 +376,82 @@ func (d *ForwardDispatcher) judgeAndInstall(key flowKey, packet *forwardPacket, if verdict.Action == ActionBypass && verdict.Port != nil { verdict.Action = ActionFlow } - d.access.RLock() - defer d.access.RUnlock() + s.access.Lock() + defer s.access.Unlock() if d.returnPath.closed.Load() { return false } now := d.now() + existing, loaded := s.table[key] + if loaded { + if entryExpired(existing, now) { + s.removeEntry(key, existing, FlowCloseTimeout) + } else { + handled, remove := s.handleHit(existing, packet, raw, now, meta) + if remove { + s.removeEntry(key, existing, FlowCloseReset) + } + return handled + } + } switch verdict.Action { case ActionFlow: if verdict.Port != nil { - flow, result := d.createFlow(packet, verdict) + flow, result := s.createFlow(packet, verdict) if result == createFlowOK { entry := &flowEntry{action: ActionFlow, flow: flow, idle: d.flowIdle(flow)} entry.deadline = now + int64(entry.idle) - d.insertEntry(key, entry, now) - d.forwardToPort(flow, packet, raw) + s.insertEntry(key, entry, now) + if !flow.closed.Load() { + s.forwardToPort(flow, packet, raw, meta) + } return true } if result == createFlowExhausted { - if now-d.exhaustedLogAt >= int64(exhaustedLogInterval) { - d.exhaustedLogAt = now + if now-s.exhaustedLogAt >= int64(exhaustedLogInterval) { + s.exhaustedLogAt = now d.logger.Warn("port selector range exhausted, rejecting flow to ", packet.destination) } - d.installSimple(key, ActionReject, packet.protocol, now) - d.stageReject(packet) + s.installSimple(key, ActionReject, packet.protocol, now) + s.stageReject(packet, raw, meta) return true } } - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false case ActionReject: - d.installSimple(key, ActionReject, packet.protocol, now) - d.stageReject(packet) + s.installSimple(key, ActionReject, packet.protocol, now) + s.stageReject(packet, raw, meta) return true case ActionDrop: - d.installSimple(key, ActionDrop, packet.protocol, now) + s.installSimple(key, ActionDrop, packet.protocol, now) return true case ActionHijackDNS: if packet.protocol == uint8(header.UDPProtocolNumber) { d.hijackDNSPacket(packet) return true } - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false default: - d.installSimple(key, ActionAccept, packet.protocol, now) + s.installAccept(key, packet, verdict, now) return false } } -func (d *ForwardDispatcher) installSimple(key flowKey, action FlowAction, protocol uint8, now int64) { - entry := &flowEntry{action: action, idle: d.idleTimeout(protocol, false)} +func (s *ForwardStage) installSimple(key flowKey, action FlowAction, protocol uint8, now int64) *flowEntry { + entry := &flowEntry{action: action, idle: s.dispatcher.idleTimeout(protocol, false)} entry.deadline = now + int64(entry.idle) - d.insertEntry(key, entry, now) + s.insertEntry(key, entry, now) + return entry +} + +func (s *ForwardStage) installAccept(key flowKey, packet *forwardPacket, verdict FlowVerdict, now int64) { + entry := s.installSimple(key, ActionAccept, packet.protocol, now) + if packet.protocol == uint8(header.UDPProtocolNumber) { + entry.verdict = verdict + packet.verdict = verdict + } } func (d *ForwardDispatcher) idleTimeout(protocol uint8, established bool) time.Duration { @@ -375,7 +489,7 @@ const ( const exhaustedLogInterval = 5 * time.Second -func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdict) (*forwardFlow, createFlowResult) { +func (s *ForwardStage) createFlow(packet *forwardPacket, verdict FlowVerdict) (*forwardFlow, createFlowResult) { var portAddress netip.Addr inet4Address, inet6Address := verdict.Port.PortAddresses() if packet.ipVersion == 6 { @@ -404,11 +518,11 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic if verdict.Destination.Port() != 0 && !isICMP { serverPort = verdict.Destination.Port() } - nat := d.natFor(verdict.Port) + nat := s.dispatcher.natFor(verdict.Port) if nat == nil { return nil, createFlowUnsupported } - selector, reverseKey, allocated := nat.allocateSelector(packet.protocol, portAddress, serverAddress, serverPort, packet.source.Port()) + selector, reverseKey, allocated := nat.reserveSelector(packet.protocol, portAddress, serverAddress, serverPort, packet.source.Port()) if !allocated { return nil, createFlowExhausted } @@ -418,6 +532,7 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic } flow := &forwardFlow{ nat: nat, + owner: s, reverseKey: reverseKey, effectiveMTU: effectiveMTU, protocol: packet.protocol, @@ -465,6 +580,8 @@ func (d *ForwardDispatcher) createFlow(packet *forwardPacket, verdict FlowVerdic } func (d *ForwardDispatcher) natFor(port Port) *portNAT { + d.portsAccess.Lock() + defer d.portsAccess.Unlock() nat, loaded := d.ports[port] if loaded { return nat @@ -498,51 +615,63 @@ func (d *ForwardDispatcher) natFor(port Port) *portNAT { return nat } -func (d *ForwardDispatcher) forwardToPort(flow *forwardFlow, packet *forwardPacket, raw []byte) { - if flow.effectiveMTU != 0 && uint32(len(raw)) > flow.effectiveMTU { +func (s *ForwardStage) forwardToPort(flow *forwardFlow, packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) { + effectiveMTU := flow.effectiveMTU + if meta != nil { + meta.completeChecksum(raw) + if meta.gsoSize != 0 && packet.protocol == uint8(header.TCPProtocolNumber) { + headerLength := len(raw) - len(packet.transport) + int(header.TCP(packet.transport).DataOffset()) + offloadMTU := uint32(headerLength) + uint32(meta.gsoSize) + if effectiveMTU == 0 || offloadMTU < effectiveMTU { + effectiveMTU = offloadMTU + } + } + } + if effectiveMTU != 0 && uint32(len(raw)) > effectiveMTU { if packet.protocol == uint8(header.TCPProtocolNumber) { if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) - d.resegmentTCP(flow, packet, raw) + rewriteForward(flow, packet) + s.resegmentTCP(flow, packet, raw, effectiveMTU) return } if packet.ipVersion == 4 { - ipHdr := packet.network.(header.IPv4) + ipHdr := header.IPv4(packet.network) if ipHdr.Flags()&header.IPv4FlagDontFragment == 0 { if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) + rewriteForward(flow, packet) fragments, ok := fragmentIPv4Packet(ipHdr, flow.effectiveMTU) if ok { for _, fragment := range fragments { - d.stagePort(flow.nat, fragment) + s.stagePort(flow.nat, fragment) } } return } - reply, ok := buildFragmentationNeeded(ipHdr, flow.effectiveMTU, d.writeback.ReturnHeadroom()) - if ok { - d.writebackBatch = append(d.writebackBatch, reply) - } - return - } - reply, ok := buildPacketTooBig(packet.network.(header.IPv6), flow.effectiveMTU, d.writeback.ReturnHeadroom()) - if ok { - d.writebackBatch = append(d.writebackBatch, reply) } + s.writebackBatch = append(s.writebackBatch, buildICMPError(packet, ICMPErrorPacketTooBig, flow.effectiveMTU, s.writeback.ReturnHeadroom())) return } if flow.tracker != nil { flow.tracker.CountForward(len(raw)) } - d.rewriteForward(flow, packet) - d.stagePort(flow.nat, raw) + rewriteForward(flow, packet) + if meta != nil { + raw = s.copyForStage(raw) + } + s.stagePort(flow.nat, raw) +} + +func (s *ForwardStage) copyForStage(raw []byte) []byte { + staged, _ := s.reserveSegments(1, len(raw)) + copy(staged[0], raw) + return staged[0] } -func (d *ForwardDispatcher) rewriteForward(flow *forwardFlow, packet *forwardPacket) { +func rewriteForward(flow *forwardFlow, packet *forwardPacket) { if packet.isTCPSyn() { applyRewriteRaw(packet, &flow.forwardRule) clampTCPMSS(packet, flow.effectiveMTU) @@ -552,28 +681,23 @@ func (d *ForwardDispatcher) rewriteForward(flow *forwardFlow, packet *forwardPac } } -func (d *ForwardDispatcher) stagePort(nat *portNAT, packet []byte) { - if len(nat.pending) == 0 { - d.activeNATs = append(d.activeNATs, nat) +func (s *ForwardStage) stagePort(nat *portNAT, packet []byte) { + for index := range s.ports { + if s.ports[index].nat == nat { + s.ports[index].packets = append(s.ports[index].packets, packet) + return + } } - nat.pending = append(nat.pending, packet) + s.ports = append(s.ports, stagedPort{nat: nat, packets: [][]byte{packet}}) } -func (d *ForwardDispatcher) flushPort(nat *portNAT) { - if len(nat.pending) == 0 { - return +func (s *ForwardStage) stageReject(packet *forwardPacket, raw []byte, meta *ForwardFrameMeta) { + if meta != nil { + meta.completeChecksum(raw) } - err := nat.port.WritePackets(nat.pending) - if err != nil { - d.logger.Trace(E.Cause(err, "forward packets")) - } - nat.pending = nat.pending[:0] -} - -func (d *ForwardDispatcher) stageReject(packet *forwardPacket) { - reply, ok := buildReject(packet, d.writeback.ReturnHeadroom()) + reply, ok := buildReject(packet, s.writeback.ReturnHeadroom()) if ok { - d.writebackBatch = append(d.writebackBatch, reply) + s.writebackBatch = append(s.writebackBatch, reply) } } @@ -581,44 +705,75 @@ func (d *ForwardDispatcher) ResetNetwork() { if d == nil { return } - d.resetPending.Store(true) + d.resetGeneration.Add(1) } -func (d *ForwardDispatcher) Flush() { - if d == nil || d.returnPath.closed.Load() { +func (s *ForwardStage) Flush() { + if s == nil { return } - d.access.RLock() - defer d.access.RUnlock() + d := s.dispatcher if d.returnPath.closed.Load() { return } - if d.resetPending.Swap(false) { - for key, entry := range d.table { - d.removeEntry(key, entry, FlowCloseReset) + now := d.now() + generation := d.resetGeneration.Load() + if s.resetSeen != generation { + s.resetSeen = generation + s.access.Lock() + for key, entry := range s.table { + s.removeEntry(key, entry, FlowCloseReset) } + s.access.Unlock() } - for _, nat := range d.activeNATs { - d.flushPort(nat) + for index := range s.ports { + staged := &s.ports[index] + if len(staged.packets) == 0 { + continue + } + err := staged.nat.port.WritePackets(staged.packets) + if err != nil { + d.logger.Trace(E.Cause(err, "forward packets")) + } + clear(staged.packets) + staged.packets = staged.packets[:0] } - d.activeNATs = d.activeNATs[:0] - if retain := max(d.segmentUsed, segmentRetainCount); len(d.segmentBuffers) > retain { - clear(d.segmentBuffers[retain:]) - d.segmentBuffers = d.segmentBuffers[:retain] - d.segmentSizes = d.segmentSizes[:retain] + if retain := max(s.segmentUsed, segmentRetainCount); len(s.segmentBuffers) > retain { + clear(s.segmentBuffers[retain:]) + s.segmentBuffers = s.segmentBuffers[:retain] + s.segmentSizes = s.segmentSizes[:retain] } - d.segmentUsed = 0 - if len(d.writebackBatch) > 0 { - err := d.writeback.WriteReturnPackets(d.writebackBatch) + s.segmentUsed = 0 + if len(s.writebackBatch) > 0 { + err := s.writeback.WriteReturnPackets(s.writebackBatch) if err != nil { d.logger.Trace(E.Cause(err, "write back packets")) } - d.writebackBatch = d.writebackBatch[:0] + clear(s.writebackBatch) + s.writebackBatch = s.writebackBatch[:0] + } + if now-s.lastSweep >= int64(flowSweepInterval) { + s.lastSweep = now + s.access.Lock() + s.sweep(now) + s.access.Unlock() } - d.maybeSweep(d.now()) } -func (d *ForwardDispatcher) entryExpired(entry *flowEntry, now int64) bool { +func (s *ForwardStage) sweepDue() (time.Duration, bool) { + if s == nil { + return 0, false + } + s.access.Lock() + count := len(s.table) + s.access.Unlock() + if count == 0 { + return 0, false + } + return max(time.Duration(s.lastSweep+int64(flowSweepInterval)-s.dispatcher.now()), 0), true +} + +func entryExpired(entry *flowEntry, now int64) bool { if now <= entry.deadline { return false } @@ -633,14 +788,14 @@ func (d *ForwardDispatcher) entryExpired(entry *flowEntry, now int64) bool { return true } -func (d *ForwardDispatcher) tombstoneEntry(entry *flowEntry, now int64) { +func tombstoneEntry(entry *flowEntry, now int64) { entry.action = ActionDrop entry.idle = flowTombstoneTimeout entry.deadline = now + int64(entry.idle) } -func (d *ForwardDispatcher) removeEntry(key flowKey, entry *flowEntry, reason FlowCloseReason) { - delete(d.table, key) +func (s *ForwardStage) removeEntry(key flowKey, entry *flowEntry, reason FlowCloseReason) { + delete(s.table, key) if entry.flow != nil { if reason == FlowCloseTimeout && entry.flow.finForward.Load() && entry.flow.finReverse.Load() { reason = FlowCloseFinished @@ -650,23 +805,23 @@ func (d *ForwardDispatcher) removeEntry(key flowKey, entry *flowEntry, reason Fl } } -func (d *ForwardDispatcher) insertEntry(key flowKey, entry *flowEntry, now int64) { - if len(d.table) >= flowTableCapacity { - d.evictEntries(now) +func (s *ForwardStage) insertEntry(key flowKey, entry *flowEntry, now int64) { + if len(s.table) >= max(flowTableCapacity/int(s.dispatcher.stageCount.Load()), flowStageCapacity) { + s.evictEntries(now) } - d.table[key] = entry + s.table[key] = entry } -func (d *ForwardDispatcher) evictEntries(now int64) { +func (s *ForwardStage) evictEntries(now int64) { var ( freed int visited int oldestKey flowKey oldest *flowEntry ) - for key, entry := range d.table { - if d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) + for key, entry := range s.table { + if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) freed++ } else if oldest == nil || entry.deadline < oldest.deadline { oldestKey = key @@ -678,21 +833,17 @@ func (d *ForwardDispatcher) evictEntries(now int64) { } } if freed == 0 && oldest != nil { - d.removeEntry(oldestKey, oldest, FlowCloseReset) + s.removeEntry(oldestKey, oldest, FlowCloseReset) } } -func (d *ForwardDispatcher) maybeSweep(now int64) { - if now-d.lastSweep < int64(flowSweepInterval) { - return - } - d.lastSweep = now +func (s *ForwardStage) sweep(now int64) { visited := 0 - for key, entry := range d.table { + for key, entry := range s.table { if entry.action == ActionFlow && entry.flow.closed.Load() { - d.tombstoneEntry(entry, now) - } else if d.entryExpired(entry, now) { - d.removeEntry(key, entry, FlowCloseTimeout) + tombstoneEntry(entry, now) + } else if entryExpired(entry, now) { + s.removeEntry(key, entry, FlowCloseTimeout) } visited++ if visited >= flowSweepLimit { @@ -705,8 +856,6 @@ func isICMPProtocol(protocol uint8) bool { return protocol == uint8(header.ICMPv4ProtocolNumber) || protocol == uint8(header.ICMPv6ProtocolNumber) } -var _ Return = (*forwardReturn)(nil) - type forwardReturn struct { dispatcher *ForwardDispatcher closed atomic.Bool @@ -724,6 +873,11 @@ const ( returnDrop ) +type returnBatch struct { + writeback ForwardWriteback + packets [][]byte +} + func (r *forwardReturn) ReturnPackets(packets [][]byte) [][]byte { if r.closed.Load() { return packets @@ -739,60 +893,55 @@ func (r *forwardReturn) ReturnPackets(packets [][]byte) [][]byte { } headroom := r.dispatcher.writeback.ReturnHeadroom() now := r.dispatcher.now() - - if len(packets) == 1 { - switch r.classifyReturn(packets[0], natList, revMap, headroom, now) { - case returnWrite: - if err := r.dispatcher.writeback.WriteReturnPackets(packets[:1]); err != nil { - r.dispatcher.logger.Trace(E.Cause(err, "write return packets")) - } - return packets[:0] - case returnDrop: - return packets[:0] - default: - return packets - } - } - unconsumed := packets[:0] - var writeBatch [][]byte + var batches []returnBatch for _, raw := range packets { - switch r.classifyReturn(raw, natList, revMap, headroom, now) { + decision, writeback := r.classifyReturn(raw, natList, revMap, headroom, now) + switch decision { case returnWrite: - writeBatch = append(writeBatch, raw) + index := slices.IndexFunc(batches, func(batch returnBatch) bool { return batch.writeback == writeback }) + if index < 0 { + batches = append(batches, returnBatch{writeback: writeback}) + index = len(batches) - 1 + } + batches[index].packets = append(batches[index].packets, raw) case returnDrop: default: unconsumed = append(unconsumed, raw) } } - if len(writeBatch) > 0 { - if err := r.dispatcher.writeback.WriteReturnPackets(writeBatch); err != nil { + for _, batch := range batches { + err := batch.writeback.WriteReturnPackets(batch.packets) + if err != nil { r.dispatcher.logger.Trace(E.Cause(err, "write return packets")) } } return unconsumed } -func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap map[netip.Addr]*portNAT, headroom int, now int64) returnDecision { +func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap map[netip.Addr]*portNAT, headroom int, now int64) (returnDecision, ForwardWriteback) { if len(raw) < headroom+header.IPv4MinimumSize { - return returnPass + return returnPass, nil } parsed, ok := parseForwardPacket(raw[headroom:]) if !ok || parsed.fragment { - return returnPass + return returnPass, nil } if !parsed.hasFlow { - if parsed.isICMPError() && returnICMPError(natList, revMap, &parsed) { - return returnWrite + if parsed.isICMPError() { + flow := returnICMPError(natList, revMap, &parsed) + if flow != nil { + return returnWrite, flow.owner.writeback + } } - return returnPass + return returnPass, nil } flow := findReverseFlow(natList, revMap, parsed.flowKey()) if flow == nil { - return returnPass + return returnPass, nil } if flow.closed.Load() { - return returnDrop + return returnDrop, nil } if flow.tracker != nil { flow.tracker.CountReverse(len(raw) - headroom) @@ -805,7 +954,7 @@ func (r *forwardReturn) classifyReturn(raw []byte, natList []*portNAT, revMap ma } else { applyRewrite(&parsed, &flow.reverseRule) } - return returnWrite + return returnWrite, flow.owner.writeback } func findReverseFlow(natList []*portNAT, revMap map[netip.Addr]*portNAT, key flowKey) *forwardFlow { @@ -822,27 +971,28 @@ func findReverseFlow(natList []*portNAT, revMap map[netip.Addr]*portNAT, key flo return nil } -func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed *forwardPacket) bool { +func returnICMPError(natList []*portNAT, revMap map[netip.Addr]*portNAT, parsed *forwardPacket) *forwardFlow { inner, ok := parsed.icmpErrorInner() if !ok { - return false + return nil } embedded, parsedInner := parseEmbedded(inner) if !parsedInner { - return false + return nil } flow := findReverseFlow(natList, revMap, embedded.flowKey().reversed()) if flow == nil || flow.closed.Load() { - return false + return nil } rewriteEmbeddedSource(&embedded, addrToTCPIP(flow.clientAddress), flow.clientSelector, true) if flow.dnatAddress || flow.dnatPort { rewriteEmbeddedDestination(&embedded, addrToTCPIP(flow.clientDestinationAddress), flow.clientDestinationPort, flow.dnatPort) } - parsed.network.SetDestinationAddr(flow.clientAddress) - if parsed.network.SourceAddr() == flow.serverAddress { - parsed.network.SetSourceAddr(flow.clientDestinationAddress) + sourceAddress, destinationAddress := parsed.addressSlices() + copy(destinationAddress, flow.clientAddress.AsSlice()) + if parsed.source.Addr() == flow.serverAddress { + copy(sourceAddress, flow.clientDestinationAddress.AsSlice()) } recomputeChecksums(parsed) - return true + return flow } diff --git a/flow_dispatch_test.go b/flow_dispatch_test.go new file mode 100644 index 00000000..c8f54f9b --- /dev/null +++ b/flow_dispatch_test.go @@ -0,0 +1,56 @@ +package tun + +import ( + "net/netip" + "testing" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/logger" +) + +type closeOnAttachTracker struct{} + +func (closeOnAttachTracker) AttachFlow(handle FlowHandle) { handle.CloseFlow() } +func (closeOnAttachTracker) CountForward(int) {} +func (closeOnAttachTracker) CountReverse(int) {} +func (closeOnAttachTracker) FlowEstablished() {} +func (closeOnAttachTracker) CloseFlow(FlowCloseReason) {} + +type recordingPort struct { + address netip.Addr + written int +} + +func (p *recordingPort) PortAddresses() (netip.Addr, netip.Addr) { return p.address, netip.Addr{} } +func (*recordingPort) PortMTU() uint32 { return 0 } +func (*recordingPort) AttachReturn(Return) error { return nil } +func (*recordingPort) DetachReturn(Return) error { return nil } +func (p *recordingPort) WritePackets(packets [][]byte) error { + p.written += len(packets) + return nil +} + +type closeOnAttachHandler struct { + Handler + port Port +} + +func (h closeOnAttachHandler) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionFlow, Port: h.port, NewTracker: func() FlowTracker { return closeOnAttachTracker{} }} +} + +func TestForwardFlowClosedOnAttach(t *testing.T) { + port := &recordingPort{address: netip.MustParseAddr("192.0.2.1")} + dispatcher := NewForwardDispatcher(closeOnAttachHandler{port: port}, nil, logger.NOP(), 0, 0) + t.Cleanup(dispatcher.Close) + stage := dispatcher.NewStage(nil) + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(packet, 4, netip.MustParseAddrPort("198.18.0.2:12345"), netip.MustParseAddrPort("203.0.113.1:443"), []byte{1}, 1) + if !stage.Dispatch(packet) { + t.Fatal("packet was not handled") + } + stage.Flush() + if port.written != 0 { + t.Fatal("packet forwarded on a flow closed by its tracker") + } +} diff --git a/flow_dns.go b/flow_dns.go index 9c525073..125e9586 100644 --- a/flow_dns.go +++ b/flow_dns.go @@ -9,7 +9,6 @@ import ( "github.com/sagernet/sing/common/buf" E "github.com/sagernet/sing/common/exceptions" M "github.com/sagernet/sing/common/metadata" - N "github.com/sagernet/sing/common/network" ) func (d *ForwardDispatcher) hijackDNSPacket(packet *forwardPacket) { @@ -20,8 +19,6 @@ func (d *ForwardDispatcher) hijackDNSPacket(packet *forwardPacket) { d.handler.NewDNSPacket(header.UDP(packet.transport).Payload(), M.SocksaddrFromNetIP(packet.source), M.SocksaddrFromNetIP(packet.destination), writer) } -var _ N.PacketWriter = (*dnsResponseWriter)(nil) - type dnsResponseWriter struct { writeback ForwardWriteback source netip.AddrPort diff --git a/flow_mtu.go b/flow_mtu.go index 6ce275b3..a998774d 100644 --- a/flow_mtu.go +++ b/flow_mtu.go @@ -1,6 +1,8 @@ package tun import ( + "encoding/binary" + "github.com/sagernet/sing-tun/gtcpip/header" E "github.com/sagernet/sing/common/exceptions" ) @@ -11,16 +13,13 @@ const segmentRetainCount = 128 // Linux delivers TSO aggregates to the TUN even with IFF_VNET_HDR off // (observed on 6.x: the pre-segmentation skb is handed to the fd as-is). -func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacket, raw []byte) { +func (s *ForwardStage) resegmentTCP(flow *forwardFlow, packet *forwardPacket, raw []byte, effectiveMTU uint32) { if len(packet.transport) < header.TCPMinimumSize { return } headerLength := len(raw) - len(packet.transport) if packet.ipVersion == 6 && headerLength != header.IPv6MinimumSize { - reply, ok := buildPacketTooBig(packet.network.(header.IPv6), flow.effectiveMTU, d.writeback.ReturnHeadroom()) - if ok { - d.writebackBatch = append(d.writebackBatch, reply) - } + s.writebackBatch = append(s.writebackBatch, buildICMPError(packet, ICMPErrorPacketTooBig, effectiveMTU, s.writeback.ReturnHeadroom())) return } tcpHeaderLength := int(header.TCP(packet.transport).DataOffset()) @@ -28,7 +27,7 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke return } totalHeaderLength := headerLength + tcpHeaderLength - segmentSize := int(flow.effectiveMTU) - totalHeaderLength + segmentSize := int(effectiveMTU) - totalHeaderLength if segmentSize <= 0 { return } @@ -37,7 +36,7 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke gsoType = GSOTCPv6 } neededSegments := max((len(raw)-totalHeaderLength+segmentSize-1)/segmentSize, 1) - bufs, sizes := d.reserveSegments(neededSegments, int(flow.effectiveMTU)) + bufs, sizes := s.reserveSegments(neededSegments, int(effectiveMTU)) n, err := GSOSplit(raw, GSOOptions{ GSOType: gsoType, HdrLen: uint16(totalHeaderLength), @@ -46,30 +45,30 @@ func (d *ForwardDispatcher) resegmentTCP(flow *forwardFlow, packet *forwardPacke GSOSize: uint16(segmentSize), }, bufs, sizes, 0) if err != nil { - d.logger.Trace(E.Cause(err, "resegment packet")) + s.dispatcher.logger.Trace(E.Cause(err, "resegment packet")) return } for i := range n { - d.stagePort(flow.nat, bufs[i][:sizes[i]]) + s.stagePort(flow.nat, bufs[i][:sizes[i]]) } } -func (d *ForwardDispatcher) reserveSegments(count, size int) ([][]byte, []int) { - start := d.segmentUsed +func (s *ForwardStage) reserveSegments(count, size int) ([][]byte, []int) { + start := s.segmentUsed end := start + count - for len(d.segmentBuffers) < end { - d.segmentBuffers = append(d.segmentBuffers, make([]byte, size)) - d.segmentSizes = append(d.segmentSizes, 0) + for len(s.segmentBuffers) < end { + s.segmentBuffers = append(s.segmentBuffers, make([]byte, size)) + s.segmentSizes = append(s.segmentSizes, 0) } for i := start; i < end; i++ { - if cap(d.segmentBuffers[i]) < size { - d.segmentBuffers[i] = make([]byte, size) + if cap(s.segmentBuffers[i]) < size { + s.segmentBuffers[i] = make([]byte, size) } else { - d.segmentBuffers[i] = d.segmentBuffers[i][:size] + s.segmentBuffers[i] = s.segmentBuffers[i][:size] } } - d.segmentUsed = end - return d.segmentBuffers[start:end], d.segmentSizes[start:end] + s.segmentUsed = end + return s.segmentBuffers[start:end], s.segmentSizes[start:end] } const synthesizedTTL = 64 @@ -106,62 +105,37 @@ func fragmentIPv4Packet(packet header.IPv4, effectiveMTU uint32) ([][]byte, bool return fragments, true } -func buildFragmentationNeeded(packet header.IPv4, effectiveMTU uint32, headroom int) ([]byte, bool) { - advertised := max(effectiveMTU, header.IPv4MinimumMTU) - originalLength := min(int(packet.TotalLength()), len(packet)) - minPayloadLength := int(packet.HeaderLength()) + header.ICMPv4MinimumErrorPayloadSize - if originalLength < minPayloadLength { +func fragmentIPv6Packet(packet header.IPv6, effectiveMTU uint32, ident uint32) ([][]byte, bool) { + if len(packet) < header.IPv6MinimumSize { + return nil, false + } + payload := packet.Payload() + maxFragmentPayload := (int(effectiveMTU) - header.IPv6MinimumSize - header.IPv6FragmentHeaderSize) &^ 7 + if maxFragmentPayload <= 0 { return nil, false } - maxPayloadLength := header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - header.ICMPv4MinimumSize - payloadLength := min(originalLength, maxPayloadLength) - size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + payloadLength - buffer := make([]byte, headroom+size) - response := header.IPv4(buffer[headroom:]) - response.Encode(&header.IPv4Fields{ - TotalLength: uint16(size), - TTL: synthesizedTTL, - Protocol: uint8(header.ICMPv4ProtocolNumber), - SrcAddr: packet.DestinationAddr(), - DstAddr: packet.SourceAddr(), - }) - response.SetChecksum(^response.CalculateChecksum()) - icmpHdr := header.ICMPv4(response.Payload()) - icmpHdr.SetType(header.ICMPv4DstUnreachable) - icmpHdr.SetCode(header.ICMPv4FragmentationNeeded) - icmpHdr.SetMTU(uint16(min(advertised, uint32(0xffff)))) - copy(icmpHdr.Payload(), packet[:payloadLength]) - icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) - return buffer, true + transportProtocol := packet.NextHeader() + fragments := make([][]byte, 0, (len(payload)+maxFragmentPayload-1)/maxFragmentPayload) + for start := 0; start < len(payload); start += maxFragmentPayload { + end := min(start+maxFragmentPayload, len(payload)) + fragment := header.IPv6(make([]byte, header.IPv6MinimumSize+header.IPv6FragmentHeaderSize+end-start)) + copy(fragment, packet[:header.IPv6MinimumSize]) + fragment.SetNextHeader(header.IPv6FragmentHeader) + fragment.SetPayloadLength(uint16(header.IPv6FragmentHeaderSize + end - start)) + encodeIPv6FragmentHeader(fragment[header.IPv6MinimumSize:], transportProtocol, start, end < len(payload), ident) + copy(fragment[header.IPv6MinimumSize+header.IPv6FragmentHeaderSize:], payload[start:end]) + fragments = append(fragments, fragment) + } + return fragments, true } -func buildPacketTooBig(packet header.IPv6, effectiveMTU uint32, headroom int) ([]byte, bool) { - advertised := max(effectiveMTU, header.IPv6MinimumMTU) - originalLength := min(header.IPv6MinimumSize+int(packet.PayloadLength()), len(packet)) - if originalLength < header.IPv6MinimumSize { - return nil, false +func encodeIPv6FragmentHeader(target []byte, nextHeader uint8, offset int, more bool, ident uint32) { + target[0] = nextHeader + target[1] = 0 + value := uint16(offset) + if more { + value |= 1 } - maxPayloadLength := header.IPv6MinimumMTU - header.IPv6MinimumSize - header.ICMPv6PacketTooBigMinimumSize - payloadLength := min(originalLength, maxPayloadLength) - size := header.IPv6MinimumSize + header.ICMPv6PacketTooBigMinimumSize + payloadLength - buffer := make([]byte, headroom+size) - response := header.IPv6(buffer[headroom:]) - response.Encode(&header.IPv6Fields{ - PayloadLength: uint16(header.ICMPv6PacketTooBigMinimumSize + payloadLength), - TransportProtocol: header.ICMPv6ProtocolNumber, - HopLimit: synthesizedTTL, - SrcAddr: packet.DestinationAddr(), - DstAddr: packet.SourceAddr(), - }) - icmpHdr := header.ICMPv6(response.Payload()) - icmpHdr.SetType(header.ICMPv6PacketTooBig) - icmpHdr.SetCode(header.ICMPv6UnusedCode) - icmpHdr.SetMTU(advertised) - copy(icmpHdr.Payload(), packet[:payloadLength]) - icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ - Header: icmpHdr, - Src: response.SourceAddressSlice(), - Dst: response.DestinationAddressSlice(), - })) - return buffer, true + binary.BigEndian.PutUint16(target[2:], value) + binary.BigEndian.PutUint32(target[4:], ident) } diff --git a/flow_nat.go b/flow_nat.go index 4d2bca3b..39e4737a 100644 --- a/flow_nat.go +++ b/flow_nat.go @@ -22,8 +22,8 @@ type portNAT struct { selectorStart uint16 selectorCount uint16 - counter uint32 - pending [][]byte + allocateAccess sync.Mutex + counter uint32 } type natShard struct { @@ -70,6 +70,18 @@ func (n *portNAT) insert(key flowKey, flow *forwardFlow) { shard.access.Unlock() } +func (n *portNAT) reserve(key flowKey) bool { + shard := n.shard(key) + shard.access.Lock() + defer shard.access.Unlock() + _, occupied := shard.flows[key] + if occupied { + return false + } + shard.flows[key] = nil + return true +} + func (n *portNAT) delete(key flowKey) { shard := n.shard(key) shard.access.Lock() @@ -100,20 +112,22 @@ func (n *portNAT) selectorRange(protocol uint8) (uint16, uint32) { return n.selectorStart, uint32(n.selectorCount) } -func (n *portNAT) allocateSelector(protocol uint8, portAddress, serverAddress netip.Addr, serverPort, clientSelector uint16) (uint16, flowKey, bool) { +func (n *portNAT) reserveSelector(protocol uint8, portAddress, serverAddress netip.Addr, serverPort, clientSelector uint16) (uint16, flowKey, bool) { rangeStart, rangeCount := n.selectorRange(protocol) if clientSelector != 0 && clientSelector >= rangeStart && uint32(clientSelector-rangeStart) < rangeCount { key := n.reverseKeyFor(protocol, portAddress, serverAddress, serverPort, clientSelector) - if n.lookup(key) == nil { + if n.reserve(key) { return clientSelector, key, true } } + n.allocateAccess.Lock() + defer n.allocateAccess.Unlock() for range rangeCount { n.counter++ candidate := rangeStart + uint16(n.counter%rangeCount) key := n.reverseKeyFor(protocol, portAddress, serverAddress, serverPort, candidate) - if n.lookup(key) == nil { + if n.reserve(key) { return candidate, key, true } } diff --git a/flow_parse.go b/flow_parse.go index 2a346710..a8977176 100644 --- a/flow_parse.go +++ b/flow_parse.go @@ -20,7 +20,7 @@ func (k flowKey) reversed() flowKey { type forwardPacket struct { ipVersion uint8 protocol uint8 - network header.Network + network []byte transport []byte source netip.AddrPort destination netip.AddrPort @@ -28,12 +28,22 @@ type forwardPacket struct { icmpType uint8 fragment bool hasFlow bool + verdict FlowVerdict } func (p *forwardPacket) flowKey() flowKey { return flowKey{protocol: p.protocol, source: p.source, destination: p.destination} } +func (p *forwardPacket) addressSlices() ([]byte, []byte) { + if p.ipVersion == 4 { + ipHdr := header.IPv4(p.network) + return ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice() + } + ipHdr := header.IPv6(p.network) + return ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice() +} + func (p *forwardPacket) isTCPSyn() bool { return p.protocol == uint8(header.TCPProtocolNumber) && p.tcpFlags&header.TCPFlagSyn != 0 } @@ -48,7 +58,7 @@ func parseForwardPacket(packet []byte) (forwardPacket, bool) { parsed := forwardPacket{ ipVersion: 4, protocol: uint8(ipHdr.TransportProtocol()), - network: ipHdr, + network: packet, source: netip.AddrPortFrom(ipHdr.SourceAddr(), 0), destination: netip.AddrPortFrom(ipHdr.DestinationAddr(), 0), } @@ -63,19 +73,22 @@ func parseForwardPacket(packet []byte) (forwardPacket, bool) { if !ipHdr.IsValid(len(packet)) { return forwardPacket{}, false } - protocol, payload, fragment, transportPresent := skipIPv6ExtensionHeaders(uint8(ipHdr.TransportProtocol()), ipHdr.Payload()) + protocol := uint8(ipHdr.TransportProtocol()) parsed := forwardPacket{ ipVersion: 6, protocol: protocol, - network: ipHdr, + network: packet, source: netip.AddrPortFrom(ipHdr.SourceAddr(), 0), destination: netip.AddrPortFrom(ipHdr.DestinationAddr(), 0), - fragment: fragment, } - if fragment || !transportPresent { + switch header.IPv6ExtensionHeaderIdentifier(protocol) { + case header.IPv6HopByHopOptionsExtHdrIdentifier, header.IPv6RoutingExtHdrIdentifier, header.IPv6DestinationOptionsExtHdrIdentifier: + return forwardPacket{}, false + case header.IPv6FragmentExtHdrIdentifier: + parsed.fragment = true return parsed, true } - parsed.parseTransport(payload) + parsed.parseTransport(ipHdr.Payload()) return parsed, true default: return forwardPacket{}, false diff --git a/flow_reject.go b/flow_reject.go index af82eb5c..a6a623a8 100644 --- a/flow_reject.go +++ b/flow_reject.go @@ -1,9 +1,6 @@ package tun import ( - "net/netip" - - "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/gtcpip/header" ) @@ -14,35 +11,26 @@ func buildReject(packet *forwardPacket, headroom int) ([]byte, bool) { return nil, false } tcpHdr := header.TCP(packet.transport) - switch ipHdr := packet.network.(type) { - case header.IPv4: - return buildResetIPv4(ipHdr, tcpHdr, headroom), true - case header.IPv6: - return buildResetIPv6(ipHdr, tcpHdr, headroom), true - default: - return nil, false + if packet.ipVersion == 4 { + return buildResetIPv4(header.IPv4(packet.network), tcpHdr, headroom), true } + return buildResetIPv6(header.IPv6(packet.network), tcpHdr, headroom), true case uint8(header.UDPProtocolNumber): - switch ipHdr := packet.network.(type) { - case header.IPv4: - return buildRejectICMPv4(ipHdr, header.ICMPv4PortUnreachable, ipHdr.DestinationAddr(), headroom) - case header.IPv6: - return buildRejectICMPv6(ipHdr, header.ICMPv6PortUnreachable, ipHdr.DestinationAddr(), headroom) - default: - return nil, false - } + return buildICMPError(packet, ICMPErrorPortUnreachable, 0, headroom), true default: - switch ipHdr := packet.network.(type) { - case header.IPv4: - return buildRejectICMPv4(ipHdr, header.ICMPv4HostUnreachable, ipHdr.DestinationAddr(), headroom) - case header.IPv6: - return buildRejectICMPv6(ipHdr, header.ICMPv6AddressUnreachable, ipHdr.DestinationAddr(), headroom) - default: - return nil, false - } + return buildICMPError(packet, ICMPErrorAddressUnreachable, 0, headroom), true } } +func buildICMPError(packet *forwardPacket, errorType ICMPError, mtu uint32, headroom int) []byte { + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) + return buildICMPv4Error(ipHdr, errorType, ipHdr.DestinationAddr(), mtu, headroom) + } + ipHdr := header.IPv6(packet.network) + return buildICMPv6Error(ipHdr, errorType, ipHdr.DestinationAddr(), mtu, headroom) +} + func buildResetIPv4(origIPHdr header.IPv4, origTCPHdr header.TCP, headroom int) []byte { size := header.IPv4MinimumSize + header.TCPMinimumSize buffer := make([]byte, headroom+size) @@ -100,111 +88,3 @@ func encodeResetTCP(tcpHdr header.TCP, origTCPHdr header.TCP) { } tcpHdr.Encode(&fields) } - -func buildRejectICMPv4(ipHdr header.IPv4, code header.ICMPv4Code, source netip.Addr, headroom int) ([]byte, bool) { - const maxIPData = header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - available := maxIPData - header.ICMPv4MinimumSize - if len(ipHdr) < header.ICMPv4MinimumErrorPayloadSize { - return nil, false - } - payload := []byte(ipHdr) - if len(payload) > available { - payload = payload[:available] - } - size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + len(payload) - buffer := make([]byte, headroom+size) - newIPHdr := header.IPv4(buffer[headroom:]) - newIPHdr.Encode(&header.IPv4Fields{ - TotalLength: uint16(size), - TTL: synthesizedTTL, - Protocol: uint8(header.ICMPv4ProtocolNumber), - SrcAddr: source, - DstAddr: ipHdr.SourceAddr(), - }) - newIPHdr.SetChecksum(^newIPHdr.CalculateChecksum()) - icmpHdr := header.ICMPv4(newIPHdr.Payload()) - icmpHdr.SetType(header.ICMPv4DstUnreachable) - icmpHdr.SetCode(code) - copy(icmpHdr.Payload(), payload) - icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr[:header.ICMPv4MinimumSize], checksum.Checksum(payload, 0))) - return buffer, true -} - -func buildRejectICMPv6(ipHdr header.IPv6, code header.ICMPv6Code, source netip.Addr, headroom int) ([]byte, bool) { - const maxIPv6Data = header.IPv6MinimumMTU - header.IPv6FixedHeaderSize - available := maxIPv6Data - header.ICMPv6ErrorHeaderSize - if available < header.IPv6MinimumSize { - return nil, false - } - payload := []byte(ipHdr) - if len(payload) > available { - payload = payload[:available] - } - size := header.IPv6MinimumSize + header.ICMPv6DstUnreachableMinimumSize + len(payload) - buffer := make([]byte, headroom+size) - newIPHdr := header.IPv6(buffer[headroom:]) - newIPHdr.Encode(&header.IPv6Fields{ - PayloadLength: uint16(header.ICMPv6DstUnreachableMinimumSize + len(payload)), - TransportProtocol: header.ICMPv6ProtocolNumber, - HopLimit: synthesizedTTL, - SrcAddr: source, - DstAddr: ipHdr.SourceAddr(), - }) - icmpHdr := header.ICMPv6(newIPHdr.Payload()) - icmpHdr.SetType(header.ICMPv6DstUnreachable) - icmpHdr.SetCode(code) - icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ - Header: icmpHdr[:header.ICMPv6DstUnreachableMinimumSize], - Src: newIPHdr.SourceAddressSlice(), - Dst: newIPHdr.DestinationAddressSlice(), - PayloadCsum: checksum.Checksum(payload, 0), - PayloadLen: len(payload), - })) - copy(icmpHdr.Payload(), payload) - return buffer, true -} - -func BuildUnreachable(packet []byte, source netip.Addr, headroom int) ([]byte, bool) { - switch header.IPVersion(packet) { - case header.IPv4Version: - ipHdr := header.IPv4(packet) - if !ipHdr.IsValid(len(packet)) || ipHdr.FragmentOffset() != 0 { - return nil, false - } - sourceAddr := ipHdr.SourceAddr() - if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { - return nil, false - } - if ipHdr.TransportProtocol() == header.ICMPv4ProtocolNumber { - if len(ipHdr.Payload()) < header.ICMPv4MinimumSize || header.ICMPv4(ipHdr.Payload()).Type() != header.ICMPv4Echo { - return nil, false - } - } - replySource := ipHdr.DestinationAddr() - if source.Is4() { - replySource = source - } - return buildRejectICMPv4(ipHdr, header.ICMPv4HostUnreachable, replySource, headroom) - case header.IPv6Version: - ipHdr := header.IPv6(packet) - if !ipHdr.IsValid(len(packet)) { - return nil, false - } - sourceAddr := ipHdr.SourceAddr() - if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { - return nil, false - } - if ipHdr.TransportProtocol() == header.ICMPv6ProtocolNumber { - if len(ipHdr.Payload()) < header.ICMPv6MinimumSize || header.ICMPv6(ipHdr.Payload()).Type() != header.ICMPv6EchoRequest { - return nil, false - } - } - replySource := ipHdr.DestinationAddr() - if source.Is6() { - replySource = source - } - return buildRejectICMPv6(ipHdr, header.ICMPv6NetworkUnreachable, replySource, headroom) - default: - return nil, false - } -} diff --git a/flow_rewrite.go b/flow_rewrite.go index a4227654..e99deaa6 100644 --- a/flow_rewrite.go +++ b/flow_rewrite.go @@ -1,6 +1,7 @@ package tun import ( + "bytes" "encoding/binary" "github.com/sagernet/sing-tun/gtcpip" @@ -18,30 +19,17 @@ type rewriteRule struct { } func applyRewrite(packet *forwardPacket, rule *rewriteRule) { - oldSource := packet.network.SourceAddress() - oldDestination := packet.network.DestinationAddress() - newSource := oldSource - newDestination := oldDestination + sourceAddress, destinationAddress := packet.addressSlices() + var addressDelta uint64 if rule.sourceAddress.Len() > 0 { - newSource = rule.sourceAddress + addressDelta += rewriteAddress(sourceAddress, rule.sourceAddress.AsSlice()) } if rule.destinationAddress.Len() > 0 { - newDestination = rule.destinationAddress + addressDelta += rewriteAddress(destinationAddress, rule.destinationAddress.AsSlice()) } - if ipHdr, isIPv4 := packet.network.(header.IPv4); isIPv4 { - if newSource != oldSource { - ipHdr.SetSourceAddressWithChecksumUpdate(newSource) - } - if newDestination != oldDestination { - ipHdr.SetDestinationAddressWithChecksumUpdate(newDestination) - } - } else { - if newSource != oldSource { - packet.network.SetSourceAddress(newSource) - } - if newDestination != oldDestination { - packet.network.SetDestinationAddress(newDestination) - } + if packet.ipVersion == 4 && addressDelta != 0 { + ipHdr := header.IPv4(packet.network) + ipHdr.SetChecksum(updateChecksum(ipHdr.Checksum(), addressDelta)) } transport := packet.transport switch packet.protocol { @@ -49,18 +37,10 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { if len(transport) < header.TCPMinimumSize { return } - tcpHdr := header.TCP(transport) - if newSource != oldSource { - tcpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource, true) - } - if newDestination != oldDestination { - tcpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination, true) - } - if rule.rewriteSourcePort { - tcpHdr.SetSourcePortWithChecksumUpdate(rule.sourcePort) - } - if rule.rewriteDestinationPort { - tcpHdr.SetDestinationPortWithChecksumUpdate(rule.destinationPort) + delta := addressDelta + rewritePorts(transport, rule) + if delta != 0 { + tcpHdr := header.TCP(transport) + tcpHdr.SetChecksum(updateChecksum(tcpHdr.Checksum(), delta)) } case uint8(header.UDPProtocolNumber): if len(transport) < header.UDPMinimumSize { @@ -68,25 +48,12 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } udpHdr := header.UDP(transport) if packet.ipVersion == 4 && udpHdr.Checksum() == 0 { - if rule.rewriteSourcePort { - udpHdr.SetSourcePort(rule.sourcePort) - } - if rule.rewriteDestinationPort { - udpHdr.SetDestinationPort(rule.destinationPort) - } + rewritePorts(transport, rule) return } - if newSource != oldSource { - udpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource, true) - } - if newDestination != oldDestination { - udpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination, true) - } - if rule.rewriteSourcePort { - udpHdr.SetSourcePortWithChecksumUpdate(rule.sourcePort) - } - if rule.rewriteDestinationPort { - udpHdr.SetDestinationPortWithChecksumUpdate(rule.destinationPort) + delta := addressDelta + rewritePorts(transport, rule) + if delta != 0 { + udpHdr.SetChecksum(updateChecksum(udpHdr.Checksum(), delta)) } if udpHdr.Checksum() == 0 { udpHdr.SetChecksum(0xffff) @@ -106,11 +73,8 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { return } icmpHdr := header.ICMPv6(transport) - if newSource != oldSource { - icmpHdr.UpdateChecksumPseudoHeaderAddress(oldSource, newSource) - } - if newDestination != oldDestination { - icmpHdr.UpdateChecksumPseudoHeaderAddress(oldDestination, newDestination) + if addressDelta != 0 { + icmpHdr.SetChecksum(updateChecksum(icmpHdr.Checksum(), addressDelta)) } if rule.rewriteSourcePort { icmpHdr.SetIdentWithChecksumUpdate(rule.sourcePort) @@ -120,12 +84,49 @@ func applyRewrite(packet *forwardPacket, rule *rewriteRule) { } } +func rewriteAddress(field []byte, address []byte) uint64 { + if bytes.Equal(field, address) { + return 0 + } + var delta uint64 + for offset := 0; offset < len(field); offset += 4 { + delta += uint64(^binary.BigEndian.Uint32(field[offset:])) + uint64(binary.BigEndian.Uint32(address[offset:])) + } + copy(field, address) + return delta +} + +func rewritePorts(transport []byte, rule *rewriteRule) uint64 { + var delta uint64 + if rule.rewriteSourcePort { + delta += checksumDelta(binary.BigEndian.Uint16(transport), rule.sourcePort) + binary.BigEndian.PutUint16(transport, rule.sourcePort) + } + if rule.rewriteDestinationPort { + delta += checksumDelta(binary.BigEndian.Uint16(transport[2:]), rule.destinationPort) + binary.BigEndian.PutUint16(transport[2:], rule.destinationPort) + } + return delta +} + +func checksumDelta(previous uint16, next uint16) uint64 { + if previous == next { + return 0 + } + return uint64(^previous) + uint64(next) +} + +func updateChecksum(current uint16, delta uint64) uint16 { + return ^checksum.Fold(uint64(^current) + delta) +} + func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { + sourceAddress, destinationAddress := packet.addressSlices() if rule.sourceAddress.Len() > 0 { - packet.network.SetSourceAddress(rule.sourceAddress) + copy(sourceAddress, rule.sourceAddress.AsSlice()) } if rule.destinationAddress.Len() > 0 { - packet.network.SetDestinationAddress(rule.destinationAddress) + copy(destinationAddress, rule.destinationAddress.AsSlice()) } transport := packet.transport switch packet.protocol { @@ -175,7 +176,9 @@ func applyRewriteRaw(packet *forwardPacket, rule *rewriteRule) { } func recomputeChecksums(packet *forwardPacket) { - if ipHdr, isIPv4 := packet.network.(header.IPv4); isIPv4 { + sourceAddress, destinationAddress := packet.addressSlices() + if packet.ipVersion == 4 { + ipHdr := header.IPv4(packet.network) ipHdr.SetChecksum(0) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) } @@ -188,7 +191,7 @@ func recomputeChecksums(packet *forwardPacket) { tcpHdr := header.TCP(transport) tcpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(tcpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, packet.network.SourceAddressSlice(), packet.network.DestinationAddressSlice(), uint16(len(transport))) + pseudoChecksum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, sourceAddress, destinationAddress, uint16(len(transport))) tcpHdr.SetChecksum(^tcpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum))) case uint8(header.UDPProtocolNumber): if len(transport) < header.UDPMinimumSize { @@ -200,7 +203,7 @@ func recomputeChecksums(packet *forwardPacket) { } udpHdr.SetChecksum(0) payloadChecksum := checksum.Checksum(udpHdr.Payload(), 0) - pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, packet.network.SourceAddressSlice(), packet.network.DestinationAddressSlice(), udpHdr.Length()) + pseudoChecksum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpHdr.Length()) udpChecksum := ^udpHdr.CalculateChecksum(checksum.Combine(pseudoChecksum, payloadChecksum)) if udpChecksum == 0 { udpChecksum = 0xffff @@ -221,8 +224,8 @@ func recomputeChecksums(packet *forwardPacket) { icmpHdr.SetChecksum(0) icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ Header: icmpHdr, - Src: packet.network.SourceAddressSlice(), - Dst: packet.network.DestinationAddressSlice(), + Src: sourceAddress, + Dst: destinationAddress, })) } } @@ -240,13 +243,7 @@ func clampTCPMSS(packet *forwardPacket, effectiveMTU uint32) { if tcpHeaderLength < header.TCPMinimumSize || tcpHeaderLength > len(transport) { return } - var networkHeaderLength int - switch packet.ipVersion { - case 4: - networkHeaderLength = len(packet.network.(header.IPv4)) - len(transport) - default: - networkHeaderLength = len(packet.network.(header.IPv6)) - len(transport) - } + networkHeaderLength := len(packet.network) - len(transport) if effectiveMTU <= uint32(networkHeaderLength+header.TCPMinimumSize) { return } diff --git a/go.mod b/go.mod index fbba8e2c..deffaaf4 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( github.com/mdlayher/netlink v1.11.2 github.com/sagernet/fswatch v0.1.2 github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 - github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a + github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf github.com/sagernet/nftables v0.3.0-mod.4 - github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 + github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0 github.com/stretchr/testify v1.11.1 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc diff --git a/go.sum b/go.sum index 0c72db92..d644c5de 100644 --- a/go.sum +++ b/go.sum @@ -22,12 +22,12 @@ github.com/sagernet/fswatch v0.1.2 h1:/TT7k4mkce1qFPxamLO842WjqBgbTBiXP2mlUjp9PF github.com/sagernet/fswatch v0.1.2/go.mod h1:5BpGmpUQVd3Mc5r313HRpvADHRg3/rKn5QbwFteB880= github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1 h1:IdQ7yTKkB2wv8txwshxUroPlO4npOYAV71xb7xQ7Lys= github.com/sagernet/gvisor v0.0.0-20260727.0-sing-box-mod.1/go.mod h1:9O3SQskYuCfdHNvHEsWuEAgoyKEF74PiWp4NsNUia8g= -github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a h1:ObwtHN2VpqE0ZNjr6sGeT00J8uU7JF4cNUdb44/Duis= -github.com/sagernet/netlink v0.0.0-20240612041022-b9a21c07ac6a/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= +github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf h1:b/3rm+KvxzuYRcnMdLq/hmcqX6/2B8k24l6SDHL2Piw= +github.com/sagernet/netlink v0.0.0-20260814022025-64455d367bbf/go.mod h1:xLnfdiJbSp8rNqYEdIW/6eDO4mVoogml14Bh2hSiFpM= github.com/sagernet/nftables v0.3.0-mod.4 h1:vnOtcDYeSXv2e5RoRuGH0lrpttQFJ8iC4ICS2nhlDSo= github.com/sagernet/nftables v0.3.0-mod.4/go.mod h1:8kslHG4VvYNihcco+i6uxIX7qbT8A56T0y5q7U44ZaQ= -github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8 h1:dyRIj+MZ2rc9JVzJoG04jxu+MpvHrLIZLJr0QjNAMGg= -github.com/sagernet/sing v0.8.12-0.20260717023913-84ab32b56cb8/go.mod h1:olXxWQNqRW/l2Q6JI3b2Qmz8iQnIFlOeeH8bx6JhgUA= +github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0 h1:rn2MLe/qiqCxvRIHfT9WRc22eNLBTRCKlY39pZTWRdg= +github.com/sagernet/sing v0.9.5-0.20260917142815-f37989cadbf0/go.mod h1:K3Owt3xPhHugvlnlPPxZJ/exXdaJfEPOTNorGk4AXjo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= diff --git a/gtcpip/checksum/checksum_default.go b/gtcpip/checksum/checksum_default.go index 99a2d753..932481f0 100644 --- a/gtcpip/checksum/checksum_default.go +++ b/gtcpip/checksum/checksum_default.go @@ -1,4 +1,4 @@ -//go:build !amd64 +//go:build !amd64 && !arm64 package checksum diff --git a/gtcpip/checksum/checksum_ts.go b/gtcpip/checksum/checksum_ts.go index f6766d3f..14aa3a47 100644 --- a/gtcpip/checksum/checksum_ts.go +++ b/gtcpip/checksum/checksum_ts.go @@ -1,4 +1,4 @@ -//go:build amd64 +//go:build amd64 || arm64 package checksum diff --git a/gtcpip/checksum/checksum_unsafe.go b/gtcpip/checksum/checksum_unsafe.go index 66b7ab67..558dc6db 100644 --- a/gtcpip/checksum/checksum_unsafe.go +++ b/gtcpip/checksum/checksum_unsafe.go @@ -52,7 +52,7 @@ func calculateChecksum(buf []byte, odd bool, initial uint16) (uint16, bool) { // buf = buf[1:] is skipped because it's unused and nogo will // complain. } - return reduce(acc), odd + return Fold(acc), odd } // On little-endian architectures, multi-byte loads from buf will load @@ -147,14 +147,14 @@ func calculateChecksum(buf []byte, odd bool, initial uint16) (uint16, bool) { } // Reduce the checksum to 16 bits and undo byte swaps before returning. - acc16 := bswapIfLittleEndian16(reduce(acc)) + acc16 := bswapIfLittleEndian16(Fold(acc)) if bswapped { acc16 = bits.ReverseBytes16(acc16) } return acc16, odd } -func reduce(acc uint64) uint16 { +func Fold(acc uint64) uint16 { // Ideally we would do: // return uint16(acc>>48) +' uint16(acc>>32) +' uint16(acc>>16) +' uint16(acc) // for more instruction-level parallelism; however, there is no diff --git a/gtcpip/header/checksum.go b/gtcpip/header/checksum.go index 303502cc..72274600 100644 --- a/gtcpip/header/checksum.go +++ b/gtcpip/header/checksum.go @@ -22,21 +22,22 @@ import ( "github.com/sagernet/sing-tun/gtcpip" "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/internal/tschecksum" ) // PseudoHeaderChecksum calculates the pseudo-header checksum for the given // destination protocol and network address. Pseudo-headers are needed by // transport layers when calculating their own checksum. func PseudoHeaderChecksum(protocol tcpip.TransportProtocolNumber, srcAddr []byte, dstAddr []byte, totalLen uint16) uint16 { + if (len(srcAddr) == 4 || len(srcAddr) == 16) && (len(dstAddr) == 4 || len(dstAddr) == 16) { + return tschecksum.PseudoHeaderChecksum(uint8(protocol), srcAddr, dstAddr, totalLen) + } xsum := checksum.Checksum(srcAddr, 0) xsum = checksum.Checksum(dstAddr, xsum) - - // Add the length portion of the checksum to the pseudo-checksum. - var tmp [2]byte - binary.BigEndian.PutUint16(tmp[:], totalLen) - xsum = checksum.Checksum(tmp[:], xsum) - - return checksum.Checksum([]byte{0, uint8(protocol)}, xsum) + var trailer [4]byte + binary.BigEndian.PutUint16(trailer[:2], totalLen) + trailer[3] = uint8(protocol) + return checksum.Checksum(trailer[:], xsum) } // checksumUpdate2ByteAlignedUint16 updates a uint16 value in a calculated diff --git a/gtcpip/header/ipv4.go b/gtcpip/header/ipv4.go index 3041eae9..65b7ca17 100644 --- a/gtcpip/header/ipv4.go +++ b/gtcpip/header/ipv4.go @@ -478,10 +478,16 @@ func (b IPv4) SetDestinationAddress(addr tcpip.Address) { // CalculateChecksum calculates the checksum of the IPv4 header. func (b IPv4) CalculateChecksum() uint16 { - // return checksum.Checksum(b[:b.HeaderLength()], 0) - xsum0 := checksum.Checksum(b[:xsum], 0) - xsum0 = checksum.Checksum(b[xsum+2:b.HeaderLength()], xsum0) - return xsum0 + _ = b[IPv4MinimumSize-1] + sum := uint64(binary.BigEndian.Uint32(b)) + + uint64(binary.BigEndian.Uint32(b[id:])) + + uint64(binary.BigEndian.Uint16(b[ttl:])) + + uint64(binary.BigEndian.Uint32(b[srcAddr:])) + + uint64(binary.BigEndian.Uint32(b[dstAddr:])) + for offset := IPv4MinimumSize; offset < int(b.HeaderLength()); offset += 4 { + sum += uint64(binary.BigEndian.Uint32(b[offset:])) + } + return checksum.Fold(sum) } // Encode encodes all the fields of the IPv4 header. @@ -573,8 +579,7 @@ func (b IPv4) IsChecksumValid() bool { // same set of octets, including the checksum field. If the result // is all 1 bits (-0 in 1's complement arithmetic), the check // succeeds. - //return b.CalculateChecksum() == 0xffff - return checksum.Checksum(b[:b.HeaderLength()], 0) == 0xffff + return checksum.Combine(b.CalculateChecksum(), b.Checksum()) == 0xffff } // IsV4MulticastAddress determines if the provided address is an IPv4 multicast diff --git a/gtcpip/header/udp.go b/gtcpip/header/udp.go index ce7708e1..531d1d77 100644 --- a/gtcpip/header/udp.go +++ b/gtcpip/header/udp.go @@ -113,11 +113,9 @@ func (b UDP) SetLength(length uint16) { // CalculateChecksum calculates the checksum of the UDP packet, given the // checksum of the network-layer pseudo-header and the checksum of the payload. func (b UDP) CalculateChecksum(partialChecksum uint16) uint16 { - // Calculate the rest of the checksum.\ - // return checksum.Checksum(b[:UDPMinimumSize], partialChecksum) - xsum := checksum.Checksum(b[:udpChecksum], partialChecksum) - xsum = checksum.Checksum(b[udpChecksum+2:UDPMinimumSize], xsum) - return xsum + return checksum.Fold(uint64(partialChecksum) + + uint64(binary.BigEndian.Uint32(b[udpSrcPort:])) + + uint64(binary.BigEndian.Uint16(b[udpLength:]))) } // IsChecksumValid returns true iff the UDP header's checksum is valid. diff --git a/icmp_error.go b/icmp_error.go new file mode 100644 index 00000000..88a41365 --- /dev/null +++ b/icmp_error.go @@ -0,0 +1,203 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + + "github.com/sagernet/sing-tun/gtcpip/header" +) + +type ICMPError uint8 + +const ( + ICMPErrorNoRoute ICMPError = iota + ICMPErrorAddressUnreachable + ICMPErrorPortUnreachable + ICMPErrorSourcePolicy + ICMPErrorHopLimitExceeded + ICMPErrorPacketTooBig +) + +func BuildICMPError(packet []byte, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) ([]byte, bool) { + switch header.IPVersion(packet) { + case header.IPv4Version: + ipHdr := header.IPv4(packet) + if !ipHdr.IsValid(len(packet)) || ipHdr.FragmentOffset() != 0 { + return nil, false + } + sourceAddr := ipHdr.SourceAddr() + if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { + return nil, false + } + destinationAddr := ipHdr.DestinationAddr() + if destinationAddr.IsMulticast() || destinationAddr == netip.AddrFrom4([4]byte{0xff, 0xff, 0xff, 0xff}) { + return nil, false + } + if ipHdr.TransportProtocol() == header.ICMPv4ProtocolNumber { + if len(ipHdr.Payload()) < header.ICMPv4MinimumSize { + return nil, false + } + switch header.ICMPv4(ipHdr.Payload()).Type() { + case header.ICMPv4DstUnreachable, header.ICMPv4SrcQuench, header.ICMPv4Redirect, header.ICMPv4TimeExceeded, header.ICMPv4ParamProblem: + return nil, false + } + } + if !source.Is4() { + source = destinationAddr + } + return buildICMPv4Error(ipHdr, errorType, source, mtu, headroom), true + case header.IPv6Version: + ipHdr := header.IPv6(packet) + if !ipHdr.IsValid(len(packet)) { + return nil, false + } + sourceAddr := ipHdr.SourceAddr() + if sourceAddr.IsUnspecified() || sourceAddr.IsMulticast() { + return nil, false + } + destinationAddr := ipHdr.DestinationAddr() + if destinationAddr.IsMulticast() && errorType != ICMPErrorPacketTooBig { + return nil, false + } + if ipHdr.TransportProtocol() == header.ICMPv6ProtocolNumber { + if len(ipHdr.Payload()) < header.ICMPv6MinimumSize { + return nil, false + } + icmpType := header.ICMPv6(ipHdr.Payload()).Type() + if icmpType < header.ICMPv6EchoRequest || icmpType == header.ICMPv6RedirectMsg { + return nil, false + } + } + if !source.Is6() || source.Is4In6() { + source = destinationAddr + } + return buildICMPv6Error(ipHdr, errorType, source, mtu, headroom), true + default: + return nil, false + } +} + +func buildICMPv4Error(packet header.IPv4, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) []byte { + icmpType := header.ICMPv4DstUnreachable + var icmpCode header.ICMPv4Code + switch errorType { + case ICMPErrorNoRoute, ICMPErrorAddressUnreachable: + icmpCode = header.ICMPv4HostUnreachable + case ICMPErrorPortUnreachable: + icmpCode = header.ICMPv4PortUnreachable + case ICMPErrorSourcePolicy: + icmpCode = header.ICMPv4AdminProhibited + case ICMPErrorHopLimitExceeded: + icmpType = header.ICMPv4TimeExceeded + icmpCode = header.ICMPv4TTLExceeded + case ICMPErrorPacketTooBig: + icmpCode = header.ICMPv4FragmentationNeeded + } + const maxPayloadLength = header.IPv4MinimumProcessableDatagramSize - header.IPv4MinimumSize - header.ICMPv4MinimumSize + payloadLength := min(int(packet.TotalLength()), len(packet), maxPayloadLength) + size := header.IPv4MinimumSize + header.ICMPv4MinimumSize + payloadLength + buffer := make([]byte, headroom+size) + response := header.IPv4(buffer[headroom:]) + response.Encode(&header.IPv4Fields{ + TotalLength: uint16(size), + TTL: synthesizedTTL, + Protocol: uint8(header.ICMPv4ProtocolNumber), + SrcAddr: source, + DstAddr: packet.SourceAddr(), + }) + response.SetChecksum(^response.CalculateChecksum()) + icmpHdr := header.ICMPv4(response.Payload()) + icmpHdr.SetType(icmpType) + icmpHdr.SetCode(icmpCode) + if errorType == ICMPErrorPacketTooBig { + icmpHdr.SetMTU(uint16(min(max(mtu, header.IPv4MinimumMTU), 0xffff))) + } + copy(icmpHdr.Payload(), packet[:payloadLength]) + icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) + return buffer +} + +func buildICMPv6Error(packet header.IPv6, errorType ICMPError, source netip.Addr, mtu uint32, headroom int) []byte { + icmpType := header.ICMPv6DstUnreachable + var icmpCode header.ICMPv6Code + switch errorType { + case ICMPErrorNoRoute: + icmpCode = header.ICMPv6NetworkUnreachable + case ICMPErrorAddressUnreachable: + icmpCode = header.ICMPv6AddressUnreachable + case ICMPErrorPortUnreachable: + icmpCode = header.ICMPv6PortUnreachable + case ICMPErrorSourcePolicy: + icmpCode = header.ICMPv6Policy + case ICMPErrorHopLimitExceeded: + icmpType = header.ICMPv6TimeExceeded + icmpCode = header.ICMPv6HopLimitExceeded + case ICMPErrorPacketTooBig: + icmpType = header.ICMPv6PacketTooBig + icmpCode = header.ICMPv6UnusedCode + } + const maxPayloadLength = header.IPv6MinimumMTU - header.IPv6MinimumSize - header.ICMPv6ErrorHeaderSize + payloadLength := min(header.IPv6MinimumSize+int(packet.PayloadLength()), len(packet), maxPayloadLength) + size := header.IPv6MinimumSize + header.ICMPv6ErrorHeaderSize + payloadLength + buffer := make([]byte, headroom+size) + response := header.IPv6(buffer[headroom:]) + response.Encode(&header.IPv6Fields{ + PayloadLength: uint16(header.ICMPv6ErrorHeaderSize + payloadLength), + TransportProtocol: header.ICMPv6ProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: source, + DstAddr: packet.SourceAddr(), + }) + icmpHdr := header.ICMPv6(response.Payload()) + icmpHdr.SetType(icmpType) + icmpHdr.SetCode(icmpCode) + if errorType == ICMPErrorPacketTooBig { + icmpHdr.SetMTU(max(mtu, header.IPv6MinimumMTU)) + } + copy(icmpHdr[header.ICMPv6ErrorHeaderSize:], packet[:payloadLength]) + icmpHdr.SetChecksum(header.ICMPv6Checksum(header.ICMPv6ChecksumParams{ + Header: icmpHdr, + Src: response.SourceAddressSlice(), + Dst: response.DestinationAddressSlice(), + })) + return buffer +} + +func IPTransportProtocol(packet []byte) (uint8, bool) { + switch header.IPVersion(packet) { + case header.IPv4Version: + ipHdr := header.IPv4(packet) + if !ipHdr.IsValid(len(packet)) { + return 0, false + } + return ipHdr.Protocol(), true + case header.IPv6Version: + ipHdr := header.IPv6(packet) + if !ipHdr.IsValid(len(packet)) { + return 0, false + } + protocol := ipHdr.NextHeader() + payload := ipHdr.Payload() + for { + var fragment, transportPresent bool + protocol, payload, fragment, transportPresent = skipIPv6ExtensionHeaders(protocol, payload) + if transportPresent { + return protocol, true + } + if !fragment || len(payload) < header.IPv6FragmentHeaderSize { + return 0, false + } + protocol = payload[0] + if binary.BigEndian.Uint16(payload[2:])>>3 != 0 { + switch header.IPv6ExtensionHeaderIdentifier(protocol) { + case header.IPv6HopByHopOptionsExtHdrIdentifier, header.IPv6RoutingExtHdrIdentifier, header.IPv6DestinationOptionsExtHdrIdentifier, header.IPv6FragmentExtHdrIdentifier: + return 0, false + } + return protocol, true + } + payload = payload[header.IPv6FragmentHeaderSize:] + } + default: + return 0, false + } +} diff --git a/internal/afd/afd_windows.go b/internal/afd/afd_windows.go new file mode 100644 index 00000000..3a012200 --- /dev/null +++ b/internal/afd/afd_windows.go @@ -0,0 +1,197 @@ +package afd + +import ( + "math" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + IOCTL_AFD_POLL = 0x00012024 + + POLL_RECEIVE = 0x0001 + POLL_RECEIVE_EXPEDITED = 0x0002 + POLL_SEND = 0x0004 + POLL_DISCONNECT = 0x0008 + POLL_ABORT = 0x0010 + POLL_LOCAL_CLOSE = 0x0020 + POLL_ACCEPT = 0x0080 + POLL_CONNECT_FAIL = 0x0100 + + SIO_BASE_HANDLE = 0x48000022 + SIO_BSP_HANDLE_POLL = 0x4800001D + + STATUS_PENDING = 0x00000103 + STATUS_CANCELLED = 0xC0000120 + STATUS_NOT_FOUND = 0xC0000225 + + FILE_OPEN = 0x00000001 + OBJ_CASE_INSENSITIVE = 0x00000040 +) + +type PollHandleInfo struct { + Handle windows.Handle + Events uint32 + Status uint32 +} + +type PollInfo struct { + Timeout int64 + NumberOfHandles uint32 + Exclusive uint32 + Handles [1]PollHandleInfo +} + +type OverlappedEntry struct { + CompletionKey uintptr + Overlapped *windows.Overlapped + Internal uintptr + NumberOfBytesTransferred uint32 +} + +type UnicodeString struct { + Length uint16 + MaximumLength uint16 + Buffer *uint16 +} + +type ObjectAttributes struct { + Length uint32 + RootDirectory windows.Handle + ObjectName *UnicodeString + Attributes uint32 + SecurityDescriptor uintptr + SecurityQualityOfService uintptr +} + +type Device struct { + handle windows.Handle +} + +func Open(iocp windows.Handle, name string) (*Device, error) { + deviceName := `\Device\Afd\` + name + deviceNameUTF16, err := windows.UTF16FromString(deviceName) + if err != nil { + return nil, err + } + unicodeString := UnicodeString{ + Length: uint16(len(deviceName) * 2), + MaximumLength: uint16(len(deviceName) * 2), + Buffer: &deviceNameUTF16[0], + } + objectAttributes := ObjectAttributes{ + Length: uint32(unsafe.Sizeof(ObjectAttributes{})), + ObjectName: &unicodeString, + Attributes: OBJ_CASE_INSENSITIVE, + } + var handle windows.Handle + var ioStatusBlock windows.IO_STATUS_BLOCK + err = NtCreateFile(&handle, windows.SYNCHRONIZE, &objectAttributes, &ioStatusBlock, nil, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, FILE_OPEN, 0, 0, 0) + if err != nil { + return nil, err + } + _, err = windows.CreateIoCompletionPort(handle, iocp, 0, 0) + if err != nil { + windows.CloseHandle(handle) + return nil, err + } + err = windows.SetFileCompletionNotificationModes(handle, windows.FILE_SKIP_SET_EVENT_ON_HANDLE) + if err != nil { + windows.CloseHandle(handle) + return nil, err + } + return &Device{handle: handle}, nil +} + +// Poll completes through the completion port with the io status block pointer as the overlapped pointer. +func (d *Device) Poll(baseSocket windows.Handle, events uint32, ioStatusBlock *windows.IO_STATUS_BLOCK, pollInfo *PollInfo) error { + pollInfo.Timeout = math.MaxInt64 + pollInfo.NumberOfHandles = 1 + pollInfo.Exclusive = 0 + pollInfo.Handles[0].Handle = baseSocket + pollInfo.Handles[0].Events = events + pollInfo.Handles[0].Status = 0 + ioStatusBlock.Status = windows.NTStatus(STATUS_PENDING) + size := uint32(unsafe.Sizeof(*pollInfo)) + status := NtDeviceIoControlFile(d.handle, 0, 0, uintptr(unsafe.Pointer(ioStatusBlock)), ioStatusBlock, IOCTL_AFD_POLL, unsafe.Pointer(pollInfo), size, unsafe.Pointer(pollInfo), size) + if status != 0 && uint32(status) != STATUS_PENDING { + return status + } + return nil +} + +func (d *Device) Cancel(ioStatusBlock *windows.IO_STATUS_BLOCK) error { + if uint32(ioStatusBlock.Status) != STATUS_PENDING { + return nil + } + var cancelIOStatusBlock windows.IO_STATUS_BLOCK + err := NtCancelIoFileEx(d.handle, ioStatusBlock, &cancelIOStatusBlock) + if err != nil { + if ntstatus, isStatus := err.(windows.NTStatus); isStatus && (uint32(ntstatus) == STATUS_CANCELLED || uint32(ntstatus) == STATUS_NOT_FOUND) { + return nil + } + return err + } + return nil +} + +func (d *Device) Close() error { + return windows.CloseHandle(d.handle) +} + +func BaseSocket(socket windows.Handle) (windows.Handle, error) { + var baseSocket windows.Handle + var bytesReturned uint32 + for { + err := windows.WSAIoctl(socket, SIO_BASE_HANDLE, nil, 0, (*byte)(unsafe.Pointer(&baseSocket)), uint32(unsafe.Sizeof(baseSocket)), &bytesReturned, nil, 0) + if err != nil { + err = windows.WSAIoctl(socket, SIO_BSP_HANDLE_POLL, nil, 0, (*byte)(unsafe.Pointer(&baseSocket)), uint32(unsafe.Sizeof(baseSocket)), &bytesReturned, nil, 0) + if err != nil { + return socket, nil + } + } + if baseSocket == socket { + return baseSocket, nil + } + socket = baseSocket + } +} + +type WaitCompletionPacket struct { + handle windows.Handle +} + +func WaitCompletionPacketSupported() bool { + return procNtCreateWaitCompletionPacket.Find() == nil && procNtAssociateWaitCompletionPacket.Find() == nil && procNtCancelWaitCompletionPacket.Find() == nil +} + +func NewWaitCompletionPacket() (*WaitCompletionPacket, error) { + var handle windows.Handle + err := NtCreateWaitCompletionPacket(&handle, windows.GENERIC_ALL, nil) + if err != nil { + return nil, err + } + return &WaitCompletionPacket{handle: handle}, nil +} + +func (p *WaitCompletionPacket) Associate(iocp windows.Handle, target windows.Handle, key uintptr) (bool, error) { + var alreadySignaled byte + err := NtAssociateWaitCompletionPacket(p.handle, iocp, target, key, 0, 0, 0, &alreadySignaled) + if err != nil { + return false, err + } + return alreadySignaled != 0, nil +} + +func (p *WaitCompletionPacket) Cancel() error { + err := NtCancelWaitCompletionPacket(p.handle, 1) + if status, isStatus := err.(windows.NTStatus); isStatus && uint32(status) == STATUS_CANCELLED { + return nil + } + return err +} + +func (p *WaitCompletionPacket) Close() error { + return windows.CloseHandle(p.handle) +} diff --git a/internal/afd/zsyscall_windows.go b/internal/afd/zsyscall_windows.go new file mode 100644 index 00000000..d6432a54 --- /dev/null +++ b/internal/afd/zsyscall_windows.go @@ -0,0 +1,81 @@ +package afd + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + + procGetQueuedCompletionStatusEx = modkernel32.NewProc("GetQueuedCompletionStatusEx") + procNtCancelIoFileEx = modntdll.NewProc("NtCancelIoFileEx") + procNtCreateFile = modntdll.NewProc("NtCreateFile") + procNtDeviceIoControlFile = modntdll.NewProc("NtDeviceIoControlFile") + procNtCreateWaitCompletionPacket = modntdll.NewProc("NtCreateWaitCompletionPacket") + procNtAssociateWaitCompletionPacket = modntdll.NewProc("NtAssociateWaitCompletionPacket") + procNtCancelWaitCompletionPacket = modntdll.NewProc("NtCancelWaitCompletionPacket") +) + +func GetQueuedCompletionStatusEx(cphandle windows.Handle, entries *OverlappedEntry, count uint32, numRemoved *uint32, timeout uint32, alertable bool) syscall.Errno { + var alertableValue uint32 + if alertable { + alertableValue = 1 + } + r1, _, e1 := syscall.SyscallN(procGetQueuedCompletionStatusEx.Addr(), uintptr(cphandle), uintptr(unsafe.Pointer(entries)), uintptr(count), uintptr(unsafe.Pointer(numRemoved)), uintptr(timeout), uintptr(alertableValue)) + if r1 == 0 { + if e1 != 0 { + return e1 + } + return syscall.EINVAL + } + return 0 +} + +func NtCancelIoFileEx(handle windows.Handle, ioRequestToCancel *windows.IO_STATUS_BLOCK, ioStatusBlock *windows.IO_STATUS_BLOCK) error { + r0, _, _ := syscall.SyscallN(procNtCancelIoFileEx.Addr(), uintptr(handle), uintptr(unsafe.Pointer(ioRequestToCancel)), uintptr(unsafe.Pointer(ioStatusBlock))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtCreateFile(handle *windows.Handle, access uint32, objectAttributes *ObjectAttributes, ioStatusBlock *windows.IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eaBuffer uintptr, eaLength uint32) error { + r0, _, _ := syscall.SyscallN(procNtCreateFile.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(objectAttributes)), uintptr(unsafe.Pointer(ioStatusBlock)), uintptr(unsafe.Pointer(allocationSize)), uintptr(attributes), uintptr(share), uintptr(disposition), uintptr(options), uintptr(eaBuffer), uintptr(eaLength)) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtDeviceIoControlFile(handle windows.Handle, event windows.Handle, apcRoutine uintptr, apcContext uintptr, ioStatusBlock *windows.IO_STATUS_BLOCK, ioControlCode uint32, inputBuffer unsafe.Pointer, inputBufferLength uint32, outputBuffer unsafe.Pointer, outputBufferLength uint32) windows.NTStatus { + r0, _, _ := syscall.SyscallN(procNtDeviceIoControlFile.Addr(), uintptr(handle), uintptr(event), uintptr(apcRoutine), uintptr(apcContext), uintptr(unsafe.Pointer(ioStatusBlock)), uintptr(ioControlCode), uintptr(inputBuffer), uintptr(inputBufferLength), uintptr(outputBuffer), uintptr(outputBufferLength)) + return windows.NTStatus(r0) +} + +func NtCreateWaitCompletionPacket(handle *windows.Handle, access uint32, objectAttributes *ObjectAttributes) error { + r0, _, _ := syscall.SyscallN(procNtCreateWaitCompletionPacket.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(objectAttributes))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtAssociateWaitCompletionPacket(handle windows.Handle, iocp windows.Handle, target windows.Handle, keyContext uintptr, apcContext uintptr, ioStatus uint32, ioStatusInformation uintptr, alreadySignaled *byte) error { + r0, _, _ := syscall.SyscallN(procNtAssociateWaitCompletionPacket.Addr(), uintptr(handle), uintptr(iocp), uintptr(target), keyContext, apcContext, uintptr(ioStatus), ioStatusInformation, uintptr(unsafe.Pointer(alreadySignaled))) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} + +func NtCancelWaitCompletionPacket(handle windows.Handle, removeSignaledPacket byte) error { + r0, _, _ := syscall.SyscallN(procNtCancelWaitCompletionPacket.Addr(), uintptr(handle), uintptr(removeSignaledPacket)) + if r0 != 0 { + return windows.NTStatus(r0) + } + return nil +} diff --git a/internal/checksum_test/sum_bench_test.go b/internal/checksum_test/sum_bench_test.go index fb4aaa4c..9b6ff266 100644 --- a/internal/checksum_test/sum_bench_test.go +++ b/internal/checksum_test/sum_bench_test.go @@ -2,12 +2,37 @@ package checksum_test import ( "crypto/rand" + "strconv" "testing" "github.com/sagernet/sing-tun/gtcpip/checksum" "github.com/sagernet/sing-tun/internal/tschecksum" ) +var benchSizes = []int{64, 128, 512, 1460, 8500, 65535} + +func benchChecksum(b *testing.B, size int, fn func([]byte, uint16) uint16) { + data := make([]byte, size) + rand.Read(data) + b.SetBytes(int64(size)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + fn(data, 0) + } +} + +func BenchmarkTsChecksumSizes(b *testing.B) { + for _, size := range benchSizes { + b.Run(strconv.Itoa(size), func(b *testing.B) { benchChecksum(b, size, tschecksum.Checksum) }) + } +} + +func BenchmarkGChecksumSizes(b *testing.B) { + for _, size := range benchSizes { + b.Run(strconv.Itoa(size), func(b *testing.B) { benchChecksum(b, size, checksum.ChecksumDefault) }) + } +} + func BenchmarkTsChecksum(b *testing.B) { packet := make([][]byte, 1000) for i := range 1000 { diff --git a/internal/checksum_test/sum_test.go b/internal/checksum_test/sum_test.go new file mode 100644 index 00000000..f6fb8d7e --- /dev/null +++ b/internal/checksum_test/sum_test.go @@ -0,0 +1,40 @@ +package checksum_test + +import ( + "crypto/rand" + mathrand "math/rand" + "testing" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/internal/tschecksum" +) + +func TestChecksumDifferential(t *testing.T) { + lengths := make([]int, 0, 2054) + for length := 0; length <= 2049; length++ { + lengths = append(lengths, length) + } + lengths = append(lengths, 4096, 8500, 9000, 65535) + backing := make([]byte, 65535+16) + for _, pattern := range []string{"random", "ones"} { + if pattern == "random" { + rand.Read(backing) + } else { + for i := range backing { + backing[i] = 0xff + } + } + for _, length := range lengths { + for offset := range 16 { + data := backing[offset : offset+length] + initial := uint16(mathrand.Uint32()) + expected := checksum.ChecksumDefault(data, initial) + actual := tschecksum.Checksum(data, initial) + if actual != expected { + t.Fatalf("mismatch: pattern=%s length=%d offset=%d initial=%#04x got %#04x want %#04x", + pattern, length, offset, initial, actual, expected) + } + } + } + } +} diff --git a/internal/fdbased_darwin/endpoint.go b/internal/fdbased_darwin/endpoint.go index b3292a2e..a99b5871 100644 --- a/internal/fdbased_darwin/endpoint.go +++ b/internal/fdbased_darwin/endpoint.go @@ -261,7 +261,7 @@ func New(opts *Options) (stack.LinkEndpoint, error) { } var batchSize int if opts.MultiPendingPackets { - batchSize = int((512*1024)/(opts.MTU)) + 1 + batchSize = int((512*1024)/opts.MTU) + 1 } else { batchSize = 1 } diff --git a/internal/fdbased_darwin/packet_dispatchers.go b/internal/fdbased_darwin/packet_dispatchers.go index afe362f6..4bad9b27 100644 --- a/internal/fdbased_darwin/packet_dispatchers.go +++ b/internal/fdbased_darwin/packet_dispatchers.go @@ -121,7 +121,7 @@ func newRecvMMsgDispatcher(fd int, e *endpoint, opts *Options) (linkDispatcher, } var batchSize int if opts.MultiPendingPackets { - batchSize = int((512*1024)/(opts.MTU)) + 1 + batchSize = int((512*1024)/opts.MTU) + 1 } else { batchSize = 1 } diff --git a/internal/tschecksum/checksum_amd64.go b/internal/tschecksum/checksum_amd64.go index 85b925a0..954512f9 100644 --- a/internal/tschecksum/checksum_amd64.go +++ b/internal/tschecksum/checksum_amd64.go @@ -2,22 +2,20 @@ package tschecksum import "golang.org/x/sys/cpu" -var checksum = checksumAMD64 +var ( + useAVX2 = cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 + useSSE2 = cpu.X86.HasSSE2 +) // Checksum computes an IP checksum starting with the provided initial value. // The length of data should be at least 128 bytes for best performance. Smaller // buffers will still compute a correct result. func Checksum(data []byte, initial uint16) uint16 { - return checksum(data, initial) -} - -func init() { - if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { - checksum = checksumAVX2 - return + if useAVX2 { + return checksumAVX2(data, initial) } - if cpu.X86.HasSSE2 { - checksum = checksumSSE2 - return + if useSSE2 { + return checksumSSE2(data, initial) } + return checksumAMD64(data, initial) } diff --git a/internal/tschecksum/checksum_arm64.go b/internal/tschecksum/checksum_arm64.go new file mode 100644 index 00000000..f1964b80 --- /dev/null +++ b/internal/tschecksum/checksum_arm64.go @@ -0,0 +1,8 @@ +package tschecksum + +//go:noescape +func checksumNEON(b []byte, initial uint16) uint16 + +func Checksum(data []byte, initial uint16) uint16 { + return checksumNEON(data, initial) +} diff --git a/internal/tschecksum/checksum_arm64.s b/internal/tschecksum/checksum_arm64.s new file mode 100644 index 00000000..35c64cf2 --- /dev/null +++ b/internal/tschecksum/checksum_arm64.s @@ -0,0 +1,133 @@ +#include "textflag.h" + +// func checksumNEON(b []byte, initial uint16) uint16 +TEXT ·checksumNEON(SB), NOSPLIT|NOFRAME, $0-34 + MOVD b_base+0(FP), R0 + MOVD b_len+8(FP), R1 + MOVHU initial+24(FP), R2 + REV16W R2, R2 + TBZ $0, R1, neonEvenLength + SUB $1, R1 + MOVBU (R0)(R1), R3 + ADD R3, R2 + +neonEvenLength: + CMP $128, R1 + BGE neonChunk + CMP $64, R1 + BLT neonTail32 + LDP (R0), (R4, R5) + LDP 16(R0), (R6, R7) + LDP 32(R0), (R8, R9) + LDP 48(R0), (R10, R11) + ADDS R4, R2 + ADCS R5, R2 + ADCS R6, R2 + ADCS R7, R2 + ADCS R8, R2 + ADCS R9, R2 + ADCS R10, R2 + ADCS R11, R2 + ADC $0, R2 + ADD $64, R0 + SUB $64, R1 + B neonTail32 + +neonChunk: + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + + AND $-64, R1, R3 + MOVD $1048576, R4 + CMP R4, R3 + CSEL LT, R3, R4, R3 + SUB R3, R1 + ADD R3, R0, R5 + +neonInner: + VLD1.P 64(R0), [V16.B16, V17.B16, V18.B16, V19.B16] + VUADDW V16.H4, V0.S4, V0.S4 + VUADDW2 V16.H8, V1.S4, V1.S4 + VUADDW V17.H4, V2.S4, V2.S4 + VUADDW2 V17.H8, V3.S4, V3.S4 + VUADDW V18.H4, V4.S4, V4.S4 + VUADDW2 V18.H8, V5.S4, V5.S4 + VUADDW V19.H4, V6.S4, V6.S4 + VUADDW2 V19.H8, V7.S4, V7.S4 + CMP R5, R0 + BNE neonInner + + VADD V1.S4, V0.S4, V0.S4 + VADD V3.S4, V2.S4, V2.S4 + VADD V5.S4, V4.S4, V4.S4 + VADD V7.S4, V6.S4, V6.S4 + VADD V2.S4, V0.S4, V0.S4 + VADD V6.S4, V4.S4, V4.S4 + VUADDLV V0.S4, V16 + VUADDLV V4.S4, V17 + VMOV V16.D[0], R6 + VMOV V17.D[0], R7 + ADDS R6, R2 + ADCS R7, R2 + ADC $0, R2 + + CMP $64, R1 + BGE neonChunk + +neonTail32: + TBZ $5, R1, neonTail16 + LDP (R0), (R4, R5) + LDP 16(R0), (R6, R7) + ADDS R4, R2 + ADCS R5, R2 + ADCS R6, R2 + ADCS R7, R2 + ADC $0, R2 + ADD $32, R0 + +neonTail16: + TBZ $4, R1, neonTail8 + LDP (R0), (R4, R5) + ADDS R4, R2 + ADCS R5, R2 + ADC $0, R2 + ADD $16, R0 + +neonTail8: + TBZ $3, R1, neonTail4 + MOVD (R0), R4 + ADDS R4, R2 + ADC $0, R2 + ADD $8, R0 + +neonTail4: + TBZ $2, R1, neonTail2 + MOVWU (R0), R4 + ADDS R4, R2 + ADC $0, R2 + ADD $4, R0 + +neonTail2: + TBZ $1, R1, neonFold + MOVHU (R0), R4 + ADDS R4, R2 + ADC $0, R2 + +neonFold: + AND $0xffffffff, R2, R3 + ADD R2>>32, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + AND $0xffff, R2, R3 + ADD R2>>16, R3, R2 + REV16W R2, R2 + MOVH R2, ret+32(FP) + RET diff --git a/internal/tschecksum/checksum_generic.go b/internal/tschecksum/checksum_generic.go index 2d6c1348..81e7cdbb 100644 --- a/internal/tschecksum/checksum_generic.go +++ b/internal/tschecksum/checksum_generic.go @@ -1,7 +1,7 @@ // This file contains IP checksum algorithms that are not specific to any // architecture and don't use hardware acceleration. -//go:build !amd64 +//go:build !amd64 && !arm64 package tschecksum diff --git a/internal/winsys/constants.go b/internal/winsys/constants.go index 1173ea9a..03e6e65e 100644 --- a/internal/winsys/constants.go +++ b/internal/winsys/constants.go @@ -103,6 +103,25 @@ var FWPM_CONDITION_IP_REMOTE_PORT = windows.GUID{ Data4: [8]byte{0x91, 0xb4, 0x68, 0xf6, 0x74, 0xee, 0x67, 0x4b}, } +var FWPM_CONDITION_IP_LOCAL_PORT = windows.GUID{ + Data1: 0x0c1ba1af, + Data2: 0x5765, + Data3: 0x453f, + Data4: [8]byte{0xaf, 0x22, 0xa8, 0xf7, 0x91, 0xac, 0x77, 0x5b}, +} + +var FWPM_CONDITION_IP_REMOTE_ADDRESS = windows.GUID{ + Data1: 0xb235ae9a, + Data2: 0x1d64, + Data3: 0x49b8, + Data4: [8]byte{0xa4, 0x4c, 0x5f, 0xf3, 0xd9, 0x09, 0x50, 0x45}, +} + +var ( + FWPM_CONDITION_ICMP_TYPE = FWPM_CONDITION_IP_LOCAL_PORT + FWPM_CONDITION_ICMP_CODE = FWPM_CONDITION_IP_REMOTE_PORT +) + var FWPM_LAYER_ALE_AUTH_CONNECT_V4 = windows.GUID{ Data1: 0xc38d57d1, Data2: 0x05a7, @@ -132,7 +151,8 @@ var FWPM_CONDITION_ALE_APP_ID = windows.GUID{ } const ( - IPPROTO_UDP uint32 = 17 + IPPROTO_UDP uint32 = 17 + IPPROTO_ICMPV6 uint32 = 58 ) const ( diff --git a/internal/winsys/types_windows.go b/internal/winsys/types_windows.go index af6f3c04..01ca6f3e 100644 --- a/internal/winsys/types_windows.go +++ b/internal/winsys/types_windows.go @@ -68,6 +68,10 @@ type FWP_VALUE0 struct { type FWP_CONDITION_VALUE0 FWP_VALUE0 +type FWP_BYTE_ARRAY16 struct { + ByteArray16 [16]uint8 +} + type FWPM_FILTER_CONDITION0 struct { FieldKey windows.GUID // Windows type: GUID MatchType uint32 diff --git a/internal/wintun/memmod/memmod_windows.go b/internal/wintun/memmod/memmod_windows.go index 0b345ba6..37e25560 100644 --- a/internal/wintun/memmod/memmod_windows.go +++ b/internal/wintun/memmod/memmod_windows.go @@ -67,7 +67,7 @@ func (module *Module) copySections(address, size uintptr, oldHeaders *IMAGE_NT_H // Always use position from file to support alignments smaller than page size (allocation above will align to page size). dest := module.codeBase + uintptr(sections[i].VirtualAddress) // NOTE: On 64bit systems we truncate to 32bit here but expand again later when "PhysicalAddress" is used. - sections[i].SetPhysicalAddress((uint32)(dest & 0xffffffff)) + sections[i].SetPhysicalAddress(uint32(dest & 0xffffffff)) dst := unsafe.Slice((*byte)(a2p(dest)), sectionSize) for j := range dst { dst[j] = 0 @@ -94,7 +94,7 @@ func (module *Module) copySections(address, size uintptr, oldHeaders *IMAGE_NT_H address+uintptr(sections[i].PointerToRawData), uintptr(sections[i].SizeOfRawData)) // NOTE: On 64bit systems we truncate to 32bit here but expand again later when "PhysicalAddress" is used. - sections[i].SetPhysicalAddress((uint32)(dest & 0xffffffff)) + sections[i].SetPhysicalAddress(uint32(dest & 0xffffffff)) } return nil diff --git a/internal/wintun/memmod/syscall_windows.go b/internal/wintun/memmod/syscall_windows.go index b79be69e..e1d25d2d 100644 --- a/internal/wintun/memmod/syscall_windows.go +++ b/internal/wintun/memmod/syscall_windows.go @@ -118,7 +118,7 @@ type IMAGE_NT_HEADERS struct { func (ntheader *IMAGE_NT_HEADERS) Sections() []IMAGE_SECTION_HEADER { return (*[0xffff]IMAGE_SECTION_HEADER)(unsafe.Pointer( - (uintptr)(unsafe.Pointer(ntheader)) + + uintptr(unsafe.Pointer(ntheader)) + unsafe.Offsetof(ntheader.OptionalHeader) + uintptr(ntheader.FileHeader.SizeOfOptionalHeader)))[:ntheader.FileHeader.NumberOfSections] } diff --git a/internal/wintun/session_windows.go b/internal/wintun/session_windows.go index 306f3b12..99656491 100644 --- a/internal/wintun/session_windows.go +++ b/internal/wintun/session_windows.go @@ -59,11 +59,11 @@ func (session Session) ReadWaitEvent() (handle windows.Handle) { return } -func (session Session) ReceivePacket() (packet []byte, err error) { +func (session Session) ReceivePacket() (packet []byte, errno syscall.Errno) { var packetSize uint32 r0, _, e1 := syscall.SyscallN(procWintunReceivePacket.Addr(), session.handle, uintptr(unsafe.Pointer(&packetSize))) if r0 == 0 { - err = e1 + errno = e1 return } packet = unsafe.Slice((*byte)(unsafe.Pointer(r0)), packetSize) @@ -74,10 +74,10 @@ func (session Session) ReleaseReceivePacket(packet []byte) { syscall.SyscallN(procWintunReleaseReceivePacket.Addr(), session.handle, uintptr(unsafe.Pointer(&packet[0]))) } -func (session Session) AllocateSendPacket(packetSize int) (packet []byte, err error) { +func (session Session) AllocateSendPacket(packetSize int) (packet []byte, errno syscall.Errno) { r0, _, e1 := syscall.SyscallN(procWintunAllocateSendPacket.Addr(), session.handle, uintptr(packetSize)) if r0 == 0 { - err = e1 + errno = e1 return } packet = unsafe.Slice((*byte)(unsafe.Pointer(r0)), packetSize) diff --git a/monitor_linux.go b/monitor_linux.go index 5a13a7f7..b57a1f29 100644 --- a/monitor_linux.go +++ b/monitor_linux.go @@ -67,9 +67,13 @@ func (m *networkUpdateMonitor) Start() error { if err != nil { return E.Cause(err, "create netlink socket") } + groups := uint32(netlinkGroups) + if runtime.GOOS == "android" { + groups |= unix.RTMGRP_IPV4_RULE | 1<<(unix.RTNLGRP_IPV6_RULE-1) + } err = unix.Bind(netlinkSocket, &unix.SockaddrNetlink{ Family: unix.AF_NETLINK, - Groups: netlinkGroups, + Groups: groups, }) if err != nil { unix.Close(netlinkSocket) diff --git a/nfqueue_linux.go b/nfqueue_linux.go index 73151c6b..7d789128 100644 --- a/nfqueue_linux.go +++ b/nfqueue_linux.go @@ -17,45 +17,48 @@ import ( ) type nfqueueHandler struct { - ctx context.Context - cancel context.CancelFunc - handler Handler - logger logger.Logger - nfq *nfqueue.Nfqueue - queue uint16 - inputMark uint32 - outputMark uint32 - resetMark uint32 - closed atomic.Bool + ctx context.Context + cancel context.CancelFunc + handler AutoRedirectHandler + logger logger.Logger + nfq *nfqueue.Nfqueue + queue uint16 + inputMark uint32 + outputMark uint32 + resetMark uint32 + tproxyMark uint32 + markMask uint32 + repeatOnAccept bool + closed atomic.Bool } -type nfqueueOptions struct { - Context context.Context - Handler Handler - Logger logger.Logger - Queue uint16 - InputMark uint32 - OutputMark uint32 - ResetMark uint32 -} - -func newNFQueueHandler(options nfqueueOptions) (*nfqueueHandler, error) { - ctx, cancel := context.WithCancel(options.Context) - return &nfqueueHandler{ - ctx: ctx, - cancel: cancel, - handler: options.Handler, - logger: options.Logger, - queue: options.Queue, - inputMark: options.InputMark, - outputMark: options.OutputMark, - resetMark: options.ResetMark, - }, nil +func (h *nfqueueHandler) acceptVerdict(packet preMatchPacket) (verdict int, mark uint32) { + if !h.repeatOnAccept { + return nfqueue.NfAccept, 0 + } + if packet.protocol != uint8(unix.IPPROTO_TCP) { + return nfqueue.NfRepeat, h.inputMark + } + if packet.destination.Addr().Is6() { + if h.tproxyMark != 0 { + return nfqueue.NfRepeat, h.tproxyMark + } + return nfqueue.NfRepeat, h.inputMark + } + return nfqueue.NfAccept, 0 } -func (h *nfqueueHandler) setVerdict(packetID uint32, verdict int, mark uint32) { +func (h *nfqueueHandler) setVerdict(attr nfqueue.Attribute, verdict int, mark uint32) { + packetID := *attr.PacketID var err error if mark != 0 { + if h.markMask != 0 { + var originalMark uint32 + if attr.Mark != nil { + originalMark = *attr.Mark + } + mark = originalMark&^h.markMask | mark + } err = h.nfq.SetVerdictWithOption(packetID, verdict, nfqueue.WithMark(mark)) } else { err = h.nfq.SetVerdict(packetID, verdict) @@ -65,14 +68,17 @@ func (h *nfqueueHandler) setVerdict(packetID uint32, verdict int, mark uint32) { } } +// Without a bypass flag every packet a remaining NFQUEUE rule sends to an unbound queue is +// dropped, so the queue is unbound in Close rather than with the caller's context. func (h *nfqueueHandler) Start() error { + h.ctx, h.cancel = context.WithCancel(context.Background()) config := nfqueue.Config{ NfQueue: h.queue, MaxPacketLen: 0xFFFF, MaxQueueLen: 4096, Copymode: nfqueue.NfQnlCopyPacket, AfFamily: unix.AF_UNSPEC, - Flags: nfqueue.NfQaCfgFlagFailOpen | nfqueue.NfQaCfgFlagGSO, + Flags: nfqueue.NfQaCfgFlagGSO, } nfq, err := nfqueue.Open(&config) @@ -259,12 +265,11 @@ func (h *nfqueueHandler) handlePacket(attr nfqueue.Attribute) int { return 0 } - packetID := *attr.PacketID payload := *attr.Payload packet, loaded := parsePreMatchPacket(payload) if !loaded { - h.setVerdict(packetID, nfqueue.NfAccept, 0) + h.setVerdict(attr, nfqueue.NfAccept, 0) return 0 } @@ -275,27 +280,25 @@ func (h *nfqueueHandler) handlePacket(attr nfqueue.Attribute) int { packet.firstPacket, ) - // nf_reinject: NF_REPEAT re-runs the queueing chain from its first rule, - // while NF_ACCEPT continues at the next hook, skipping the remaining - // rules of the chain. switch verdict.Action { case ActionBypass: - h.setVerdict(packetID, nfqueue.NfRepeat, h.outputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.outputMark) case ActionReject: if packet.protocol == uint8(unix.IPPROTO_TCP) { - h.setVerdict(packetID, nfqueue.NfRepeat, h.resetMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.resetMark) } else { - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) } case ActionDrop: - h.setVerdict(packetID, nfqueue.NfDrop, 0) + h.setVerdict(attr, nfqueue.NfDrop, 0) case ActionFlow, ActionHijackDNS: - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) default: if packet.protocol == uint8(unix.IPPROTO_TCP) { - h.setVerdict(packetID, nfqueue.NfAccept, 0) + acceptVerdict, acceptMark := h.acceptVerdict(packet) + h.setVerdict(attr, acceptVerdict, acceptMark) } else { - h.setVerdict(packetID, nfqueue.NfRepeat, h.inputMark) + h.setVerdict(attr, nfqueue.NfRepeat, h.inputMark) } } diff --git a/ping/port.go b/ping/port.go index e0cc2a1d..b0b8afc7 100644 --- a/ping/port.go +++ b/ping/port.go @@ -64,7 +64,7 @@ func (p *Port) AttachReturn(returnPath tun.Return) error { if slices.Contains(p.returnPaths, returnPath) { return nil } - p.returnPaths = append(p.returnPaths[:len(p.returnPaths):len(p.returnPaths)], returnPath) + p.returnPaths = append(slices.Clip(p.returnPaths), returnPath) return nil } diff --git a/redirect.go b/redirect.go index dcf3e720..17ef6182 100644 --- a/redirect.go +++ b/redirect.go @@ -2,36 +2,48 @@ package tun import ( "context" + "net/netip" "github.com/sagernet/sing/common/control" "github.com/sagernet/sing/common/logger" - - "go4.org/netipx" + N "github.com/sagernet/sing/common/network" ) const ( DefaultAutoRedirectInputMark = 0x2023 DefaultAutoRedirectOutputMark = 0x2024 DefaultAutoRedirectResetMark = 0x2025 + DefaultAutoRedirectTProxyMark = 0x2026 DefaultAutoRedirectNFQueue = 100 + + DefaultAutoRedirectInputMarkAndroid = 0x400000 + DefaultAutoRedirectOutputMarkAndroid = 0x200000 + DefaultAutoRedirectResetMarkAndroid = 0x600000 + DefaultAutoRedirectTProxyMarkAndroid = 0x800000 ) type AutoRedirect interface { Start() error Close() error - UpdateRouteAddressSet() + UpdateRouteAddressSet() error +} + +type AutoRedirectHandler interface { + JudgeFlow(network uint8, source netip.AddrPort, destination netip.AddrPort, firstPacket []byte) FlowVerdict + N.TCPConnectionHandlerEx } type AutoRedirectOptions struct { - TunOptions *Options - Context context.Context - Handler Handler - Logger logger.Logger - NetworkMonitor NetworkUpdateMonitor - InterfaceFinder control.InterfaceFinder - TableName string - DisableNFTables bool - CustomRedirectPort func() int - RouteAddressSet *[]*netipx.IPSet - RouteExcludeAddressSet *[]*netipx.IPSet + TunOptions *Options + Context context.Context + Handler AutoRedirectHandler + Logger logger.Logger + NetworkMonitor NetworkUpdateMonitor + InterfaceFinder control.InterfaceFinder + TableName string + DisableNFTables bool + CustomRedirectPort func() int + CustomRedirectListenerFD func() (int, error) + RouteAddressSet func() (include []netip.Prefix, exclude []netip.Prefix, err error) + AndroidVPNService bool } diff --git a/redirect_ingress_discard_android.go b/redirect_ingress_discard_android.go new file mode 100644 index 00000000..291a6771 --- /dev/null +++ b/redirect_ingress_discard_android.go @@ -0,0 +1,177 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "os" + "runtime" + "unsafe" + + "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +// ConnectivityService.updateIngressToVpnAddressFiltering keys this map by the +// VPN address (IPv4-mapped) with the tun ifindex in both value slots, and the +// cgroup skb ingress program in netd.c drops packets for app uids to that +// address from any other interface, which includes the de-NATed replies of +// bypassed flows. ConnectivityService updates entries from LinkProperties diffs. +const androidIngressDiscardMapPath = "/sys/fs/bpf/netd_shared/map_netd_ingress_discard_map" + +type bpfObjectAttribute struct { + pathname uint64 + fileDescriptor uint32 + fileFlags uint32 +} + +type bpfMapAttribute struct { + mapDescriptor uint32 + _ uint32 + key uint64 + value uint64 + flags uint64 +} + +type bpfObjectInfoAttribute struct { + fileDescriptor uint32 + length uint32 + info uint64 +} + +func bpfObjectGet(path string) (int, error) { + pathBytes, err := unix.BytePtrFromString(path) + if err != nil { + return 0, err + } + var pinner runtime.Pinner + pinner.Pin(pathBytes) + defer pinner.Unpin() + attribute := bpfObjectAttribute{pathname: uint64(uintptr(unsafe.Pointer(pathBytes)))} + descriptor, _, errno := unix.Syscall(unix.SYS_BPF, unix.BPF_OBJ_GET, uintptr(unsafe.Pointer(&attribute)), unsafe.Sizeof(attribute)) + if errno != 0 { + return 0, errno + } + var info struct { + mapType uint32 + id uint32 + keySize uint32 + valueSize uint32 + } + pinner.Pin(&info) + infoAttribute := bpfObjectInfoAttribute{ + fileDescriptor: uint32(descriptor), + length: uint32(unsafe.Sizeof(info)), + info: uint64(uintptr(unsafe.Pointer(&info))), + } + _, _, errno = unix.Syscall(unix.SYS_BPF, unix.BPF_OBJ_GET_INFO_BY_FD, uintptr(unsafe.Pointer(&infoAttribute)), unsafe.Sizeof(infoAttribute)) + if errno != 0 { + unix.Close(int(descriptor)) + return 0, errno + } + if info.mapType != unix.BPF_MAP_TYPE_HASH || info.keySize != 16 || info.valueSize != 8 { + unix.Close(int(descriptor)) + return 0, E.New("unexpected ingress discard map layout: type ", info.mapType, ", key ", info.keySize, ", value ", info.valueSize) + } + return int(descriptor), nil +} + +func bpfMapCall(command uintptr, mapDescriptor int, key []byte, value []byte, flags uint64) error { + var pinner runtime.Pinner + pinner.Pin(&key[0]) + defer pinner.Unpin() + attribute := bpfMapAttribute{ + mapDescriptor: uint32(mapDescriptor), + key: uint64(uintptr(unsafe.Pointer(&key[0]))), + flags: flags, + } + if value != nil { + pinner.Pin(&value[0]) + attribute.value = uint64(uintptr(unsafe.Pointer(&value[0]))) + } + _, _, errno := unix.Syscall(unix.SYS_BPF, command, uintptr(unsafe.Pointer(&attribute)), unsafe.Sizeof(attribute)) + if errno != 0 { + return errno + } + return nil +} + +func (r *autoRedirect) removeAndroidIngressDiscardRulesLocked() error { + if !r.enableIPv4 { + return nil + } + _, err := os.Stat(androidIngressDiscardMapPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return E.Cause(err, "stat ingress discard map") + } + mapDescriptor, err := bpfObjectGet(androidIngressDiscardMapPath) + if err != nil { + if err == unix.ENOENT { + return nil + } + return E.Cause(err, "open ingress discard map") + } + defer unix.Close(mapDescriptor) + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return E.Cause(err, "find tun interface") + } + for _, prefix := range r.tunOptions.Inet4Address { + address := prefix.Addr() + key := address.As16() + var value [8]byte + err = bpfMapCall(unix.BPF_MAP_LOOKUP_ELEM, mapDescriptor, key[:], value[:], 0) + if err == unix.ENOENT { + continue + } + if err != nil { + return E.Cause(err, "read ingress discard rule for ", address) + } + if int(binary.NativeEndian.Uint32(value[:4])) != tunLink.Attrs().Index || + int(binary.NativeEndian.Uint32(value[4:])) != tunLink.Attrs().Index { + continue + } + err = bpfMapCall(unix.BPF_MAP_DELETE_ELEM, mapDescriptor, key[:], nil, 0) + if err != nil && err != unix.ENOENT { + return E.Cause(err, "remove ingress discard rule for ", address) + } + if r.androidIngressDiscardValues == nil { + r.androidIngressDiscardValues = make(map[netip.Addr][8]byte) + } + r.androidIngressDiscardValues[address] = value + r.logger.Debug("removed ingress discard rule for ", address) + } + return nil +} + +func (r *autoRedirect) restoreAndroidIngressDiscardRulesLocked() { + values := r.androidIngressDiscardValues + r.androidIngressDiscardValues = nil + if len(values) == 0 { + return + } + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return + } + mapDescriptor, err := bpfObjectGet(androidIngressDiscardMapPath) + if err != nil { + r.logger.Error("restore ingress discard rules: open map: ", err) + return + } + defer unix.Close(mapDescriptor) + for address, value := range values { + if int(binary.NativeEndian.Uint32(value[:4])) != tunLink.Attrs().Index { + continue + } + key := address.As16() + err = bpfMapCall(unix.BPF_MAP_UPDATE_ELEM, mapDescriptor, key[:], value[:], unix.BPF_NOEXIST) + if err != nil && err != unix.EEXIST { + r.logger.Error("restore ingress discard rule for ", address, ": ", err) + } + } +} diff --git a/redirect_iptables.go b/redirect_iptables.go index 1b7948aa..05b7a43d 100644 --- a/redirect_iptables.go +++ b/redirect_iptables.go @@ -3,61 +3,704 @@ package tun import ( + "net" + "net/netip" + "os" "os/exec" + "runtime" + "slices" + "strconv" "strings" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" F "github.com/sagernet/sing/common/format" + "github.com/sagernet/sing/common/ranges" ) -func (r *autoRedirect) setupIPTables() error { - if r.enableIPv4 { - err := r.setupIPTablesForFamily(r.iptablesPath) - if err != nil { - return E.Cause(err, "setup iptables") +const ( + iptablesTableNAT = "nat" + iptablesTableMangle = "mangle" + iptablesTableFilter = "filter" +) + +type iptablesHook int + +const ( + iptablesHookOutput iptablesHook = iota + iptablesHookPrerouting +) + +type iptablesChainKind int + +const ( + iptablesKindNAT iptablesChainKind = iota + iptablesKindRoute + iptablesKindFilter +) + +type iptablesFamily struct { + path string + isIPv6 bool + tproxy bool + dnsServer netip.Addr + chainCounter int +} + +type iptablesMatch struct { + positive []string + negative []string +} + +type iptablesLocalChain struct { + family *iptablesFamily + table string + name string + next string + hook iptablesHook + kind iptablesChainKind + dnsHijack bool +} + +type iptablesInsert struct { + table string + base string + args []string +} + +type iptablesBuilder struct { + redirect *autoRedirect + family *iptablesFamily + table string + name string + chain string + err error +} + +func iptablesMark(value uint32, mask uint32) string { + return iptablesHex(value) + "/" + iptablesHex(mask) +} + +func iptablesHex(value uint32) string { + return "0x" + strconv.FormatUint(uint64(value), 16) +} + +func iptablesUIDRange(uidRange ranges.Range[uint32]) string { + if uidRange.Start == uidRange.End { + return F.ToString(uidRange.Start) + } + return F.ToString(uidRange.Start, "-", uidRange.End) +} + +func (r *autoRedirect) iptablesChain(family *iptablesFamily, table string, name string) *iptablesBuilder { + builder := &iptablesBuilder{ + redirect: r, + family: family, + table: table, + name: name, + chain: name, + } + builder.err = iptablesRun(family.path, "-t", table, "-N", name) + return builder +} + +func (b *iptablesBuilder) add(args ...string) { + if b.err != nil { + return + } + b.err = iptablesRun(b.family.path, slices.Concat([]string{"-t", b.table, "-A", b.chain}, args)...) +} + +func (b *iptablesBuilder) newChain(prefix string) string { + b.family.chainCounter++ + name := F.ToString(b.redirect.tableName, "-", prefix, "-", b.family.chainCounter) + if b.err == nil { + b.err = iptablesRun(b.family.path, "-t", b.table, "-N", name) + } + return name +} + +func (b *iptablesBuilder) includeOnly(matches []iptablesMatch) { + if len(matches) == 0 { + return + } + if len(matches) == 1 && len(matches[0].negative) > 0 { + b.add(slices.Concat(matches[0].negative, []string{"-j", "RETURN"})...) + return + } + next := b.newChain("include") + for _, match := range matches { + b.add(slices.Concat(match.positive, []string{"-j", next})...) + } + b.chain = next +} + +func (b *iptablesBuilder) branch(prefix string) *iptablesBuilder { + name := b.newChain(prefix) + b.add("-j", name) + return &iptablesBuilder{ + redirect: b.redirect, + family: b.family, + table: b.table, + name: name, + chain: name, + err: b.err, + } +} + +func (b *iptablesBuilder) absorb(child *iptablesBuilder) { + if b.err == nil { + b.err = child.err + } +} + +func (b *iptablesBuilder) localGate(hook iptablesHook, kind iptablesChainKind, dnsHijack bool) { + next := b.newChain("cont") + name := b.newChain("local") + b.add("-j", name) + chain := iptablesLocalChain{ + family: b.family, + table: b.table, + name: name, + next: next, + hook: hook, + kind: kind, + dnsHijack: dnsHijack, + } + if b.err == nil { + b.err = b.redirect.iptablesFillLocalChain(chain, b.redirect.localAddresses) + } + b.redirect.iptablesLocalChains = append(b.redirect.iptablesLocalChains, chain) + b.chain = next +} + +func (b *iptablesBuilder) addVPNServiceGate() { + if b.redirect.androidVPNService { + b.add("!", "-o", b.redirect.tunOptions.Name, "-j", "RETURN") + } +} + +func (b *iptablesBuilder) addRedirect() { + loopbackAddresses := b.redirect.tunOptions.Inet4LoopbackAddress + if b.family.isIPv6 { + loopbackAddresses = b.redirect.tunOptions.Inet6LoopbackAddress + } + redirect := []string{"-p", "tcp", "-j", "REDIRECT", "--to-ports", strconv.Itoa(int(b.redirect.redirectPort()))} + if len(loopbackAddresses) == 0 { + b.add(redirect...) + return + } + name := b.newChain("redirect") + b.add("-j", name) + parent := b.chain + b.chain = name + for _, address := range loopbackAddresses { + b.add("-d", address.String(), "-j", "RETURN") + } + b.add(redirect...) + b.chain = parent +} + +func (r *autoRedirect) findLocalAddresses() ([]netip.Prefix, error) { + err := r.interfaceFinder.Update() + if err != nil { + return nil, E.Cause(err, "update interfaces") + } + return common.Uniq(common.FlatMap(r.interfaceFinder.Interfaces(), func(it control.Interface) []netip.Prefix { + return common.Map(common.Filter(it.Addresses, func(prefix netip.Prefix) bool { + return it.Name == "lo" || prefix.Addr().IsGlobalUnicast() + }), netip.Prefix.Masked) + })), nil +} + +func (r *autoRedirect) iptablesFillLocalChain(chain iptablesLocalChain, addresses []netip.Prefix) error { + prefixes := common.Filter(addresses, func(it netip.Prefix) bool { + if chain.family.isIPv6 { + return it.Addr().Is6() && !it.Addr().Is4In6() + } + return it.Addr().Is4() + }) + if chain.dnsHijack { + action := func(protocol string) []string { + if chain.kind == iptablesKindFilter { + return []string{"-j", "RETURN"} + } + if chain.kind == iptablesKindNAT && !chain.family.isIPv6 && protocol == "udp" { + return []string{"-j", "DNAT", "--to-destination", chain.family.dnsServer.String()} + } + return []string{"-g", chain.next} + } + if chain.hook == iptablesHookPrerouting { + for _, prefix := range prefixes { + for _, protocol := range []string{"udp", "tcp"} { + err := iptablesRun(chain.family.path, slices.Concat([]string{ + "-t", chain.table, "-A", chain.name, + "-s", prefix.String(), "-p", protocol, "--dport", "53", + }, action(protocol))...) + if err != nil { + return err + } + } + } + } else { + for _, protocol := range []string{"udp", "tcp"} { + err := iptablesRun(chain.family.path, slices.Concat([]string{ + "-t", chain.table, "-A", chain.name, + "!", "-o", "lo", "-p", protocol, "--dport", "53", + }, action(protocol))...) + if err != nil { + return err + } + } } } - if r.enableIPv6 { - err := r.setupIPTablesForFamily(r.ip6tablesPath) + for _, prefix := range prefixes { + err := iptablesRun(chain.family.path, "-t", chain.table, "-A", chain.name, "-d", prefix.String(), "-j", "RETURN") if err != nil { - return E.Cause(err, "setup ip6tables") + return err } } - return nil + return iptablesRun(chain.family.path, "-t", chain.table, "-A", chain.name, "-j", chain.next) } -func (r *autoRedirect) setupIPTablesForFamily(iptablesPath string) error { - tableNameOutput := r.tableName + "-output" - redirectPort := r.redirectPort() - // OUTPUT - err := r.runShell(iptablesPath, "-t nat -N", tableNameOutput) +func (r *autoRedirect) updateIPTablesLocalAddresses() error { + localAddresses, err := r.findLocalAddresses() if err != nil { return err } - err = r.runShell(iptablesPath, "-t nat -A", tableNameOutput, - "-p tcp -o", r.tunOptions.Name, - "-j REDIRECT --to-ports", redirectPort) - if err != nil { - return err + if slices.Equal(localAddresses, r.localAddresses) { + return nil } - err = r.runShell(iptablesPath, "-t nat -I OUTPUT -j", tableNameOutput) + r.localAddresses = localAddresses + for _, chain := range r.iptablesLocalChains { + err = iptablesRun(chain.family.path, "-t", chain.table, "-F", chain.name) + if err != nil { + return err + } + err = r.iptablesFillLocalChain(chain, localAddresses) + if err != nil { + return err + } + } + return nil +} + +func (r *autoRedirect) iptablesAddOwnIngressReturn(builder *iptablesBuilder) { + builder.add("-i", r.tunOptions.Name, "-j", "RETURN") + for _, name := range r.tunOptions.BridgeInterface { + builder.add("-i", name, "-j", "RETURN") + } +} + +func (r *autoRedirect) iptablesAddExcludeRules(builder *iptablesBuilder, hook iptablesHook, kind iptablesChainKind, dnsHijack bool) { + options := r.tunOptions + mask := r.effectiveMarkMask() + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + if hook == iptablesHookOutput && kind != iptablesKindFilter { + builder.add("-m", "mark", "--mark", outputMark, "-j", "RETURN") + if kind == iptablesKindRoute { + builder.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + } + } + if kind == iptablesKindNAT { + builder.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + builder.add("-p", "tcp", "-m", "connmark", "--mark", iptablesMark(options.AutoRedirectInputMark, mask), "-j", "RETURN") + } + if hook == iptablesHookPrerouting { + r.iptablesAddOwnIngressReturn(builder) + builder.includeOnly(common.Map(options.IncludeInterface, func(it string) iptablesMatch { + return iptablesMatch{positive: []string{"-i", it}, negative: []string{"!", "-i", it}} + })) + for _, name := range options.ExcludeInterface { + builder.add("-i", name, "-j", "RETURN") + } + // xt_mac reports no match on interfaces without an ethernet header + // before applying the invert flag, so an inverted form would keep + // non-ethernet ingress included. + builder.includeOnly(common.Map(options.IncludeMACAddress, func(it net.HardwareAddr) iptablesMatch { + return iptablesMatch{positive: []string{"-m", "mac", "--mac-source", it.String()}} + })) + for _, address := range options.ExcludeMACAddress { + builder.add("-m", "mac", "--mac-source", address.String(), "-j", "RETURN") + } + } else { + builder.includeOnly(common.Map(options.IncludeUID, func(it ranges.Range[uint32]) iptablesMatch { + return iptablesMatch{ + positive: []string{"-m", "owner", "--uid-owner", iptablesUIDRange(it)}, + negative: []string{"-m", "owner", "!", "--uid-owner", iptablesUIDRange(it)}, + } + })) + for _, uidRange := range options.ExcludeUID { + builder.add("-m", "owner", "--uid-owner", iptablesUIDRange(uidRange), "-j", "RETURN") + } + } + routeAddress := options.Inet4RouteAddress + routeExcludeAddress := options.Inet4RouteExcludeAddress + if builder.family.isIPv6 { + routeAddress = options.Inet6RouteAddress + routeExcludeAddress = options.Inet6RouteExcludeAddress + } + builder.includeOnly(common.Map(routeAddress, func(it netip.Prefix) iptablesMatch { + return iptablesMatch{positive: []string{"-d", it.String()}, negative: []string{"!", "-d", it.String()}} + })) + for _, prefix := range routeExcludeAddress { + builder.add("-d", prefix.String(), "-j", "RETURN") + } + builder.localGate(hook, kind, dnsHijack) +} + +func (r *autoRedirect) iptablesAddPreMatchRules(builder *iptablesBuilder, hook iptablesHook, dnsHijack bool) { + options := r.tunOptions + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + resetMark := iptablesMark(r.tunOptions.AutoRedirectResetMark, mask) + if hook == iptablesHookOutput { + if r.androidVPNService { + builder.add("!", "-o", options.Name, "-j", "RETURN") + } + } else { + builder.add("-i", "lo", "-j", "RETURN") + r.iptablesAddOwnIngressReturn(builder) + } + builder.add("-m", "conntrack", "--ctdir", "REPLY", "-j", "RETURN") + if builder.family.tproxy { + tproxyMark := iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask) + builder.add("-m", "connmark", "--mark", tproxyMark, "-j", "RETURN") + builder.add("-m", "mark", "--mark", tproxyMark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "mark", "--mark", tproxyMark, "-j", "RETURN") + } + for _, mark := range []string{outputMark, inputMark} { + builder.add("-m", "mark", "--mark", mark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "mark", "--mark", mark, "-j", "RETURN") + builder.add("-m", "connmark", "--mark", mark, "-j", "CONNMARK", "--restore-mark", "--nfmask", maskValue, "--ctmask", maskValue) + builder.add("-m", "connmark", "--mark", mark, "-j", "RETURN") + } + builder.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + builder.add("-p", "tcp", "!", "--tcp-flags", "SYN,ACK", "SYN", "-j", "RETURN") + r.iptablesAddExcludeRules(builder, hook, iptablesKindFilter, dnsHijack) + queue := []string{"-j", "NFQUEUE", "--queue-num", strconv.Itoa(int(r.effectiveNFQueue())), "--queue-bypass"} + builder.add(slices.Concat([]string{"-p", "tcp"}, queue)...) + builder.add(slices.Concat([]string{"-p", "udp"}, queue)...) + if builder.family.isIPv6 { + builder.add(slices.Concat([]string{"-p", "icmpv6", "--icmpv6-type", strconv.Itoa(int(header.ICMPv6EchoRequest))}, queue)...) + } else { + builder.add(slices.Concat([]string{"-p", "icmp", "--icmp-type", strconv.Itoa(int(header.ICMPv4Echo))}, queue)...) + } +} + +func (r *autoRedirect) iptablesAddMarkRules(builder *iptablesBuilder) { + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + builder.add("-j", "MARK", "--set-xmark", inputMark) + builder.add("-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) +} + +func (r *autoRedirect) iptablesAddLoopbackReroute(builder *iptablesBuilder, hook iptablesHook) { + loopbackAddresses := r.tunOptions.Inet4LoopbackAddress + if builder.family.isIPv6 { + loopbackAddresses = r.tunOptions.Inet6LoopbackAddress + } + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + for _, address := range loopbackAddresses { + builder.add("-p", "tcp", "-d", address.String(), "-m", "mark", "!", "--mark", inputMark, "-j", "MARK", "--set-xmark", inputMark) + if hook == iptablesHookOutput { + builder.add("-p", "tcp", "-d", address.String(), "-m", "mark", "--mark", inputMark, + "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + } + } +} + +func (r *autoRedirect) setupIPTables() error { + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + localAddresses, err := r.findLocalAddresses() if err != nil { return err } - if r.redirectServer != nil { - tableNameInput := r.tableName + "-input" - err = r.runShell(iptablesPath, "-t filter -N", tableNameInput) + r.localAddresses = localAddresses + var families []*iptablesFamily + if r.enableIPv4 { + families = append(families, &iptablesFamily{path: r.iptablesPath}) + } + if r.enableIPv6 { + families = append(families, &iptablesFamily{path: r.ip6tablesPath, isIPv6: true, tproxy: r.tproxyEnabled()}) + } + for _, family := range families { + err = r.setupIPTablesForFamily(family) if err != nil { return err } - err = r.runShell(iptablesPath, "-t filter -A", tableNameInput, - "-p tcp --dport", redirectPort, - "-m conntrack ! --ctstate DNAT", - "-j REJECT --reject-with tcp-reset") - if err != nil { - return err + } + return r.setupIPTablesUnreachable() +} + +func (r *autoRedirect) setupIPTablesForFamily(family *iptablesFamily) error { + options := r.tunOptions + mask := r.effectiveMarkMask() + maskValue := iptablesHex(mask) + inputMark := iptablesMark(r.tunOptions.AutoRedirectInputMark, mask) + outputMark := iptablesMark(r.tunOptions.AutoRedirectOutputMark, mask) + resetMark := iptablesMark(r.tunOptions.AutoRedirectResetMark, mask) + loopbackAddresses := options.Inet4LoopbackAddress + if family.isIPv6 { + loopbackAddresses = options.Inet6LoopbackAddress + } + dnsHijack := options.DNSModeOrDefault() == DNSModeHijack + if dnsHijack { + var ( + dnsServers []netip.Addr + dnsErr error + ) + if family.isIPv6 { + dnsServers, dnsErr = options.Inet6DNSAddress() + } else { + dnsServers, dnsErr = options.Inet4DNSAddress() + } + if dnsErr != nil { + return dnsErr } - err = r.runShell(iptablesPath, "-t filter -I INPUT -j", tableNameInput) + if len(dnsServers) == 0 { + dnsHijack = false + } else { + family.dnsServer = dnsServers[0] + } + } + markProtocols := iptablesMarkProtocols(family) + var inserts []iptablesInsert + + if len(loopbackAddresses) > 0 { + if !r.shouldSkipOutputChain() { + outputLoopback := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-loopback") + r.iptablesAddLoopbackReroute(outputLoopback, iptablesHookOutput) + if outputLoopback.err != nil { + return outputLoopback.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-j", outputLoopback.name}}) + } + preroutingLoopback := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-loopback") + r.iptablesAddLoopbackReroute(preroutingLoopback, iptablesHookPrerouting) + if preroutingLoopback.err != nil { + return preroutingLoopback.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-j", preroutingLoopback.name}}) + } + + if !r.shouldSkipOutputChain() { + outputPreMatch := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-prematch") + r.iptablesAddPreMatchRules(outputPreMatch, iptablesHookOutput, dnsHijack) + if outputPreMatch.err != nil { + return outputPreMatch.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-j", outputPreMatch.name}}) + } + preroutingPreMatch := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-prematch") + r.iptablesAddPreMatchRules(preroutingPreMatch, iptablesHookPrerouting, dnsHijack) + if preroutingPreMatch.err != nil { + return preroutingPreMatch.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-j", preroutingPreMatch.name}}) + + if !r.shouldSkipOutputChain() { + outputMarkChain := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-mark") + outputMarkChain.addVPNServiceGate() + if family.isIPv6 && !family.tproxy { + outputMarkChain.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + } + r.iptablesAddExcludeRules(outputMarkChain, iptablesHookOutput, iptablesKindRoute, dnsHijack) + r.iptablesAddMarkRules(outputMarkChain) + if outputMarkChain.err != nil { + return outputMarkChain.err + } + for _, protocol := range markProtocols { + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", protocol, "-j", outputMarkChain.name}}) + } + } + preroutingMarkChain := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-mark") + r.iptablesAddOwnIngressReturn(preroutingMarkChain) + if family.isIPv6 && !family.tproxy { + preroutingMarkChain.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + } + preroutingMarkChain.add("-m", "connmark", "--mark", inputMark, "-j", "MARK", "--set-xmark", inputMark) + preroutingMarkChain.add("-m", "connmark", "--mark", inputMark, "-j", "RETURN") + preroutingMarkChain.add("-m", "connmark", "--mark", outputMark, "-j", "MARK", "--set-xmark", outputMark) + preroutingMarkChain.add("-m", "connmark", "--mark", outputMark, "-j", "RETURN") + include := preroutingMarkChain.branch("include") + r.iptablesAddExcludeRules(include, iptablesHookPrerouting, iptablesKindRoute, dnsHijack) + r.iptablesAddMarkRules(include) + preroutingMarkChain.absorb(include) + preroutingMarkChain.add("-m", "mark", "--mark", inputMark, "-j", "RETURN") + preroutingMarkChain.add("-j", "MARK", "--set-xmark", outputMark) + preroutingMarkChain.add("-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + if preroutingMarkChain.err != nil { + return preroutingMarkChain.err + } + for _, protocol := range markProtocols { + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-p", protocol, "-j", preroutingMarkChain.name}}) + } + if r.androidVPNService && !family.isIPv6 { + // A TCP flow addressed to the router itself leaves the pre-match chain + // without a connmark, the accepted socket inherits the unmarked SYN + // through tcp_fwmark_accept, and netd's VPN rule then routes its replies + // into the tun. + preroutingLocal := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-local") + r.iptablesAddOwnIngressReturn(preroutingLocal) + preroutingLocal.add("-m", "connmark", "!", "--mark", iptablesMark(0, mask), "-j", "RETURN") + preroutingLocal.add("-j", "MARK", "--set-xmark", outputMark) + localInclude := preroutingLocal.branch("include") + r.iptablesAddExcludeRules(localInclude, iptablesHookPrerouting, iptablesKindRoute, dnsHijack) + localInclude.add("-j", "MARK", "--set-xmark", iptablesMark(0, mask)) + preroutingLocal.absorb(localInclude) + preroutingLocal.add("-m", "mark", "--mark", outputMark, "-j", "CONNMARK", "--save-mark", "--nfmask", maskValue, "--ctmask", maskValue) + if preroutingLocal.err != nil { + return preroutingLocal.err + } + inserts = append(inserts, iptablesInsert{ + iptablesTableMangle, "PREROUTING", + []string{"-p", "tcp", "-m", "mark", "--mark", iptablesMark(0, mask), "-j", preroutingLocal.name}, + }) + } + + if !family.isIPv6 { + if !r.shouldSkipOutputChain() { + outputNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-output") + outputNAT.addVPNServiceGate() + r.iptablesAddExcludeRules(outputNAT, iptablesHookOutput, iptablesKindNAT, dnsHijack) + outputNAT.addRedirect() + if outputNAT.err != nil { + return outputNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "OUTPUT", []string{"-j", outputNAT.name}}) + } + preroutingNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-prerouting") + preroutingNAT.add("-p", "tcp", "-m", "mark", "--mark", resetMark, "-j", "RETURN") + r.iptablesAddExcludeRules(preroutingNAT, iptablesHookPrerouting, iptablesKindNAT, dnsHijack) + preroutingNAT.addRedirect() + r.iptablesAddMarkRules(preroutingNAT) + if preroutingNAT.err != nil { + return preroutingNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "PREROUTING", []string{"-j", preroutingNAT.name}}) + if r.androidVPNService { + postroutingNAT := r.iptablesChain(family, iptablesTableNAT, r.tableName+"-postrouting") + for _, prefix := range options.Inet4Address { + postroutingNAT.add("-s", prefix.Masked().String(), "!", "-o", options.Name, "-m", "mark", "--mark", outputMark, "-j", "MASQUERADE") + } + if postroutingNAT.err != nil { + return postroutingNAT.err + } + inserts = append(inserts, iptablesInsert{iptablesTableNAT, "POSTROUTING", []string{"-j", postroutingNAT.name}}) + postroutingMangle := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-postrouting") + for _, prefix := range options.Inet4Address { + postroutingMangle.add("-s", prefix.Masked().String(), "!", "-o", options.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-m", "mark", "--mark", outputMark, "-j", "TCPMSS", "--clamp-mss-to-pmtu") + } + if postroutingMangle.err != nil { + return postroutingMangle.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "POSTROUTING", []string{"-j", postroutingMangle.name}}) + } + } + + if r.androidVPNService && (!family.isIPv6 || family.tproxy) { + // The redirect server runs in the app process, and without the + // protectedFromVpn bit netd's VPN rule routes its replies back + // into the tun instead of delivering them over loopback. + outputProtect := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-protect") + if family.isIPv6 { + outputProtect.add("-m", "connmark", "--mark", iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask), + "-m", "conntrack", "--ctdir", "REPLY", + "-j", "MARK", "--set-xmark", iptablesMark(androidSecureVPNMask, androidSecureVPNMask)) + } else { + outputProtect.add("-p", "tcp", "--sport", strconv.Itoa(int(r.redirectPort())), + "-j", "MARK", "--set-xmark", iptablesMark(androidSecureVPNMask, androidSecureVPNMask)) + } + if outputProtect.err != nil { + return outputProtect.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", "tcp", "-j", outputProtect.name}}) + } + + if family.tproxy { + tproxyMark := iptablesMark(r.tunOptions.AutoRedirectTProxyMark, mask) + tproxyAction := []string{ + "-p", "tcp", "-j", "TPROXY", + "--on-port", strconv.Itoa(int(r.redirectPort())), "--tproxy-mark", tproxyMark, + } + if !r.shouldSkipOutputChain() { + outputTProxy := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-output-tproxy") + // Under VpnService the app's own sockets carry the protectedFromVpn bit instead of + // the routing mark, so netd routes them past the tun. + outputTProxy.addVPNServiceGate() + outputTProxy.add("-m", "mark", "--mark", resetMark, "-j", "RETURN") + outputTProxy.add("-m", "connmark", "--mark", tproxyMark, "-m", "conntrack", "--ctdir", "ORIGINAL", + "-j", "MARK", "--set-xmark", tproxyMark) + outputTProxy.add("-m", "mark", "--mark", tproxyMark, "-j", "RETURN") + r.iptablesAddExcludeRules(outputTProxy, iptablesHookOutput, iptablesKindNAT, dnsHijack) + outputTProxy.add("-j", "MARK", "--set-xmark", tproxyMark) + outputTProxy.add("-j", "CONNMARK", "--set-xmark", tproxyMark) + if outputTProxy.err != nil { + return outputTProxy.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "OUTPUT", []string{"-p", "tcp", "-j", outputTProxy.name}}) + } + preroutingTProxy := r.iptablesChain(family, iptablesTableMangle, r.tableName+"-prerouting-tproxy") + preroutingTProxy.add(slices.Concat([]string{"-m", "mark", "--mark", tproxyMark}, tproxyAction)...) + preroutingTProxy.add("-m", "mark", "--mark", outputMark, "-j", "RETURN") + preroutingTProxy.add("-m", "mark", "--mark", resetMark, "-j", "RETURN") + preroutingTProxy.add(slices.Concat([]string{ + "-m", "connmark", "--mark", tproxyMark, + "-m", "conntrack", "--ctdir", "ORIGINAL", + }, tproxyAction)...) + r.iptablesAddExcludeRules(preroutingTProxy, iptablesHookPrerouting, iptablesKindNAT, dnsHijack) + preroutingTProxy.add("-j", "CONNMARK", "--set-xmark", tproxyMark) + preroutingTProxy.add(tproxyAction...) + if preroutingTProxy.err != nil { + return preroutingTProxy.err + } + inserts = append(inserts, iptablesInsert{iptablesTableMangle, "PREROUTING", []string{"-p", "tcp", "-j", preroutingTProxy.name}}) + } + + input := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-input") + input.add("-p", "tcp", "--dport", strconv.Itoa(int(r.redirectPort())), + "-m", "conntrack", "!", "--ctstate", "DNAT", + "-j", "REJECT", "--reject-with", "tcp-reset") + if input.err != nil { + return input.err + } + inserts = append(inserts, iptablesInsert{iptablesTableFilter, "INPUT", []string{"-j", input.name}}) + + reset := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-reset") + reset.add("-p", "tcp", "!", "--tcp-flags", "RST", "RST", "-m", "mark", "--mark", resetMark, "-j", "REJECT", "--reject-with", "tcp-reset") + if reset.err != nil { + return reset.err + } + inserts = append(inserts, + iptablesInsert{iptablesTableFilter, "INPUT", []string{"-j", reset.name}}, + iptablesInsert{iptablesTableFilter, "OUTPUT", []string{"-j", reset.name}}, + iptablesInsert{iptablesTableFilter, "FORWARD", []string{"-j", reset.name}}, + ) + + // netd ends its tetherctrl_FORWARD chain with an unconditional DROP and + // only accepts its own downstream/upstream interface pairs, so forwarded + // flows marked into the tun need their own accept. + if runtime.GOOS == "android" { + forward := r.iptablesChain(family, iptablesTableFilter, r.tableName+"-forward") + forward.add("-o", options.Name, "-m", "mark", "--mark", inputMark, "-j", "ACCEPT") + forward.add("-i", options.Name, "-m", "connmark", "--mark", inputMark, "-j", "ACCEPT") + if forward.err != nil { + return forward.err + } + inserts = append(inserts, iptablesInsert{iptablesTableFilter, "FORWARD", []string{"-j", forward.name}}) + } + + for _, insert := range slices.Backward(inserts) { + err := iptablesRun(family.path, slices.Concat([]string{"-t", insert.table, "-I", insert.base}, insert.args)...) if err != nil { return err } @@ -65,39 +708,167 @@ func (r *autoRedirect) setupIPTablesForFamily(iptablesPath string) error { return nil } +func iptablesMarkProtocols(family *iptablesFamily) []string { + var protocols []string + if family.isIPv6 { + protocols = []string{"udp", "icmpv6"} + } else { + protocols = []string{"udp", "icmp"} + } + if family.isIPv6 && !family.tproxy { + protocols = append(protocols, "tcp") + } + return protocols +} + +func (r *autoRedirect) setupIPTablesUnreachable() error { + if !r.tunOptions.StrictRoute || r.enableIPv4 == r.enableIPv6 { + return nil + } + path, err := findIPTablesBinary(r.enableIPv4) + if err != nil { + r.logger.Warn("strict route is not available for the disabled address family: ", err) + return nil + } + family := &iptablesFamily{path: path, isIPv6: r.enableIPv4} + outputName := r.tableName + "-unreachable-output" + outputBuilder := r.iptablesChain(family, iptablesTableFilter, outputName) + r.iptablesAddExcludeRules(outputBuilder, iptablesHookOutput, iptablesKindFilter, false) + outputBuilder.add("-j", "REJECT") + if outputBuilder.err != nil { + return outputBuilder.err + } + forwardName := r.tableName + "-unreachable-forward" + forwardBuilder := r.iptablesChain(family, iptablesTableFilter, forwardName) + r.iptablesAddExcludeRules(forwardBuilder, iptablesHookPrerouting, iptablesKindFilter, false) + forwardBuilder.add("-j", "REJECT") + if forwardBuilder.err != nil { + return forwardBuilder.err + } + err = iptablesRun(path, "-t", iptablesTableFilter, "-I", "OUTPUT", "-j", outputName) + if err != nil { + return err + } + return iptablesRun(path, "-t", iptablesTableFilter, "-I", "FORWARD", "-j", forwardName) +} + +func findIPTablesBinary(isIPv6 bool) (string, error) { + name := "iptables" + if isIPv6 { + name = "ip6tables" + } + if runtime.GOOS == "android" { + path := "/system/bin/" + name + _, err := os.Stat(path) + if err != nil { + return "", err + } + return path, nil + } + return exec.LookPath(name) +} + func (r *autoRedirect) cleanupIPTables() { - if r.enableIPv4 { - r.cleanupIPTablesForFamily(r.iptablesPath) + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + r.iptablesLocalChains = nil + var paths []string + for _, isIPv6 := range []bool{false, true} { + path, err := findIPTablesBinary(isIPv6) + if err == nil { + paths = append(paths, path) + } } - if r.enableIPv6 { - r.cleanupIPTablesForFamily(r.ip6tablesPath) + for _, path := range common.Uniq(paths) { + r.cleanupIPTablesForFamily(path) + } +} + +var ( + iptablesChainSuffixes = []string{ + "forward", "input", "output", "output-loopback", "output-mark", "output-prematch", + "output-protect", "output-tproxy", "postrouting", "prerouting", "prerouting-local", + "prerouting-loopback", "prerouting-mark", "prerouting-prematch", "prerouting-tproxy", + "reset", "unreachable-forward", "unreachable-output", + } + iptablesBranchPrefixes = []string{"cont", "include", "local", "redirect"} +) + +func (r *autoRedirect) ownsIPTablesChain(name string) bool { + suffix, found := strings.CutPrefix(name, r.tableName+"-") + if !found { + return false + } + if slices.Contains(iptablesChainSuffixes, suffix) { + return true + } + prefix, counter, found := strings.Cut(suffix, "-") + if !found || !slices.Contains(iptablesBranchPrefixes, prefix) { + return false } + _, err := strconv.ParseUint(counter, 10, 32) + return err == nil } func (r *autoRedirect) cleanupIPTablesForFamily(iptablesPath string) { - tableNameOutput := r.tableName + "-output" - tableNameInput := r.tableName + "-input" - - _ = r.runShell(iptablesPath, "-t nat -D OUTPUT -j", tableNameOutput) - _ = r.runShell(iptablesPath, "-t nat -F", tableNameOutput) - _ = r.runShell(iptablesPath, "-t nat -X", tableNameOutput) - _ = r.runShell(iptablesPath, "-t filter -D INPUT -j", tableNameInput) - _ = r.runShell(iptablesPath, "-t filter -F", tableNameInput) - _ = r.runShell(iptablesPath, "-t filter -X", tableNameInput) -} - -func (r *autoRedirect) runShell(commands ...any) error { - commandStr := strings.Join(F.MapToString(commands), " ") - var command *exec.Cmd - if r.androidSu { - command = exec.Command(r.suPath, "-c", commandStr) - } else { - commandArray := strings.Split(commandStr, " ") - command = exec.Command(commandArray[0], commandArray[1:]...) + for _, table := range []string{iptablesTableNAT, iptablesTableMangle, iptablesTableFilter} { + output, err := iptablesRunOutput(iptablesPath, "-t", table, "-S") + if err != nil { + continue + } + var chains []string + var references [][]string + for line := range strings.SplitSeq(output, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + switch fields[0] { + case "-N": + if r.ownsIPTablesChain(fields[1]) { + chains = append(chains, fields[1]) + } + case "-A": + if r.ownsIPTablesChain(fields[1]) { + continue + } + for index := 2; index < len(fields)-1; index++ { + if fields[index] == "-j" && r.ownsIPTablesChain(fields[index+1]) { + references = append(references, fields[1:]) + break + } + } + } + } + for _, reference := range references { + _ = iptablesRun(iptablesPath, slices.Concat([]string{"-t", table, "-D"}, reference)...) + } + for _, chain := range chains { + _ = iptablesRun(iptablesPath, "-t", table, "-F", chain) + } + for _, chain := range chains { + _ = iptablesRun(iptablesPath, "-t", table, "-X", chain) + } } - combinedOutput, err := command.CombinedOutput() +} + +// netd runs its own iptables commands whenever a network (including the VPN +// interface) appears or disappears, so a run without -w fails with "Can't +// lock /system/etc/xtables.lock" while netd holds the lock. +func iptablesRun(path string, args ...string) error { + args = append([]string{"-w"}, args...) + combinedOutput, err := exec.Command(path, args...).CombinedOutput() if err != nil { - return E.Extend(err, F.ToString(commandStr, ": ", string(combinedOutput))) + return E.Extend(err, F.ToString(path, " ", strings.Join(args, " "), ": ", strings.TrimSpace(string(combinedOutput)))) } return nil } + +func iptablesRunOutput(path string, args ...string) (string, error) { + args = append([]string{"-w"}, args...) + output, err := exec.Command(path, args...).Output() + if err != nil { + return "", E.Extend(err, F.ToString(path, " ", strings.Join(args, " "))) + } + return string(output), nil +} diff --git a/redirect_linux.go b/redirect_linux.go index dd8c61f4..798b63f1 100644 --- a/redirect_linux.go +++ b/redirect_linux.go @@ -5,193 +5,128 @@ import ( "net/netip" "os" "os/exec" - "path/filepath" "runtime" - "strings" "sync" + "syscall" + "github.com/sagernet/netlink" "github.com/sagernet/nftables" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/logger" - M "github.com/sagernet/sing/common/metadata" "github.com/sagernet/sing/common/x/list" - "go4.org/netipx" + "golang.org/x/sys/unix" ) type autoRedirect struct { - tunOptions *Options - ctx context.Context - handler Handler - logger logger.Logger - tableName string - networkMonitor NetworkUpdateMonitor - ownedNetworkMonitor bool - networkListener *list.Element[NetworkUpdateCallback] - interfaceFinder control.InterfaceFinder - localAddresses []netip.Prefix - customRedirectPortFunc func() int - customRedirectPort int - redirectServer *redirectServer - enableIPv4 bool - enableIPv6 bool - iptablesPath string - ip6tablesPath string - useNFTables bool - androidSu bool - suPath string - routeAddressSet *[]*netipx.IPSet - routeExcludeAddressSet *[]*netipx.IPSet - nfqueueHandler *nfqueueHandler - nfqueueEnabled bool - redirectRouteTableIndex int - redirectRouteAccess sync.Mutex - redirectRoutesActive bool - dockerFirewallMonitor *nftables.Monitor - dockerFirewallDone chan struct{} + tunOptions *Options + ctx context.Context + handler AutoRedirectHandler + logger logger.Logger + tableName string + networkMonitor NetworkUpdateMonitor + ownedNetworkMonitor bool + networkListener *list.Element[NetworkUpdateCallback] + interfaceListener *list.Element[DefaultInterfaceUpdateCallback] + interfaceFinder control.InterfaceFinder + localAddresses []netip.Prefix + customRedirectPortFunc func() int + customRedirectPort int + customRedirectListenerFD func() (int, error) + customRedirectTransparent bool + routeAddressSet func() ([]netip.Prefix, []netip.Prefix, error) + bypassRouteTableIndex int + bypassRouteAccess sync.Mutex + bypassRouteActive bool + bypassIncludeAddresses []netip.Prefix + bypassExcludeAddresses []netip.Prefix + redirectServer *RedirectServer + enableIPv4 bool + enableIPv6 bool + iptablesPath string + ip6tablesPath string + useNFTables bool + androidVPNService bool + nfqueueHandler *nfqueueHandler + iptablesAccess sync.Mutex + iptablesLocalChains []iptablesLocalChain + redirectRouteTableIndex int + tproxyRouteTableIndex int + redirectRouteAccess sync.Mutex + redirectRoutesActive bool + androidVPNServiceRuleAccess sync.Mutex + androidVPNServiceRules []*netlink.Rule + androidVPNServiceRulesActive bool + androidIngressDiscardValues map[netip.Addr][8]byte + dockerFirewallMonitor *nftables.Monitor + dockerFirewallDone chan struct{} } func NewAutoRedirect(options AutoRedirectOptions) (AutoRedirect, error) { - if options.TunOptions.AutoRedirectInputMark == 0 { - options.TunOptions.AutoRedirectInputMark = DefaultAutoRedirectInputMark - } r := &autoRedirect{ - tunOptions: options.TunOptions, - ctx: options.Context, - handler: options.Handler, - logger: options.Logger, - networkMonitor: options.NetworkMonitor, - interfaceFinder: options.InterfaceFinder, - tableName: options.TableName, - useNFTables: runtime.GOOS != "android" && !options.DisableNFTables, - customRedirectPortFunc: options.CustomRedirectPort, - routeAddressSet: options.RouteAddressSet, - routeExcludeAddressSet: options.RouteExcludeAddressSet, + tunOptions: options.TunOptions, + ctx: options.Context, + handler: options.Handler, + logger: options.Logger, + networkMonitor: options.NetworkMonitor, + interfaceFinder: options.InterfaceFinder, + tableName: options.TableName, + useNFTables: runtime.GOOS != "android" && !options.DisableNFTables, + androidVPNService: options.AndroidVPNService, + customRedirectPortFunc: options.CustomRedirectPort, + customRedirectListenerFD: options.CustomRedirectListenerFD, + routeAddressSet: options.RouteAddressSet, } if options.TunOptions.NetNs != "" { r.interfaceFinder = &networkNamespaceInterfaceFinder{control.NewDefaultInterfaceFinder(), options.TunOptions} } - return r, nil -} - -func findAndroidSuPath() (string, error) { - searchPaths := common.Uniq(common.FilterNotDefault(append( - strings.Split(os.Getenv("PATH"), ":"), - "/system/bin", - "/system/xbin", - "/product/bin", - "/odm/bin", - "/vendor/bin", - "/vendor/xbin", - "/apex/com.android.runtime/bin", - "/sbin", - ))) - for _, searchPath := range searchPaths { - suPath, err := exec.LookPath(filepath.Join(searchPath, "su")) - if err == nil { - return suPath, nil + for _, mark := range []struct { + field *uint32 + defaultValue uint32 + androidDefaultValue uint32 + }{ + {&r.tunOptions.AutoRedirectInputMark, DefaultAutoRedirectInputMark, DefaultAutoRedirectInputMarkAndroid}, + {&r.tunOptions.AutoRedirectOutputMark, DefaultAutoRedirectOutputMark, DefaultAutoRedirectOutputMarkAndroid}, + {&r.tunOptions.AutoRedirectResetMark, DefaultAutoRedirectResetMark, DefaultAutoRedirectResetMarkAndroid}, + {&r.tunOptions.AutoRedirectTProxyMark, DefaultAutoRedirectTProxyMark, DefaultAutoRedirectTProxyMarkAndroid}, + } { + value := effectiveMark(*mark.field, mark.defaultValue, mark.androidDefaultValue) + if *mark.field != 0 && *mark.field != value { + r.logger.Warn("configured mark ", iptablesHex(*mark.field), " overlaps the fwmark bits reserved by Android and is replaced by ", iptablesHex(value)) } + *mark.field = value } - return "", E.New("su not found in ", strings.Join(searchPaths, ":")) + return r, nil } func (r *autoRedirect) Start() error { - var err error - if runtime.GOOS == "android" { - r.enableIPv4 = true - r.iptablesPath = "/system/bin/iptables" - userId := os.Getuid() - if userId != 0 { - r.androidSu = true - r.suPath, err = findAndroidSuPath() - if err != nil { - return E.Cause(err, "root permission is required for auto redirect") - } - } - } else { - if r.tunOptions.NetNs != "" && !r.useNFTables { - return E.New("auto_redirect in network namespace requires nftables") - } - if r.useNFTables { - err = runInNetworkNamespace(r.tunOptions.NetNs, r.initializeNFTables) - if err != nil { - return E.Cause(err, "missing nftables support") - } - } - if len(r.tunOptions.Inet4Address) > 0 { - r.enableIPv4 = true - if !r.useNFTables { - r.iptablesPath, err = exec.LookPath("iptables") - if err != nil { - return E.Cause(err, "iptables is required") - } - } - } - if len(r.tunOptions.Inet6Address) > 0 { - r.enableIPv6 = true - if !r.useNFTables { - r.ip6tablesPath, err = exec.LookPath("ip6tables") - if err != nil { - if !r.enableIPv4 { - return E.Cause(err, "ip6tables is required") - } else { - r.enableIPv6 = false - r.logger.Error("device has no ip6tables nat support: ", err) - } - } - } - } + err := r.initializeBackend() + if err != nil { + return err } - if r.customRedirectPortFunc != nil { - r.customRedirectPort = r.customRedirectPortFunc() + err = r.startRedirectServer() + if err != nil { + return err } - if r.customRedirectPort == 0 { - var listenAddr netip.Addr - if runtime.GOOS == "android" { - listenAddr = netip.AddrFrom4([4]byte{127, 0, 0, 1}) - } else if r.enableIPv6 { - listenAddr = netip.IPv6Unspecified() - } else { - listenAddr = netip.IPv4Unspecified() - } - server := newRedirectServer(r.ctx, r.handler, r.logger, listenAddr) - err = runInNetworkNamespace(r.tunOptions.NetNs, server.Start) - if err != nil { - return E.Cause(err, "start redirect server") - } - r.redirectServer = server + err = r.startNFQueue() + if err != nil { + _ = r.Close() + return E.Cause(err, "start nfqueue") } if r.useNFTables { - if r.handler != nil { - var handler *nfqueueHandler - handler, err = newNFQueueHandler(nfqueueOptions{ - Context: r.ctx, - Handler: r.handler, - Logger: r.logger, - Queue: r.effectiveNFQueue(), - InputMark: r.tunOptions.AutoRedirectInputMark, - OutputMark: r.effectiveOutputMark(), - ResetMark: r.effectiveResetMark(), - }) - if err != nil { - r.logger.Warn("nfqueue not available, pre-match disabled (missing nfnetlink_queue and nft_queue kernel module?): ", err) - } else if err = runInNetworkNamespace(r.tunOptions.NetNs, handler.Start); err != nil { - r.logger.Warn("nfqueue start failed, pre-match disabled (missing nfnetlink_queue and nft_queue kernel module?): ", err) - } else { - r.nfqueueHandler = handler - r.nfqueueEnabled = true - } - } if r.tunOptions.NetNs != "" { var monitor NetworkUpdateMonitor monitor, err = NewNetworkUpdateMonitor(r.logger) if err != nil { + _ = r.Close() return E.Cause(err, "create netns network monitor") } err = runInNetworkNamespace(r.tunOptions.NetNs, monitor.Start) if err != nil { + _ = monitor.Close() + _ = r.Close() return E.Cause(err, "start netns network monitor") } r.networkMonitor = monitor @@ -206,29 +141,244 @@ func (r *autoRedirect) Start() error { if r.tunOptions.AutoRedirectMarkMode { setupErr = r.setupRedirectRoutes() if setupErr != nil { - r.cleanupNFTables() return E.Cause(setupErr, "setup redirect routes") } } return nil }) if err != nil { + _ = r.Close() return err } - } else { - r.cleanupIPTables() - err = r.setupIPTables() + return nil + } + r.cleanupIPTables() + err = r.setupRedirectRoutes() + if err != nil { + _ = r.Close() + return E.Cause(err, "setup redirect routes") + } + if r.androidVPNService { + err = r.setupAndroidVPNServiceRules() if err != nil { - return E.Cause(err, "setup iptables") + _ = r.Close() + return E.Cause(err, "setup android vpn service rules") } + r.setupBypassRoute() } + err = r.setupIPTables() + if err != nil { + _ = r.Close() + return E.Cause(err, "setup iptables") + } + r.registerNetworkCallback(func() { + updateErr := r.updateIPTablesNetwork() + if updateErr != nil { + r.logger.Error(updateErr) + } + }) return nil } -func (r *autoRedirect) Close() error { - if r.nfqueueHandler != nil { - r.nfqueueHandler.Close() +func (r *autoRedirect) registerNetworkCallback(callback NetworkUpdateCallback) { + if r.networkMonitor != nil { + r.networkListener = r.networkMonitor.RegisterCallback(callback) + return + } + if r.tunOptions.InterfaceMonitor != nil { + r.interfaceListener = r.tunOptions.InterfaceMonitor.RegisterCallback(func(defaultInterface *control.Interface, flags int) { + callback() + }) + return + } + r.logger.Warn("no network monitor available, address and route updates are disabled") +} + +func (r *autoRedirect) unregisterNetworkCallback() { + if r.networkListener != nil { + r.networkMonitor.UnregisterCallback(r.networkListener) + r.networkListener = nil + } + if r.interfaceListener != nil { + r.tunOptions.InterfaceMonitor.UnregisterCallback(r.interfaceListener) + r.interfaceListener = nil + } +} + +func (r *autoRedirect) initializeBackend() error { + var err error + // iptables-nft refuses DROP in the nat table, where the MPTCP fallback drop belongs. + if r.tunOptions.ExcludeMPTCP && !r.useNFTables { + return E.New("exclude_mptcp requires nftables") + } + if runtime.GOOS == "android" { + if os.Getuid() != 0 { + return E.New("auto_redirect requires root") + } + if len(r.tunOptions.Inet4Address) > 0 { + r.iptablesPath, err = findIPTablesBinary(false) + if err != nil { + return E.Cause(err, "iptables is required") + } + r.enableIPv4 = true + } + if len(r.tunOptions.Inet6Address) > 0 { + r.ip6tablesPath, err = findIPTablesBinary(true) + if err != nil { + return E.Cause(err, "ip6tables is required") + } + r.enableIPv6 = true + } + return nil + } + if r.tunOptions.NetNs != "" && !r.useNFTables { + return E.New("auto_redirect in network namespace requires nftables") } + if r.useNFTables { + err = runInNetworkNamespace(r.tunOptions.NetNs, r.initializeNFTables) + if err != nil { + return E.Cause(err, "missing nftables support") + } + } + if len(r.tunOptions.Inet4Address) > 0 { + r.enableIPv4 = true + if !r.useNFTables { + r.iptablesPath, err = exec.LookPath("iptables") + if err != nil { + return E.Cause(err, "iptables is required") + } + err = exec.Command(r.iptablesPath, "-w", "-t", iptablesTableNAT, "-S").Run() + if err != nil { + return E.Cause(err, "iptables nat table is required") + } + } + } + if len(r.tunOptions.Inet6Address) > 0 { + r.enableIPv6 = true + if !r.useNFTables { + r.ip6tablesPath, err = exec.LookPath("ip6tables") + if err != nil { + return E.Cause(err, "ip6tables is required") + } + } + } + return nil +} + +func (r *autoRedirect) tproxyEnabled() bool { + return !r.useNFTables && r.enableIPv6 && (!r.androidVPNService || r.customRedirectTransparent) +} + +func (r *autoRedirect) prepareCustomTransparentListener() error { + fd, err := r.customRedirectListenerFD() + if err != nil { + return err + } + defer syscall.Close(fd) + err = syscall.SetsockoptInt(fd, syscall.SOL_IPV6, unix.IPV6_TRANSPARENT, 1) + if err != nil { + return E.Cause(err, "set IPV6_TRANSPARENT") + } + err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, int(r.tunOptions.AutoRedirectOutputMark)) + if err != nil { + return E.Cause(err, "set SO_MARK") + } + return nil +} + +func (r *autoRedirect) startRedirectServer() error { + if r.customRedirectPortFunc != nil { + r.customRedirectPort = r.customRedirectPortFunc() + } + if r.customRedirectPort > 0 { + if !r.useNFTables && r.enableIPv6 && r.androidVPNService && r.customRedirectListenerFD != nil { + err := r.prepareCustomTransparentListener() + if err != nil { + r.logger.Warn("IPv6 TCP falls back to the tun stack: prepare transparent listener: ", err) + } else { + r.customRedirectTransparent = true + } + } + return nil + } + var listenAddress netip.Addr + if r.enableIPv6 { + listenAddress = netip.IPv6Unspecified() + } else { + listenAddress = netip.IPv4Unspecified() + } + server := NewRedirectServer(r.ctx, r.handler, r.logger, listenAddress) + if r.tproxyEnabled() { + server.transparent = true + server.listenMark = r.tunOptions.AutoRedirectOutputMark + } + err := runInNetworkNamespace(r.tunOptions.NetNs, server.Start) + if err != nil { + return E.Cause(err, "start redirect server") + } + r.redirectServer = server + return nil +} + +const tcpOptionMultipathTCP = 30 + +func (r *autoRedirect) startNFQueue() error { + var markMask uint32 + if !r.useNFTables { + markMask = r.effectiveMarkMask() + } + var tproxyMark uint32 + if r.tproxyEnabled() { + tproxyMark = r.tunOptions.AutoRedirectTProxyMark + } + handler := &nfqueueHandler{ + handler: r.handler, + logger: r.logger, + queue: r.effectiveNFQueue(), + inputMark: r.tunOptions.AutoRedirectInputMark, + outputMark: r.tunOptions.AutoRedirectOutputMark, + resetMark: r.tunOptions.AutoRedirectResetMark, + tproxyMark: tproxyMark, + markMask: markMask, + // A queue verdict leaves the whole iptables mangle hook, while nftables base chains + // continue on NF_ACCEPT. + repeatOnAccept: !r.useNFTables, + } + err := runInNetworkNamespace(r.tunOptions.NetNs, handler.Start) + if err != nil { + return err + } + r.nfqueueHandler = handler + return nil +} + +func (r *autoRedirect) updateIPTablesNetwork() error { + r.iptablesAccess.Lock() + defer r.iptablesAccess.Unlock() + err := r.updateIPTablesLocalAddresses() + if err != nil { + err = E.Cause(err, "update local address chains") + } + routeErr := r.updateRedirectRoutes() + if routeErr != nil { + routeErr = E.Cause(routeErr, "update redirect routes") + } + var vpnServiceErr error + var bypassErr error + if r.androidVPNService { + vpnServiceErr = r.updateAndroidVPNServiceRules() + if vpnServiceErr != nil { + vpnServiceErr = E.Cause(vpnServiceErr, "update android vpn service rules") + } + bypassErr = r.updateBypassRoute() + if bypassErr != nil { + bypassErr = E.Cause(bypassErr, "update bypass route") + } + } + return E.Errors(err, routeErr, vpnServiceErr, bypassErr) +} + +func (r *autoRedirect) Close() error { if r.useNFTables { _ = runInNetworkNamespace(r.tunOptions.NetNs, func() error { r.cleanupNFTables() @@ -237,22 +387,27 @@ func (r *autoRedirect) Close() error { }) if r.ownedNetworkMonitor { _ = r.networkMonitor.Close() + r.ownedNetworkMonitor = false } } else { + r.unregisterNetworkCallback() r.cleanupIPTables() - } - return common.Close( - common.PtrOrNil(r.redirectServer), - ) -} - -func (r *autoRedirect) UpdateRouteAddressSet() { - if r.useNFTables { - err := runInNetworkNamespace(r.tunOptions.NetNs, r.nftablesUpdateRouteAddressSet) - if err != nil { - r.logger.Error("update route address set: ", err) + r.cleanupRedirectRoutes() + if r.androidVPNService { + r.cleanupBypassRoute() + r.cleanupAndroidVPNServiceRules() } } + if r.nfqueueHandler != nil { + r.nfqueueHandler.Close() + r.nfqueueHandler = nil + } + if r.redirectServer != nil { + err := r.redirectServer.Close() + r.redirectServer = nil + return err + } + return nil } func (r *autoRedirect) initializeNFTables() error { @@ -273,21 +428,43 @@ func (r *autoRedirect) redirectPort() uint16 { if r.customRedirectPort > 0 { return uint16(r.customRedirectPort) } - return M.AddrPortFromNet(r.redirectServer.listener.Addr()).Port() + return r.redirectServer.Port() } -func (r *autoRedirect) effectiveOutputMark() uint32 { - if r.tunOptions.AutoRedirectOutputMark != 0 { - return r.tunOptions.AutoRedirectOutputMark - } - return DefaultAutoRedirectOutputMark +// netd's Fwmark owns bits 0-19 (netId, explicitlySelected, protectedFromVpn, +// permission) and bit 20 (uidBillingDone, written by the bw_* chains); bits +// 29-30 are the vendor field and bit 31 is ingress_cpu_wakeup. The mark is also +// mirrored into the connmark, where netd's StrictController owns bits 24-25. +const androidReservedMarkMask = 0xE31FFFFF + +// netd (server/RouteController.cpp) identifies a network's route table as +// ifindex + ROUTE_TABLE_OFFSET_FROM_INDEX, selects VPN traffic with the +// secure (fwmark 0x0/0x20000) and bypassable (fwmark 0x0/0x30000) uid range +// rules, the local network with a rule matching fwmark 0x0/0x10000, and the +// default network with a rule matching fwmark 0x0/0xffff without a uid range. +const ( + androidRouteTableOffsetFromIndex = 1000 + androidExplicitSelectMask = 0x10000 + androidSecureVPNMask = 0x20000 + androidBypassableVPNMask = 0x30000 + androidNetIDMask = 0xFFFF +) + +func (r *autoRedirect) effectiveMarkMask() uint32 { + return r.tunOptions.AutoRedirectInputMark | r.tunOptions.AutoRedirectOutputMark | r.tunOptions.AutoRedirectResetMark | r.tunOptions.AutoRedirectTProxyMark } -func (r *autoRedirect) effectiveResetMark() uint32 { - if r.tunOptions.AutoRedirectResetMark != 0 { - return r.tunOptions.AutoRedirectResetMark +func effectiveMark(value uint32, defaultValue uint32, androidDefaultValue uint32) uint32 { + if runtime.GOOS == "android" { + if value != 0 && value&androidReservedMarkMask == 0 { + return value + } + return androidDefaultValue + } + if value != 0 { + return value } - return DefaultAutoRedirectResetMark + return defaultValue } func (r *autoRedirect) effectiveNFQueue() uint16 { diff --git a/redirect_nftables.go b/redirect_nftables.go index 70e21466..b491a903 100644 --- a/redirect_nftables.go +++ b/redirect_nftables.go @@ -30,11 +30,6 @@ func (r *autoRedirect) setupNFTables() error { Family: nftables.TableFamilyINet, }) - err = r.nftablesCreateAddressSets(nft, table, false) - if err != nil { - return E.Cause(err, "create address sets") - } - err = r.interfaceFinder.Update() if err != nil { return E.Cause(err, "update interfaces") @@ -54,18 +49,13 @@ func (r *autoRedirect) setupNFTables() error { return E.Cause(err, "create loopback address sets") } - if r.nfqueueEnabled { - err = r.nftablesCreatePreMatchChains(nft, table) - if err != nil { - return E.Cause(err, "create pre-match chains") - } + err = r.nftablesCreatePreMatchChains(nft, table) + if err != nil { + return E.Cause(err, "create pre-match chains") } if !r.shouldSkipOutputChain() { - outputNATPriority := nftables.ChainPriorityMangle - if r.nfqueueEnabled { - outputNATPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityMangle + 2) - } + outputNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityMangle + 2) chainOutput := nft.AddChain(&nftables.Chain{ Name: "output", Table: table, @@ -135,12 +125,8 @@ func (r *autoRedirect) setupNFTables() error { r.nftablesCreateRedirectPortReject(nft, table, chainInput) } - preroutingNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 1) - preroutingRoutePriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) - if r.nfqueueEnabled { - preroutingNATPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) - preroutingRoutePriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 3) - } + preroutingNATPriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 2) + preroutingRoutePriority := nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest + 3) chainPreRouting := nft.AddChain(&nftables.Chain{ Name: "prerouting", Table: table, @@ -231,6 +217,7 @@ func (r *autoRedirect) setupNFTables() error { }, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chainPreRoutingUDP) nft.AddRule(&nftables.Rule{ Table: table, Chain: chainPreRoutingUDP, @@ -313,7 +300,7 @@ func (r *autoRedirect) setupNFTables() error { } } - r.networkListener = r.networkMonitor.RegisterCallback(func() { + r.registerNetworkCallback(func() { updateErr := runInNetworkNamespace(r.tunOptions.NetNs, r.updateNetworkAddresses) if updateErr != nil { r.logger.Error(updateErr) @@ -372,28 +359,8 @@ func (r *autoRedirect) nftablesUpdateLocalAddressSet() error { return nft.Flush() } -func (r *autoRedirect) nftablesUpdateRouteAddressSet() error { - nft, err := nftables.New() - if err != nil { - return E.Cause(err, "create nftables connection") - } - defer nft.CloseLasting() - table, err := nft.ListTableOfFamily(r.tableName, nftables.TableFamilyINet) - if err != nil { - return E.Cause(err, "list nftables table") - } - err = r.nftablesCreateAddressSets(nft, table, true) - if err != nil { - return E.Cause(err, "create address sets") - } - return nft.Flush() -} - func (r *autoRedirect) cleanupNFTables() { - if r.networkListener != nil { - r.networkMonitor.UnregisterCallback(r.networkListener) - r.networkListener = nil - } + r.unregisterNetworkCallback() r.stopDockerFirewallMonitor() nft, err := nftables.New() if err != nil { @@ -458,6 +425,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Verdict{Kind: expr.VerdictReturn}, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chain) } nft.AddRule(&nftables.Rule{ Table: table, @@ -468,7 +436,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Verdict{Kind: expr.VerdictReturn}, }, }) - for _, mark := range []uint32{r.effectiveOutputMark(), r.tunOptions.AutoRedirectInputMark} { + for _, mark := range []uint32{r.tunOptions.AutoRedirectOutputMark, r.tunOptions.AutoRedirectInputMark} { nft.AddRule(&nftables.Rule{ Table: table, Chain: chain, @@ -574,7 +542,7 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, &expr.Meta{Key: expr.MetaKeyMARK, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(r.effectiveResetMark())}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.NativeEndian.PutUint32(r.tunOptions.AutoRedirectResetMark)}, &expr.Counter{}, &expr.Reject{Type: unix.NFT_REJECT_TCP_RST}, }, @@ -641,3 +609,17 @@ func (r *autoRedirect) nftablesAddPreMatchRules(nft *nftables.Conn, table *nftab } return nil } + +func (r *autoRedirect) nftablesAddBridgeIngressReturn(nft *nftables.Conn, table *nftables.Table, chain *nftables.Chain) { + for _, name := range r.tunOptions.BridgeInterface { + nft.AddRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: nftablesIfname(name)}, + &expr.Verdict{Kind: expr.VerdictReturn}, + }, + }) + } +} diff --git a/redirect_nftables_rules.go b/redirect_nftables_rules.go index 40c3e0fe..69fce8af 100644 --- a/redirect_nftables_rules.go +++ b/redirect_nftables_rules.go @@ -26,48 +26,6 @@ func init() { allocSetID = 6 } -func (r *autoRedirect) nftablesCreateAddressSets( - nft *nftables.Conn, table *nftables.Table, - update bool, -) error { - routeAddressSet := *r.routeAddressSet - routeExcludeAddressSet := *r.routeExcludeAddressSet - if len(routeAddressSet) == 0 && len(routeExcludeAddressSet) == 0 { - return nil - } - - if len(routeAddressSet) > 0 { - if r.enableIPv4 { - _, err := nftablesCreateIPSet(nft, table, 1, "inet4_route_address_set", nftables.TableFamilyIPv4, routeAddressSet, nil, true, update) - if err != nil { - return E.Cause(err, "create ipv4 route address set") - } - } - if r.enableIPv6 { - _, err := nftablesCreateIPSet(nft, table, 2, "inet6_route_address_set", nftables.TableFamilyIPv6, routeAddressSet, nil, true, update) - if err != nil { - return E.Cause(err, "create ipv6 route address set") - } - } - } - - if len(routeExcludeAddressSet) > 0 { - if r.enableIPv4 { - _, err := nftablesCreateIPSet(nft, table, 3, "inet4_route_exclude_address_set", nftables.TableFamilyIPv4, routeExcludeAddressSet, nil, false, update) - if err != nil { - return E.Cause(err, "create ipv4 route exclude address set") - } - } - if r.enableIPv6 { - _, err := nftablesCreateIPSet(nft, table, 4, "inet6_route_exclude_address_set", nftables.TableFamilyIPv6, routeExcludeAddressSet, nil, false, update) - if err != nil { - return E.Cause(err, "create ipv6 route exclude address set") - } - } - } - return nil -} - func (r *autoRedirect) nftablesCreateLocalAddressSets( nft *nftables.Conn, table *nftables.Table, localAddresses []netip.Prefix, lastAddresses []netip.Prefix, @@ -209,8 +167,8 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }) } } - if r.nfqueueEnabled && chain.Type == nftables.ChainTypeNAT { - for _, mark := range []uint32{r.effectiveOutputMark(), r.tunOptions.AutoRedirectInputMark} { + if chain.Type == nftables.ChainTypeNAT { + for _, mark := range []uint32{r.tunOptions.AutoRedirectOutputMark, r.tunOptions.AutoRedirectInputMark} { nft.AddRule(&nftables.Rule{ Table: table, Chain: chain, @@ -223,7 +181,6 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }) } } - if chain.Hooknum == nftables.ChainHookPrerouting { nft.AddRule(&nftables.Rule{ Table: table, @@ -244,6 +201,7 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }, }, }) + r.nftablesAddBridgeIngressReturn(nft, table, chain) if len(r.tunOptions.IncludeInterface) > 0 { if len(r.tunOptions.IncludeInterface) > 1 { includeInterface := &nftables.Set{ @@ -673,25 +631,6 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft nftablesCreateExcludeDestinationIPSet(nft, table, chain, 6, "inet6_local_address_set", nftables.TableFamilyIPv6, false) } - routeAddressSet := *r.routeAddressSet - routeExcludeAddressSet := *r.routeExcludeAddressSet - - if r.enableIPv4 && len(routeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 1, "inet4_route_address_set", nftables.TableFamilyIPv4, true) - } - - if r.enableIPv6 && len(routeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 2, "inet6_route_address_set", nftables.TableFamilyIPv6, true) - } - - if r.enableIPv4 && len(routeExcludeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 3, "inet4_route_exclude_address_set", nftables.TableFamilyIPv4, false) - } - - if r.enableIPv6 && len(routeExcludeAddressSet) > 0 { - nftablesCreateExcludeDestinationIPSet(nft, table, chain, 4, "inet6_route_exclude_address_set", nftables.TableFamilyIPv6, false) - } - if chain.Type == nftables.ChainTypeNAT || r.tunOptions.ExcludeMPTCP { mptcpVerdict := expr.VerdictDrop if r.tunOptions.ExcludeMPTCP { @@ -712,7 +651,7 @@ func (r *autoRedirect) nftablesCreateExcludeRules(nft *nftables.Conn, table *nft }, &expr.Exthdr{ DestRegister: 1, - Type: 30, + Type: tcpOptionMultipathTCP, Offset: 0, Len: 1, Flags: unix.NFT_EXTHDR_F_PRESENT, @@ -1052,9 +991,9 @@ func (r *autoRedirect) nftablesCreateDNSHijackRulesForFamily( }, } if chain.Hooknum == nftables.ChainHookOutput { - // It looks like we can't hijack DNS requests sent to loopback. - // https://serverfault.com/questions/363899/iptables-dnat-from-loopback - // and tproxy is not available in output + // The DNAT target still runs for loopback-destined packets, but the + // reroute that follows it refuses a loopback source on a non-loopback + // device and drops the packet. exprs = append(exprs, &expr.Meta{ Key: expr.MetaKeyOIFNAME, diff --git a/redirect_route_bypass_android.go b/redirect_route_bypass_android.go new file mode 100644 index 00000000..5dd79b03 --- /dev/null +++ b/redirect_route_bypass_android.go @@ -0,0 +1,343 @@ +package tun + +import ( + "net" + "net/netip" + "slices" + + "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" + + "go4.org/netipx" + "golang.org/x/sys/unix" +) + +func (r *autoRedirect) setupBypassRoute() { + if r.routeAddressSet == nil { + return + } + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + r.cleanupStaleRouteRules(r.androidVPNServiceRulePriority(), 0) + r.bypassRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.redirectRouteTableIndex, r.tproxyRouteTableIndex) + err := r.refreshBypassAddressesLocked() + if err == nil { + err = r.rebuildBypassRouteLocked() + } + if err != nil { + r.cleanupBypassRouteLocked() + r.logger.Warn("address set destinations keep their pre-match bypass: setup bypass route: ", err) + return + } + r.bypassRouteActive = true +} + +func (r *autoRedirect) updateBypassRoute() error { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + if !r.bypassRouteActive { + return nil + } + return r.rebuildBypassRouteLocked() +} + +func (r *autoRedirect) UpdateRouteAddressSet() error { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + if !r.bypassRouteActive { + return nil + } + err := r.refreshBypassAddressesLocked() + if err != nil { + return err + } + return r.rebuildBypassRouteLocked() +} + +func (r *autoRedirect) refreshBypassAddressesLocked() error { + include, exclude, err := r.routeAddressSet() + if err != nil { + return E.Cause(err, "fetch route address set") + } + r.bypassIncludeAddresses = include + r.bypassExcludeAddresses = exclude + return nil +} + +func (r *autoRedirect) cleanupBypassRoute() { + r.bypassRouteAccess.Lock() + defer r.bypassRouteAccess.Unlock() + r.bypassRouteActive = false + r.cleanupBypassRouteLocked() +} + +func (r *autoRedirect) cleanupBypassRouteLocked() { + if r.bypassRouteTableIndex == 0 { + return + } + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + _ = netlink.RuleDel(r.bypassRouteRule(family)) + } + flushRouteTable(r.bypassRouteTableIndex) +} + +func (r *autoRedirect) rebuildBypassRouteLocked() error { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + if family == unix.AF_INET && !r.enableIPv4 || family == unix.AF_INET6 && !r.enableIPv6 { + continue + } + err := r.rebuildBypassRouteFamilyLocked(family) + if err != nil { + return err + } + } + return nil +} + +func (r *autoRedirect) rebuildBypassRouteFamilyLocked(family int) error { + tunPrefixes := r.tunOptions.Inet4Address + if family == unix.AF_INET6 { + tunPrefixes = r.tunOptions.Inet6Address + } + set, err := bypassAddressSet(family, r.bypassIncludeAddresses, r.bypassExcludeAddresses, tunPrefixes) + if err != nil { + return E.Cause(err, "build bypass address set") + } + rule := r.bypassRouteRule(family) + if len(set.Ranges()) == 0 { + _ = netlink.RuleDel(rule) + flushRouteTableFamily(r.bypassRouteTableIndex, family) + return nil + } + ruleList, err := netlink.RuleList(family) + if err != nil { + return E.Cause(err, "list rules") + } + sources := r.bypassRouteSources(family, ruleList) + // netd rebuilds its rules and tables asynchronously after a network + // change, so an update can observe them half-built; the repopulated + // tables generate route events that schedule another rebuild. + if len(sources) == 0 { + r.logger.Debug("bypass route family ", family, ": no source tables, keeping current routes") + return nil + } + routes, err := buildBypassRoutes(family, set, sources, r.bypassRouteTableIndex) + if err != nil { + return E.Cause(err, "build bypass routes") + } + if len(routes) == 0 { + r.logger.Debug("bypass route family ", family, ": no routes built from ", len(sources), " sources, keeping current routes") + return nil + } + changed, err := r.reconcileBypassRoutesLocked(family, routes) + if err != nil { + return err + } + if changed > 0 { + r.logger.Debug("bypass route family ", family, ": ", changed, " of ", len(routes), " routes changed, from ", len(sources), " sources") + } + if !slices.ContainsFunc(ruleList, func(it netlink.Rule) bool { + return it.Table == r.bypassRouteTableIndex + }) { + err = netlink.RuleAdd(rule) + if err != nil { + return E.Cause(err, "add bypass rule") + } + } + return nil +} + +func (r *autoRedirect) reconcileBypassRoutesLocked(family int, desired []netlink.Route) (int, error) { + current, err := netlink.RouteListFiltered(family, &netlink.Route{Table: r.bypassRouteTableIndex}, netlink.RT_FILTER_TABLE) + if err != nil { + return 0, E.Cause(err, "list bypass routes") + } + desiredByDestination := make(map[string]*netlink.Route, len(desired)) + for index := range desired { + desiredByDestination[desired[index].Dst.String()] = &desired[index] + } + matched := make(map[string]bool, len(desired)) + var changed int + for index := range current { + currentRoute := ¤t[index] + if currentRoute.Dst == nil { + currentRoute.Dst = defaultDestination(family) + } + destination := currentRoute.Dst.String() + desiredRoute, exists := desiredByDestination[destination] + if exists && bypassRouteEquals(currentRoute, desiredRoute) { + matched[destination] = true + continue + } + // RouteReplace only overwrites a route with the same metric; a + // current route with another metric would survive alongside it. + if !exists || currentRoute.Priority != desiredRoute.Priority { + _ = netlink.RouteDel(currentRoute) + changed++ + } + } + for index := range desired { + desiredRoute := &desired[index] + if matched[desiredRoute.Dst.String()] { + continue + } + err = netlink.RouteReplace(desiredRoute) + if err != nil { + return changed, E.Cause(err, "add bypass route ", desiredRoute.Dst) + } + changed++ + } + return changed, nil +} + +func bypassRouteEquals(left *netlink.Route, right *netlink.Route) bool { + return left.LinkIndex == right.LinkIndex && + left.Scope == right.Scope && + left.Priority == right.Priority && + left.Gw.Equal(right.Gw) && + routeSourceEquals(left.Src, right.Src) +} + +func routeSourceEquals(left *net.IPNet, right *net.IPNet) bool { + if left == nil || right == nil { + return left == right + } + return left.IP.Equal(right.IP) && slices.Equal(left.Mask, right.Mask) +} + +func (r *autoRedirect) bypassRouteRule(family int) *netlink.Rule { + rule := netlink.NewRule() + rule.Priority = r.androidVPNServiceRulePriority() + rule.Family = family + rule.Table = r.bypassRouteTableIndex + if r.tunOptions.DNSModeOrDefault() == DNSModeHijack { + rule.Invert = true + rule.Dport = netlink.NewRulePortRange(53, 53) + } + return rule +} + +func (r *autoRedirect) bypassRouteSources(family int, ruleList []netlink.Rule) []netlink.Route { + tables := discoverAndroidVPNServiceTables(ruleList, r.tunOptions.Name) + var sourceTables []int + if tables.localNetwork > 0 { + sourceTables = append(sourceTables, tables.localNetwork) + } + sourceTables = append(sourceTables, tables.physical...) + var specificRoutes []netlink.Route + var defaultRoutes []netlink.Route + for _, sourceTable := range sourceTables { + routes, listErr := netlink.RouteListFiltered(family, &netlink.Route{Table: sourceTable}, netlink.RT_FILTER_TABLE) + if listErr != nil { + continue + } + for _, route := range routes { + if route.Type != 0 && route.Type != unix.RTN_UNICAST { + continue + } + if routeDestination(family, route.Dst).Bits() == 0 { + defaultRoutes = append(defaultRoutes, route) + } else { + specificRoutes = append(specificRoutes, route) + } + } + } + slices.SortStableFunc(specificRoutes, func(left netlink.Route, right netlink.Route) int { + return routeDestination(family, right.Dst).Bits() - routeDestination(family, left.Dst).Bits() + }) + return append(specificRoutes, defaultRoutes...) +} + +func buildBypassRoutes(family int, set *netipx.IPSet, sources []netlink.Route, tableIndex int) ([]netlink.Route, error) { + remaining := set + var routes []netlink.Route + for _, source := range sources { + if len(remaining.Ranges()) == 0 { + break + } + destination := routeDestination(family, source.Dst) + if !destination.IsValid() { + continue + } + part, err := intersectPrefix(remaining, destination) + if err != nil { + return nil, err + } + if len(part.Ranges()) == 0 { + continue + } + for _, prefix := range part.Prefixes() { + route := source + route.Dst = prefixToIPNet(prefix) + route.Table = tableIndex + // NetworkStack IpClientLinkObserver imports every kernel and ra + // route on the interface into LinkProperties regardless of table. + route.Protocol = unix.RTPROT_STATIC + routes = append(routes, route) + } + var remainingBuilder netipx.IPSetBuilder + remainingBuilder.AddSet(remaining) + remainingBuilder.RemoveSet(part) + remaining, err = remainingBuilder.IPSet() + if err != nil { + return nil, err + } + } + return routes, nil +} + +func bypassAddressSet(family int, include []netip.Prefix, exclude []netip.Prefix, tunPrefixes []netip.Prefix) (*netipx.IPSet, error) { + var builder netipx.IPSetBuilder + for _, prefix := range exclude { + if prefixMatchesFamily(family, prefix) { + builder.AddPrefix(prefix) + } + } + if len(include) > 0 { + builder.AddPrefix(unspecifiedPrefix(family)) + for _, prefix := range include { + if prefixMatchesFamily(family, prefix) { + builder.RemovePrefix(prefix) + } + } + } + for _, prefix := range tunPrefixes { + builder.RemovePrefix(prefix.Masked()) + } + return builder.IPSet() +} + +func prefixMatchesFamily(family int, prefix netip.Prefix) bool { + return (family == unix.AF_INET) == prefix.Addr().Is4() +} + +func unspecifiedPrefix(family int) netip.Prefix { + if family == unix.AF_INET { + return netip.PrefixFrom(netip.IPv4Unspecified(), 0) + } + return netip.PrefixFrom(netip.IPv6Unspecified(), 0) +} + +func intersectPrefix(set *netipx.IPSet, prefix netip.Prefix) (*netipx.IPSet, error) { + var prefixBuilder netipx.IPSetBuilder + prefixBuilder.AddPrefix(prefix) + prefixSet, err := prefixBuilder.IPSet() + if err != nil { + return nil, err + } + var builder netipx.IPSetBuilder + builder.AddSet(set) + builder.Intersect(prefixSet) + return builder.IPSet() +} + +func routeDestination(family int, destination *net.IPNet) netip.Prefix { + if destination == nil { + return unspecifiedPrefix(family) + } + prefix, valid := netipx.FromStdIPNet(destination) + if !valid { + return netip.Prefix{} + } + return prefix +} diff --git a/redirect_route_linux.go b/redirect_route_linux.go index f73df11a..a55f0a71 100644 --- a/redirect_route_linux.go +++ b/redirect_route_linux.go @@ -6,6 +6,7 @@ import ( "errors" "math/rand" "net" + "slices" "github.com/sagernet/netlink" "github.com/sagernet/sing/common" @@ -18,18 +19,6 @@ import ( const redirectRouteRulePriority = 1 func (r *autoRedirect) setupRedirectRoutes() error { - for { - r.redirectRouteTableIndex = int(rand.Uint32()) - if r.redirectRouteTableIndex == r.tunOptions.IPRoute2TableIndex { - continue - } - routeList, fErr := netlink.RouteListFiltered(netlink.FAMILY_ALL, - &netlink.Route{Table: r.redirectRouteTableIndex}, - netlink.RT_FILTER_TABLE) - if len(routeList) == 0 || fErr != nil { - break - } - } err := r.interfaceFinder.Update() if err != nil { return E.Cause(err, "update interfaces") @@ -39,6 +28,11 @@ func (r *autoRedirect) setupRedirectRoutes() error { defer r.redirectRouteAccess.Unlock() r.redirectRoutesActive = false r.cleanupRedirectRoutesLocked() + r.cleanupStaleRouteRules(redirectRouteRulePriority, 0) + if r.tproxyEnabled() { + r.cleanupStaleRouteRules(redirectRouteRulePriority, r.tunOptions.AutoRedirectTProxyMark) + } + r.redirectRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.tproxyRouteTableIndex) defer func() { if err != nil { r.cleanupRedirectRoutesLocked() @@ -68,10 +62,52 @@ func (r *autoRedirect) setupRedirectRoutes() error { return E.Cause(err, "add ipv6 redirect rule") } } + if r.tproxyEnabled() { + err = r.setupTProxyRouteLocked() + if err != nil { + return err + } + } r.redirectRoutesActive = true return nil } +func (r *autoRedirect) setupTProxyRouteLocked() error { + r.tproxyRouteTableIndex = chooseRouteTableIndex(r.tunOptions.IPRoute2TableIndex, r.redirectRouteTableIndex) + loopbackLink, err := netlink.LinkByName("lo") + if err != nil { + return E.Cause(err, "find loopback interface") + } + route := netlink.Route{ + LinkIndex: loopbackLink.Attrs().Index, + Dst: defaultDestination(unix.AF_INET6), + Table: r.tproxyRouteTableIndex, + Type: unix.RTN_LOCAL, + Scope: netlink.SCOPE_HOST, + } + err = netlink.RouteAdd(&route) + if err != nil { + return E.Cause(err, "add tproxy route") + } + rule := r.tproxyRouteRule() + err = netlink.RuleAdd(rule) + if err != nil { + return E.Cause(err, "add tproxy rule") + } + return nil +} + +func (r *autoRedirect) tproxyRouteRule() *netlink.Rule { + rule := netlink.NewRule() + rule.Priority = redirectRouteRulePriority + rule.Family = unix.AF_INET6 + rule.Mark = r.tunOptions.AutoRedirectTProxyMark + rule.MarkSet = true + rule.Mask = int(r.effectiveMarkMask()) + rule.Table = r.tproxyRouteTableIndex + return rule +} + func (r *autoRedirect) currentRedirectInterfaces() []control.Interface { tunName := r.tunOptions.Name return common.Filter(r.interfaceFinder.Interfaces(), func(it control.Interface) bool { @@ -136,6 +172,10 @@ func (r *autoRedirect) cleanupRedirectRoutesLocked() { rule.Family = unix.AF_INET6 _ = netlink.RuleDel(rule) } + if r.tproxyRouteTableIndex != 0 { + flushRouteTable(r.tproxyRouteTableIndex) + _ = netlink.RuleDel(r.tproxyRouteRule()) + } } type redirectRouteKey struct { @@ -144,9 +184,6 @@ type redirectRouteKey struct { } func (r *autoRedirect) reconcileRedirectRoutesLocked(redirectInterfaces []control.Interface) error { - // Interface snapshots are not sufficient here: network managers can flush a - // route while a fast reconnect leaves the interface index and address families - // unchanged. Reconcile against the kernel's route table on every update. currentRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: r.redirectRouteTableIndex}, netlink.RT_FILTER_TABLE) @@ -242,3 +279,72 @@ func redirectRouteDestinationMatches(destination *net.IPNet, address net.IP, pre ones, bits := destination.Mask.Size() return ones == prefixBits && bits == prefixBits } + +func chooseRouteTableIndex(excludedIndexes ...int) int { + for { + tableIndex := int(rand.Uint32() & 0x7fffffff) + if tableIndex <= 255 || slices.Contains(excludedIndexes, tableIndex) { + continue + } + if !routeTableOccupied(tableIndex) { + return tableIndex + } + } +} + +func routeTableOccupied(tableIndex int) bool { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + routes, err := netlink.RouteListFiltered(family, + &netlink.Route{Table: tableIndex}, + netlink.RT_FILTER_TABLE) + if err == nil && len(routes) > 0 { + return true + } + } + return false +} + +func flushRouteTable(tableIndex int) { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + flushRouteTableFamily(tableIndex, family) + } +} + +func flushRouteTableFamily(tableIndex int, family int) { + routes, _ := netlink.RouteListFiltered(family, + &netlink.Route{Table: tableIndex}, + netlink.RT_FILTER_TABLE) + for index := range routes { + route := &routes[index] + if route.Dst == nil { + route.Dst = defaultDestination(family) + } + _ = netlink.RouteDel(route) + } +} + +func (r *autoRedirect) cleanupStaleRouteRules(priority int, mark uint32) { + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + rules, err := netlink.RuleListFiltered(family, + &netlink.Rule{Priority: priority}, + netlink.RT_FILTER_PRIORITY) + if err != nil { + continue + } + for index := range rules { + staleRule := &rules[index] + if staleRule.Mark != mark || staleRule.Table <= 255 { + continue + } + flushRouteTable(staleRule.Table) + _ = netlink.RuleDel(staleRule) + } + } +} + +func defaultDestination(family int) *net.IPNet { + if family == unix.AF_INET6 { + return &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)} + } + return &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)} +} diff --git a/redirect_route_vpnservice_android.go b/redirect_route_vpnservice_android.go new file mode 100644 index 00000000..af2803a6 --- /dev/null +++ b/redirect_route_vpnservice_android.go @@ -0,0 +1,240 @@ +package tun + +import ( + "errors" + "os" + "slices" + "strconv" + "strings" + + "github.com/sagernet/netlink" + "github.com/sagernet/sing/common" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +func (r *autoRedirect) androidVPNServiceRulePriority() int { + if r.tunOptions.IPRoute2RuleIndex != 0 { + return r.tunOptions.IPRoute2RuleIndex + } + return DefaultIPRoute2RuleIndex +} + +type androidVPNServiceTables struct { + physical []int + localNetwork int +} + +// netd adds one default network rule per interface of the default network +// (with 464xlat: the base interface and the clat interface, only the latter +// holding the IPv4 default route), so every table is kept in dump order and +// the kernel falls through on a lookup miss. netd reaches the local network +// table through the rule matching the VPN as the incoming interface; the +// explicit select mask instead matches the legacy system and legacy network +// tables first, which stay empty. +func discoverAndroidVPNServiceTables(ruleList []netlink.Rule, tunInterfaceName string) androidVPNServiceTables { + var tables androidVPNServiceTables + for _, rule := range ruleList { + if rule.Table > 0 && rule.UIDRange == nil && rule.Mark == 0 && rule.Mask == androidNetIDMask && !slices.Contains(tables.physical, rule.Table) { + tables.physical = append(tables.physical, rule.Table) + } + if rule.Table > 0 && rule.Mark == 0 && rule.UIDRange == nil && rule.IifName == tunInterfaceName && rule.OifName == "" && tables.localNetwork == 0 { + tables.localNetwork = rule.Table + } + } + return tables +} + +func (r *autoRedirect) discoverAndroidVPNServiceRules() ([]*netlink.Rule, error) { + priority := r.androidVPNServiceRulePriority() + mask := int(r.effectiveMarkMask()) + tunLink, err := netlink.LinkByName(r.tunOptions.Name) + if err != nil { + return nil, E.Cause(err, "find tun interface") + } + vpnTable := tunLink.Attrs().Index + androidRouteTableOffsetFromIndex + var ( + rules []*netlink.Rule + ruleList []netlink.Rule + ) + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + familyName := "ipv4" + if family == unix.AF_INET6 { + if !r.enableIPv6 { + continue + } + familyName = "ipv6" + } else if !r.enableIPv4 { + continue + } + ruleList, err = netlink.RuleList(family) + if err != nil { + return nil, E.Cause(err, "list ", familyName, " rules") + } + tables := discoverAndroidVPNServiceTables(ruleList, r.tunOptions.Name) + physicalTables := tables.physical + localNetworkTable := tables.localNetwork + r.logger.Debug(familyName, " vpn service rules: vpn table ", vpnTable, ", default network tables ", strings.Join(common.Map(physicalTables, strconv.Itoa), " "), ", local network table ", localNetworkTable) + inputRule := netlink.NewRule() + inputRule.Priority = priority + inputRule.Family = family + inputRule.Mark = r.tunOptions.AutoRedirectInputMark + inputRule.MarkSet = true + inputRule.Mask = mask + inputRule.Table = vpnTable + rules = append(rules, inputRule) + // A reset generated by the REJECT chain keeps the reset mark of the + // packet that triggered it and has to reach the client on the served + // network instead of the netd VPN rule that catches unmarked flows. + if localNetworkTable > 0 { + resetRule := netlink.NewRule() + resetRule.Priority = priority + resetRule.Family = family + resetRule.Mark = r.tunOptions.AutoRedirectResetMark + resetRule.MarkSet = true + resetRule.Mask = mask + resetRule.Table = localNetworkTable + rules = append(rules, resetRule) + } + // Bypassed local flows carry the tun address as their source and + // Android ships no ip6tables nat table to rewrite it. + if family == unix.AF_INET6 { + continue + } + if localNetworkTable > 0 { + localOutputRule := netlink.NewRule() + localOutputRule.Priority = priority + localOutputRule.Family = family + localOutputRule.Mark = r.tunOptions.AutoRedirectOutputMark + localOutputRule.MarkSet = true + localOutputRule.Mask = mask + localOutputRule.IifName = "lo" + localOutputRule.Table = localNetworkTable + rules = append(rules, localOutputRule) + } + for _, physicalTable := range physicalTables { + outputRule := netlink.NewRule() + outputRule.Priority = priority + outputRule.Family = family + outputRule.Mark = r.tunOptions.AutoRedirectOutputMark + outputRule.MarkSet = true + outputRule.Mask = mask + outputRule.IifName = "lo" + outputRule.Table = physicalTable + rules = append(rules, outputRule) + } + } + return rules, nil +} + +func androidVPNServiceRuleEquals(left *netlink.Rule, right *netlink.Rule) bool { + return left.Family == right.Family && + left.Priority == right.Priority && + left.Mark == right.Mark && + left.Mask == right.Mask && + left.IifName == right.IifName && + left.Table == right.Table +} + +func (r *autoRedirect) setupAndroidVPNServiceRules() error { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + priority := r.androidVPNServiceRulePriority() + inputMark := r.tunOptions.AutoRedirectInputMark + outputMark := r.tunOptions.AutoRedirectOutputMark + resetMark := r.tunOptions.AutoRedirectResetMark + for _, family := range []int{unix.AF_INET, unix.AF_INET6} { + staleRules, err := netlink.RuleListFiltered(family, &netlink.Rule{Priority: priority}, netlink.RT_FILTER_PRIORITY) + if err != nil { + continue + } + for _, staleRule := range staleRules { + if staleRule.Mark != inputMark && staleRule.Mark != outputMark && staleRule.Mark != resetMark { + continue + } + deleteRule := netlink.NewRule() + deleteRule.Family = family + deleteRule.Priority = priority + deleteRule.Mark = staleRule.Mark + deleteRule.MarkSet = true + deleteRule.Mask = staleRule.Mask + _ = netlink.RuleDel(deleteRule) + } + } + rules, err := r.discoverAndroidVPNServiceRules() + if err != nil { + return err + } + r.androidVPNServiceRulesActive = true + for _, rule := range rules { + err = netlink.RuleAdd(rule) + if err != nil && !E.IsMulti(err, unix.EEXIST, os.ErrExist) { + return E.Cause(err, "add rule ", rule) + } + r.androidVPNServiceRules = append(r.androidVPNServiceRules, rule) + } + return r.removeAndroidIngressDiscardRulesLocked() +} + +func (r *autoRedirect) updateAndroidVPNServiceRules() error { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + if !r.androidVPNServiceRulesActive { + return nil + } + rules, err := r.discoverAndroidVPNServiceRules() + if err != nil { + var linkNotFound netlink.LinkNotFoundError + if errors.As(err, &linkNotFound) { + return nil + } + return err + } + currentRules, err := netlink.RuleListFiltered(netlink.FAMILY_ALL, &netlink.Rule{Priority: r.androidVPNServiceRulePriority()}, netlink.RT_FILTER_PRIORITY) + if err != nil { + return E.Cause(err, "list current vpn service rules") + } + var errorList []error + var installedRules []*netlink.Rule + for _, installedRule := range r.androidVPNServiceRules { + if slices.ContainsFunc(rules, func(it *netlink.Rule) bool { + return androidVPNServiceRuleEquals(it, installedRule) + }) { + continue + } + deleteErr := netlink.RuleDel(installedRule) + if deleteErr != nil && !E.IsMulti(deleteErr, unix.ENOENT, os.ErrNotExist) { + errorList = append(errorList, E.Cause(deleteErr, "delete rule ", installedRule)) + installedRules = append(installedRules, installedRule) + } + } + for _, rule := range rules { + if slices.ContainsFunc(currentRules, func(it netlink.Rule) bool { + return androidVPNServiceRuleEquals(&it, rule) + }) { + installedRules = append(installedRules, rule) + continue + } + addErr := netlink.RuleAdd(rule) + if addErr != nil && !E.IsMulti(addErr, unix.EEXIST, os.ErrExist) { + errorList = append(errorList, E.Cause(addErr, "add rule ", rule)) + continue + } + installedRules = append(installedRules, rule) + } + r.androidVPNServiceRules = installedRules + errorList = append(errorList, r.removeAndroidIngressDiscardRulesLocked()) + return E.Errors(errorList...) +} + +func (r *autoRedirect) cleanupAndroidVPNServiceRules() { + r.androidVPNServiceRuleAccess.Lock() + defer r.androidVPNServiceRuleAccess.Unlock() + r.androidVPNServiceRulesActive = false + r.restoreAndroidIngressDiscardRulesLocked() + for _, rule := range r.androidVPNServiceRules { + _ = netlink.RuleDel(rule) + } + r.androidVPNServiceRules = nil +} diff --git a/redirect_route_vpnservice_stub.go b/redirect_route_vpnservice_stub.go new file mode 100644 index 00000000..45fdc608 --- /dev/null +++ b/redirect_route_vpnservice_stub.go @@ -0,0 +1,28 @@ +//go:build linux && !android + +package tun + +func (r *autoRedirect) setupAndroidVPNServiceRules() error { + return nil +} + +func (r *autoRedirect) updateAndroidVPNServiceRules() error { + return nil +} + +func (r *autoRedirect) cleanupAndroidVPNServiceRules() { +} + +func (r *autoRedirect) setupBypassRoute() { +} + +func (r *autoRedirect) updateBypassRoute() error { + return nil +} + +func (r *autoRedirect) cleanupBypassRoute() { +} + +func (r *autoRedirect) UpdateRouteAddressSet() error { + return nil +} diff --git a/redirect_server.go b/redirect_server.go index 1f138ec4..e11dffbd 100644 --- a/redirect_server.go +++ b/redirect_server.go @@ -8,6 +8,7 @@ import ( "net" "net/netip" "sync/atomic" + "syscall" "time" "github.com/sagernet/sing/common/control" @@ -15,31 +16,58 @@ import ( "github.com/sagernet/sing/common/logger" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" ) -type redirectServer struct { - ctx context.Context - handler N.TCPConnectionHandlerEx - logger logger.Logger - listenAddr netip.Addr - listener *net.TCPListener - inShutdown atomic.Bool +type RedirectServer struct { + ctx context.Context + handler N.TCPConnectionHandlerEx + logger logger.Logger + listenAddress netip.Addr + transparent bool + externalTransparent bool + listenMark uint32 + listener *net.TCPListener + inShutdown atomic.Bool +} + +func (s *RedirectServer) SetExternalTransparent() { + s.transparent = true + s.externalTransparent = true } -func newRedirectServer(ctx context.Context, handler N.TCPConnectionHandlerEx, logger logger.Logger, listenAddr netip.Addr) *redirectServer { - return &redirectServer{ - ctx: ctx, - handler: handler, - logger: logger, - listenAddr: listenAddr, +func (s *RedirectServer) ListenerFileDescriptor() (int, error) { + return control.Conn0(s.listener, func(fd uintptr) (int, error) { + return unix.Dup(int(fd)) + }) +} + +func NewRedirectServer(ctx context.Context, handler N.TCPConnectionHandlerEx, logger logger.Logger, listenAddress netip.Addr) *RedirectServer { + return &RedirectServer{ + ctx: ctx, + handler: handler, + logger: logger, + listenAddress: listenAddress, } } -func (s *redirectServer) Start() error { +func (s *RedirectServer) Start() error { var listenConfig net.ListenConfig // listenConfig.KeepAlive = C.TCPKeepAliveInitial listenConfig.KeepAlive = 10 * time.Minute - listener, err := listenConfig.Listen(s.ctx, M.NetworkFromNetAddr("tcp", s.listenAddr), M.SocksaddrFrom(s.listenAddr, 0).String()) + if s.transparent && !s.externalTransparent { + listenConfig.Control = control.Append(func(network string, address string, conn syscall.RawConn) error { + return control.Raw(conn, func(fd uintptr) error { + err := syscall.SetsockoptInt(int(fd), syscall.SOL_IPV6, unix.IPV6_TRANSPARENT, 1) + if err != nil { + return E.Cause(err, "set IPV6_TRANSPARENT") + } + return nil + }) + }, control.RoutingMark(s.listenMark)) + } + listener, err := listenConfig.Listen(s.ctx, M.NetworkFromNetAddr("tcp", s.listenAddress), M.SocksaddrFrom(s.listenAddress, 0).String()) if err != nil { return err } @@ -48,12 +76,16 @@ func (s *redirectServer) Start() error { return nil } -func (s *redirectServer) Close() error { +func (s *RedirectServer) Port() uint16 { + return M.AddrPortFromNet(s.listener.Addr()).Port() +} + +func (s *RedirectServer) Close() error { s.inShutdown.Store(true) return s.listener.Close() } -func (s *redirectServer) loopIn() { +func (s *RedirectServer) loopIn() { var retryDelay time.Duration for { conn, err := s.listener.AcceptTCP() @@ -82,13 +114,20 @@ func (s *redirectServer) loopIn() { } retryDelay = 0 source := M.SocksaddrFromNet(conn.RemoteAddr()).Unwrap() - destination, err := control.GetOriginalDestination(conn) - if err != nil { - _ = conn.SetLinger(0) - _ = conn.Close() - s.logger.Error("process redirect connection from ", source, ": invalid connection: ", err) - continue + var destination M.Socksaddr + if s.transparent && !source.IsIPv4() { + destination = M.SocksaddrFromNet(conn.LocalAddr()).Unwrap() + } else { + var originalDestination netip.AddrPort + originalDestination, err = control.GetOriginalDestination(conn) + if err != nil { + _ = conn.SetLinger(0) + _ = conn.Close() + s.logger.Error("process redirect connection from ", source, ": invalid connection: ", err) + continue + } + destination = M.SocksaddrFromNetIP(originalDestination).Unwrap() } - go s.handler.NewConnectionEx(s.ctx, conn, source, M.SocksaddrFromNetIP(destination).Unwrap(), nil) + go s.handler.NewConnectionEx(s.ctx, conn, source, destination, nil) } } diff --git a/stack.go b/stack.go index 613e45dc..3b1e9869 100644 --- a/stack.go +++ b/stack.go @@ -18,6 +18,14 @@ type Stack interface { Close() error } +type MemoryPressure uint8 + +const ( + MemoryPressureNone MemoryPressure = iota + MemoryPressureWarning + MemoryPressureCritical +) + type StackOptions struct { Context context.Context Tun Tun @@ -32,6 +40,8 @@ type StackOptions struct { ForwarderBindInterface bool IncludeAllNetworks bool InterfaceFinder control.InterfaceFinder + MemoryPressure func() MemoryPressure + TCPCongestionControl string } func NewStack( @@ -39,14 +49,8 @@ func NewStack( options StackOptions, ) (Stack, error) { switch stack { - case "": - if options.IncludeAllNetworks { - return NewGVisor(options) - } else if WithGVisor && !options.TunOptions.GSO { - return NewMixed(options) - } else { - return NewSystem(options) - } + case "", "go": + return NewGo(options) case "gvisor": return NewGVisor(options) case "mixed": diff --git a/stack_go.go b/stack_go.go new file mode 100644 index 00000000..6c6bf19f --- /dev/null +++ b/stack_go.go @@ -0,0 +1,315 @@ +package tun + +import ( + "context" + "errors" + "net/netip" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" +) + +type Go struct { + ctx context.Context + tun Tun + mtu int + handler Handler + logger logger.Logger + broadcastAddr netip.Addr + inet4LoopbackAddress []netip.Addr + inet6LoopbackAddress []netip.Addr + udpTimeout time.Duration + icmpTimeout time.Duration + udpNATOptions UDPNatOptions + memoryPressure func() MemoryPressure + udpNats []*UDPNat + fragmentIdentification atomic.Uint32 + dispatcher *ForwardDispatcher + directory goFlowDirectory + congestion *goCongestionOps + queueFactory func(stack *Go) ([]goPlatformIO, error) + access sync.Mutex + engines []*goEngine + closed atomic.Bool + validateChecksum bool +} + +func NewGo(options StackOptions) (*Go, error) { + congestion, err := goLookupCongestionControl(options.TCPCongestionControl) + if err != nil { + return nil, err + } + stack := &Go{ + congestion: congestion, + queueFactory: newGoPlatformQueues, + ctx: options.Context, + tun: options.Tun, + mtu: int(options.TunOptions.MTU), + handler: options.Handler, + logger: options.Logger, + broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), + inet4LoopbackAddress: options.TunOptions.Inet4LoopbackAddress, + inet6LoopbackAddress: options.TunOptions.Inet6LoopbackAddress, + udpTimeout: options.UDPTimeout, + icmpTimeout: options.ICMPTimeout, + udpNATOptions: UDPNatOptions{ + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, + }, + memoryPressure: options.MemoryPressure, + directory: goFlowDirectory{ + flows: make(map[flowKey]*GoConn), + udpSockets: make(map[netip.AddrPort]*GoUDPConn), + listeners: make(map[netip.AddrPort]*GoListener), + }, + } + memoryTun, isMemoryTun := options.Tun.(*MemoryTun) + if isMemoryTun { + stack.validateChecksum = true + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goMemoryIO{stack: current, tun: memoryTun, handed: make([]*buf.Buffer, 0, goReadBatch), blocked: make(map[*OutboundQueue]*goBlockedWriters)}}, nil + } + } + return stack, nil +} + +func (s *Go) Start() error { + s.access.Lock() + defer s.access.Unlock() + queues, err := s.queueFactory(s) + if err != nil { + return err + } + for index, queue := range queues { + err = queue.start() + if err == nil { + continue + } + if index > 0 && errors.Is(err, errGoQueueUnavailable) { + queues = queues[:index] + break + } + return E.Errors(err, goCloseQueues(queues[:index])) + } + natCount := uint32(1) + var udpNats []*UDPNat + if s.handler != nil { + udpNATOptions := s.udpNATOptions + udpNATOptions.Handler = s.handler + udpNATOptions.Prepare = s.prepareUDPConnection + maxSize := udpNATOptions.MaxSize + if maxSize == 0 { + if runtime.GOOS == "ios" { + maxSize = 4096 + } else { + maxSize = 16384 + } + } + if udpNATOptions.Mapping == NATMappingAddressAndPortDependent { + natCount = min(uint32(len(queues)), maxSize) + } + udpNats = make([]*UDPNat, natCount) + for index := range udpNats { + udpNATOptions.Shared = index+int(natCount) < len(queues) + udpNATOptions.MaxSize = maxSize / natCount + if uint32(index) < maxSize%natCount { + udpNATOptions.MaxSize++ + } + udpNat := NewUDPNat(udpNATOptions) + err = udpNat.Start() + if err != nil { + udpNat.Close() + for _, started := range udpNats[:index] { + started.Close() + } + return E.Errors(err, goCloseQueues(queues)) + } + udpNats[index] = udpNat + } + s.dispatcher = NewForwardDispatcher(s.handler, &goWriteback{platformIO: queues[0]}, s.logger, s.udpTimeout, s.icmpTimeout) + } + s.udpNats = udpNats + if natCount > 1 { + s.directory.udpFlows = make(map[udpNatSessionKey]*GoPacketConn) + } + s.engines = make([]*goEngine, len(queues)) + for index, queue := range queues { + s.engines[index] = newGoEngine(s, queue, len(queues)) + if udpNats != nil { + s.engines[index].udpNat = udpNats[index%int(natCount)] + } + } + for _, engine := range s.engines { + go engine.run() + } + return nil +} + +func goCloseQueues(queues []goPlatformIO) error { + var err error + for _, queue := range queues { + err = E.Errors(err, queue.close()) + } + return err +} + +func (s *Go) HasEndpoint(protocol uint8, local, remote netip.AddrPort) bool { + if s.closed.Load() || !local.IsValid() || !remote.IsValid() { + return false + } + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + remote = netip.AddrPortFrom(remote.Addr().Unmap(), remote.Port()) + if local.Addr().Is4() != remote.Addr().Is4() { + return false + } + s.directory.access.RLock() + defer s.directory.access.RUnlock() + switch protocol { + case uint8(header.TCPProtocolNumber): + return s.directory.flows[flowKey{protocol: protocol, source: remote, destination: local}] != nil || s.directory.listeners[local] != nil + case uint8(header.UDPProtocolNumber): + socket := s.directory.udpSockets[local] + return socket != nil && (!socket.connected || socket.remote == remote) + default: + return false + } +} + +type goFlowDirectory struct { + access sync.RWMutex + flows map[flowKey]*GoConn + udpFlows map[udpNatSessionKey]*GoPacketConn + udpSockets map[netip.AddrPort]*GoUDPConn + listeners map[netip.AddrPort]*GoListener +} + +func (d *goFlowDirectory) insertListener(listener *GoListener) { + d.access.Lock() + if len(listener.engine.stack.engines) > 1 { + listener.engine.tcpListeners[listener.local] = listener + } + d.listeners[listener.local] = listener + d.access.Unlock() +} + +func (d *goFlowDirectory) removeListener(listener *GoListener) { + d.access.Lock() + if len(listener.engine.stack.engines) > 1 && listener.engine.tcpListeners[listener.local] == listener { + delete(listener.engine.tcpListeners, listener.local) + } + if d.listeners[listener.local] == listener { + delete(d.listeners, listener.local) + } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookupListener(local netip.AddrPort) *GoListener { + d.access.RLock() + listener := d.listeners[local] + d.access.RUnlock() + return listener +} + +func (d *goFlowDirectory) insertUDPSocket(socket *GoUDPConn) { + d.access.Lock() + if len(socket.engine.stack.engines) > 1 { + socket.engine.udpSockets[socket.local] = socket + } + d.udpSockets[socket.local] = socket + d.access.Unlock() +} + +func (d *goFlowDirectory) removeUDPSocket(socket *GoUDPConn) { + d.access.Lock() + if len(socket.engine.stack.engines) > 1 && socket.engine.udpSockets[socket.local] == socket { + delete(socket.engine.udpSockets, socket.local) + } + if d.udpSockets[socket.local] == socket { + delete(d.udpSockets, socket.local) + } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookupUDPSocket(local netip.AddrPort) *GoUDPConn { + d.access.RLock() + socket := d.udpSockets[local] + d.access.RUnlock() + return socket +} + +func (d *goFlowDirectory) insert(key flowKey, conn *GoConn) { + d.access.Lock() + if len(conn.engine.stack.engines) > 1 { + conn.engine.flows[key] = conn + } + d.flows[key] = conn + d.access.Unlock() +} + +func (d *goFlowDirectory) remove(key flowKey, conn *GoConn) { + d.access.Lock() + if len(conn.engine.stack.engines) > 1 && conn.engine.flows[key] == conn { + delete(conn.engine.flows, key) + } + if d.flows[key] == conn { + delete(d.flows, key) + } + d.access.Unlock() +} + +func (d *goFlowDirectory) lookup(key flowKey) *GoConn { + d.access.RLock() + conn := d.flows[key] + d.access.RUnlock() + return conn +} + +func (s *Go) ResetNetwork() { + s.access.Lock() + defer s.access.Unlock() + if s.closed.Load() { + return + } + for _, udpNat := range s.udpNats { + udpNat.Purge() + } + s.dispatcher.ResetNetwork() + for _, engine := range s.engines { + engine.postMessage(&engine.resetMessage) + } +} + +func (s *Go) Close() error { + s.access.Lock() + if !s.closed.CompareAndSwap(false, true) { + s.access.Unlock() + return nil + } + for _, engine := range s.engines { + engine.postMessage(&engine.closeMessage) + } + s.access.Unlock() + for _, engine := range s.engines { + <-engine.exitSignal + } + if s.dispatcher != nil { + s.dispatcher.Close() + } + for _, udpNat := range s.udpNats { + udpNat.Close() + } + var err error + for _, engine := range s.engines { + err = E.Errors(err, engine.platformIO.close()) + } + return err +} diff --git a/stack_go_conn.go b/stack_go_conn.go new file mode 100644 index 00000000..1c2251e1 --- /dev/null +++ b/stack_go_conn.go @@ -0,0 +1,815 @@ +package tun + +import ( + "context" + "io" + "net" + "os" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/common/pipe" +) + +const ( + goPhaseJudged uint8 = iota + goPhaseEngaged + goPhaseEstablished +) + +const ( + goTCPSynReceived uint8 = iota + goTCPSynSent + goTCPEstablished + goTCPFinWait1 + goTCPFinWait2 + goTCPClosing + goTCPCloseWait + goTCPLastAck + goTCPClosed +) + +const ( + goDeathImmediate uint8 = iota + goDeathFinLinger + goDeathAbortLinger +) + +const ( + goTimeUnit = 4096 + goTimestampShift = 20 + goLocalWindowShift = 7 + goPermitWindowBit = 1 << 63 +) + +var errGoReset = E.Cause(syscall.ECONNRESET, "go: connection reset by peer") + +type goPostedReceive struct { + buffer *buf.Buffer + direct []byte + target []byte + filled int +} + +var ( + goTargetTaken goPostedReceive + goTargetFilled goPostedReceive +) + +type GoConn struct { + engine *goEngine + key flowKey + peer M.Socksaddr + local M.Socksaddr + epoch int64 + receiveNext uint64 + receiveCapacity uint64 + publishedEdge uint64 + sentEdge uint64 + receiveRoundTripMark uint64 + receiveRoundTripStamp int64 + receiveSpaceConsumed uint64 + receiveSpaceCopied uint64 + receiveSpaceStamp int64 + receiveChain goSlabChain + oooRanges *goRangeSet + dsackStart uint64 + dsackEnd uint64 + smoothedRoundTrip int32 + roundTripVariance int32 + retransmitTimeout int32 + receiveRoundTrip int32 + highestSacked uint64 + peerWindow uint64 + maxPeerWindow uint64 + ackPending uint64 + ackCovered uint64 + lastAckSent uint64 + lastActivity int64 + windowLeft1 int64 + windowLeft2 int64 + keepaliveProbes uint8 + finWait2Since int64 + scoreboard goScoreboard + timerNode goWheelNode + retransmitDeadline int64 + probeDeadline int64 + persistDeadline int64 + lingerDeadline int64 + handshakeDeadline int64 + handshakeStamp int64 + idleDeadline int64 + pacingDeadline int64 + reorderDeadline int64 + keepaliveDeadline int64 + retransmitAttempts uint8 + probeAttempts uint8 + handshakeAttempts uint8 + persistAttempts uint8 + state uint8 + + congestion *goCongestionOps + congestionPrivate any + flight goFlightSummary + roundTripMin goMinMax + congestionState uint8 + frto bool + sackReneging bool + windowLimited bool + rateAppLimited bool + probeRetransmitted bool + congestionWindow uint32 + slowStartThreshold uint32 + congestionClamp uint32 + congestionWindowCount uint32 + congestionWindowUsed uint32 + priorCongestionWindow uint32 + priorSlowStartThreshold uint32 + reductionDelivered uint32 + reductionRetransmits uint32 + reductionSegmentsOut uint32 + totalRetransmits uint64 + duplicateSegments uint64 + delivered uint32 + lost uint32 + renoSacked uint32 + reordering uint32 + reorderingSeen uint32 + maxPacketsOut uint32 + rateDelivered uint32 + rateIntervalMicros uint32 + retransmitStamp uint32 + lastTimestampEcho uint32 + undoRetransmits int32 + deliveredStamp int32 + deliveredTime int64 + lastSendStamp int64 + undoMarker uint64 + highSeq uint64 + probeHighSeq uint64 + rackEndOffset uint64 + rackRoundTripMicros int64 + rackStamp int32 + rackAdvanced bool + rackDSACKSeen bool + rackLastDelivered uint32 + reorderWindowSteps uint32 + reorderWindowPersist uint8 + windowUsageSeq uint64 + congestionWindowStamp int64 + ipVersion uint8 + deathClass uint8 + peerWindowShift uint8 + localWindowShift uint8 + peerMSS uint16 + effectiveMSS atomic.Uint32 + lastPeerWindow uint16 + handshakeLength uint8 + keyed bool + socket bool + finPending bool + sackPermitted bool + timestampsEnabled bool + discardReceive bool + finSent bool + finAcked bool + ackForced bool + ackDirty bool + onDyingList bool + clientISN uint32 + sendISN uint32 + finOffset uint64 + dyingSince int64 + ackNext *GoConn + blockedOn *goBlockedWriters + blockedPrev *GoConn + blockedNext *GoConn + dyingNext *GoConn + splice *goSpliceStream + listener *GoListener + handshakeImage [96]byte + _ [64]byte + + sendPermit atomic.Uint64 + sendPacketPermit atomic.Uint32 + packetCreditBase atomic.Uint32 + sendUnacked atomic.Uint64 + sendReleased atomic.Uint64 + receiveNextAck atomic.Uint64 + receiveEdge atomic.Uint64 + receiveAvailable atomic.Uint64 + windowUpdateAt atomic.Uint64 + deliveryState atomic.Uint64 + pacingRate atomic.Uint64 + pacingStamp atomic.Int64 + queuedBytes atomic.Int64 + departedTail atomic.Uint64 + queueThrottled atomic.Bool + firstSentStamp atomic.Int32 + appLimited atomic.Uint32 + frameLimit atomic.Uint32 + tsRecent atomic.Uint32 + _ [64]byte + + transmitAccess sync.Mutex + sentTail atomic.Uint64 + transmittedTail atomic.Uint64 + dataSentEdge atomic.Uint64 + peakFlight atomic.Uint64 + pacingRequest atomic.Int64 + dataSegmentsOut atomic.Uint32 + ident atomic.Uint32 + windowLimitedSince atomic.Bool + outboundQueue atomic.Pointer[OutboundQueue] + transmitStore goTransmitStore + descriptors goDescriptorRing + transmitScratch [goHeaderScratchSize]byte + transmitSegments [][]byte + blockedSegment goSegment + blockedFrame [][]byte + blockedMeta ForwardFrameMeta + blockedValid bool + _ [64]byte + + writeAccess sync.Mutex + bufferedTail atomic.Uint64 + writerWaiting atomic.Int32 + writeDeadline pipe.Deadline + writeSignal goSignal + _ [64]byte + + readAccess sync.Mutex + consumedTail atomic.Uint64 + readerParked atomic.Bool + postedTarget atomic.Pointer[goPostedReceive] + receiveTarget goPostedReceive + readWaitOptions N.ReadWaitOptions + readDeadline pipe.Deadline + readSignal goSignal + _ [64]byte + + access sync.Mutex + phase uint8 + receiveCapacityMax uint64 + transmitCapacity uint64 + keepaliveIdle time.Duration + keepaliveInterval time.Duration + keepaliveCount uint8 + keepaliveEnabled bool + nagle bool + linger int + dead bool + finReceived bool + drainable bool + released bool + spliced bool + userClosed bool + writeShut bool + readShut bool + finRequested bool + abort bool + writing bool + transmitterRunning bool + err error + splicePending *goSpliceStream + establishedSignal chan struct{} + finAckedSignal chan struct{} + closeSignal chan struct{} + transmitSignal goSignal + dialMessage goMessage + engageMessage goMessage + closeMessage goMessage + readShutMessage goMessage + retransmitMessage goMessage + windowMessage goMessage + transmitMessage goMessage + blockedMessage goMessage + droppedMessage goMessage + pacingMessage goMessage + throttleMessage goMessage + spliceMessage goMessage + keepaliveMessage goMessage + transmittedMessage goMessage + slabHolder goSlabHolder +} + +func (c *GoConn) initialize(engine *goEngine, key flowKey, peer M.Socksaddr, local M.Socksaddr) { + c.engine = engine + c.key = key + c.peer = peer + c.local = local + c.epoch = engine.coarseTime.Load() + c.lastSendStamp = -1 + c.readDeadline = pipe.MakeDeadline() + c.writeDeadline = pipe.MakeDeadline() + c.readSignal = make(goSignal, 1) + c.writeSignal = make(goSignal, 1) + c.transmitSignal = make(goSignal, 1) + c.establishedSignal = make(chan struct{}) + c.finAckedSignal = make(chan struct{}) + c.closeSignal = make(chan struct{}) + c.receiveCapacityMax = goReceiveCapacityMax + c.transmitCapacity = goTransmitCapacityMax + c.keepaliveIdle = tcpEstablishedTimeout + c.keepaliveInterval = goKeepaliveInterval + c.keepaliveCount = goKeepaliveCount + c.keepaliveEnabled = true + c.linger = -1 + c.receiveChain = goSlabChain{maxSlots: goReceiveCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} + c.transmitStore.chain = goSlabChain{maxSlots: goTransmitCapacityMax/goSlabSize + 1, pool: engine.slabPool, holder: &c.slabHolder} + c.descriptors.pool = &engine.descriptorPool + c.timerNode.expire = func(now int64) { engine.fireConnTimer(c, now) } + c.engageMessage = goMessage{kind: goMessageConnEngage, conn: c} + c.closeMessage = goMessage{kind: goMessageConnClose, conn: c} + c.readShutMessage = goMessage{kind: goMessageConnReadShut, conn: c} + c.retransmitMessage = goMessage{kind: goMessageConnRetransmit, conn: c} + c.windowMessage = goMessage{kind: goMessageConnWindow, conn: c} + c.transmitMessage = goMessage{kind: goMessageConnTransmit, conn: c} + c.blockedMessage = goMessage{kind: goMessageConnBlocked, conn: c} + c.droppedMessage = goMessage{kind: goMessageConnDropped, conn: c} + c.pacingMessage = goMessage{kind: goMessageConnPacing, conn: c} + c.throttleMessage = goMessage{kind: goMessageConnThrottle, conn: c} + c.spliceMessage = goMessage{kind: goMessageConnSplice, conn: c} + c.keepaliveMessage = goMessage{kind: goMessageConnKeepalive, conn: c} + c.transmittedMessage = goMessage{kind: goMessageConnTransmitted, conn: c} +} + +func (c *GoConn) transmitted(tail uint64) { + c.transmittedTail.Store(tail) + if c.sendReleased.Load() < min(c.sendUnacked.Load(), tail) { + c.engine.postMessage(&c.transmittedMessage) + } +} + +func (c *GoConn) handshaking() bool { + return c.state == goTCPSynReceived || c.state == goTCPSynSent +} + +func (c *GoConn) receiveWindowBound() uint64 { + c.access.Lock() + capacityMax := c.receiveCapacityMax + c.access.Unlock() + return min(capacityMax, uint64(0xffff)< consumed { + buffer := c.readWaitOptions.NewBuffer() + n := c.copyReceived(buffer.FreeBytes(), consumed, available) + buffer.Truncate(n) + c.advanceConsumed(n) + c.readWaitOptions.PostReturn(buffer) + return buffer, nil + } + result := c.receiveResult() + if result != nil { + return nil, result + } + c.receiveTarget.buffer = c.readWaitOptions.NewBuffer() + delivered, parkErr := c.parkReader() + if delivered { + buffer := c.receiveTarget.buffer + c.receiveTarget.buffer = nil + buffer.Truncate(c.receiveTarget.filled) + c.advanceConsumed(c.receiveTarget.filled) + c.readWaitOptions.PostReturn(buffer) + return buffer, nil + } + c.receiveTarget.buffer.Release() + c.receiveTarget.buffer = nil + if parkErr != nil { + return nil, parkErr + } + } +} + +func (c *GoConn) Read(p []byte) (int, error) { + err := c.awaitHandshake(context.Background(), c.readDeadline.Wait()) + if err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + c.readAccess.Lock() + n, err := c.readLocked(p) + c.readAccess.Unlock() + c.engine.reapAfterExit() + return n, err +} + +func (c *GoConn) readLocked(p []byte) (int, error) { + if !c.readable() { + return 0, c.receiveResult() + } + for { + select { + case <-c.readDeadline.Wait(): + return 0, os.ErrDeadlineExceeded + default: + } + available := c.receiveAvailable.Load() + consumed := c.consumedTail.Load() + if available > consumed { + n := c.copyReceived(p, consumed, available) + c.advanceConsumed(n) + return n, nil + } + result := c.receiveResult() + if result != nil { + return 0, result + } + c.receiveTarget.buffer = nil + c.receiveTarget.direct = p + delivered, parkErr := c.parkReader() + c.receiveTarget.direct = nil + if delivered { + n := c.receiveTarget.filled + c.advanceConsumed(n) + return n, nil + } + if parkErr != nil { + return 0, parkErr + } + } +} + +func (c *GoConn) readable() bool { + c.access.Lock() + defer c.access.Unlock() + return !c.dead || (c.drainable && !c.released) +} + +func (c *GoConn) receiveResult() error { + c.access.Lock() + defer c.access.Unlock() + if c.err != nil { + return c.err + } + if c.userClosed { + return net.ErrClosed + } + if c.readShut || c.finReceived || c.dead { + return io.EOF + } + return nil +} + +func (c *GoConn) wakeUser() { + c.readSignal.notify() + c.writeSignal.notify() +} + +func (c *GoConn) copyReceived(target []byte, consumed uint64, available uint64) int { + length := min(len(target), int(available-consumed)) + c.receiveChain.readAt(consumed, target[:length]) + return length +} + +func (c *GoConn) advanceConsumed(n int) { + consumed := c.consumedTail.Add(uint64(n)) + if consumed >= c.windowUpdateAt.Load() { + c.engine.postMessage(&c.windowMessage) + return + } + if c.receiveChain.held.Load() == 0 { + return + } + if consumed/goSlabSize != (consumed-uint64(n))/goSlabSize || consumed == c.receiveAvailable.Load() { + c.engine.postMessage(&c.windowMessage) + } +} + +func (c *GoConn) parkReader() (bool, error) { + target := &c.receiveTarget + target.filled = 0 + if target.buffer != nil { + target.target = target.buffer.FreeBytes() + } else { + target.target = target.direct + } + c.readSignal.drain() + c.postedTarget.Store(target) + c.readerParked.Store(true) + if c.receiveAvailable.Load() > c.consumedTail.Load() || c.receiveResult() != nil { + c.readerParked.Store(false) + return c.claimTarget(target), nil + } + deadlineSignal := c.readDeadline.Wait() + select { + case <-c.readSignal: + c.readerParked.Store(false) + return c.claimTarget(target), nil + case <-deadlineSignal: + c.readerParked.Store(false) + if c.claimTarget(target) { + return true, nil + } + if c.receiveAvailable.Load() > c.consumedTail.Load() { + return false, nil + } + return false, os.ErrDeadlineExceeded + case <-c.closeSignal: + c.readerParked.Store(false) + if c.claimTarget(target) { + return true, nil + } + return false, nil + } +} + +func (c *GoConn) claimTarget(target *goPostedReceive) bool { + if c.postedTarget.CompareAndSwap(target, nil) { + target.target = nil + return false + } + for c.postedTarget.Load() != &goTargetFilled { + <-c.readSignal + } + c.postedTarget.Store(nil) + target.target = nil + return true +} + +func (c *GoConn) Close() error { + c.access.Lock() + if c.userClosed { + c.access.Unlock() + return nil + } + c.userClosed = true + c.writeShut = true + c.readShut = true + linger := c.linger + graceful := !c.dead && !c.abort && linger != 0 && c.phase == goPhaseEstablished && !c.spliced && c.splicePending == nil && + c.receiveAvailable.Load() <= c.consumedTail.Load() + if graceful { + c.finRequested = true + } else { + c.abort = true + if c.err == nil { + c.err = net.ErrClosed + } + } + c.access.Unlock() + if graceful { + c.engine.postMessage(&c.readShutMessage) + } + c.engine.postMessage(&c.closeMessage) + c.wakeUser() + if graceful && linger > 0 { + timer := time.NewTimer(time.Duration(linger) * time.Second) + defer timer.Stop() + select { + case <-c.finAckedSignal: + case <-c.closeSignal: + case <-timer.C: + } + } + return nil +} + +func (c *GoConn) CloseRead() error { + c.access.Lock() + if c.dead { + c.access.Unlock() + return nil + } + c.readShut = true + c.access.Unlock() + c.engine.postMessage(&c.readShutMessage) + c.readSignal.notify() + return nil +} + +func (c *GoConn) CloseWrite() error { + c.access.Lock() + if c.dead { + c.access.Unlock() + return nil + } + c.writeShut = true + c.finRequested = true + c.access.Unlock() + c.engine.postMessage(&c.closeMessage) + c.writeSignal.notify() + return nil +} + +func (c *GoConn) LocalAddr() net.Addr { + if c.socket { + return c.local.TCPAddr() + } + return c.peer.TCPAddr() +} + +func (c *GoConn) RemoteAddr() net.Addr { + if c.socket { + return c.peer.TCPAddr() + } + return c.local.TCPAddr() +} + +func (c *GoConn) SetDeadline(t time.Time) error { + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil +} + +func (c *GoConn) SetReadDeadline(t time.Time) error { + c.readDeadline.Set(t) + return nil +} + +func (c *GoConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline.Set(t) + return nil +} + +type goSignal chan struct{} + +func (s goSignal) notify() bool { + select { + case s <- struct{}{}: + return true + default: + return false + } +} + +func (s goSignal) drain() { + select { + case <-s: + default: + } +} + +var ( + _ N.ReadWaiter = (*GoConn)(nil) + _ N.ExtendedWriter = (*GoConn)(nil) + _ N.FrontHeadroom = (*GoConn)(nil) + _ N.WriterWithMTU = (*GoConn)(nil) + _ N.ReadCloser = (*GoConn)(nil) + _ N.WriteCloser = (*GoConn)(nil) + _ N.EarlyReader = (*GoConn)(nil) + _ N.EarlyWriter = (*GoConn)(nil) + _ N.HandshakeSuccess = (*GoConn)(nil) + _ N.HandshakeFailure = (*GoConn)(nil) +) diff --git a/stack_go_conn_options.go b/stack_go_conn_options.go new file mode 100644 index 00000000..5a45fbb1 --- /dev/null +++ b/stack_go_conn_options.go @@ -0,0 +1,87 @@ +package tun + +import ( + "net" + "time" +) + +const ( + goKeepaliveDefaultIdle = 5 * time.Minute + goKeepaliveDefaultInterval = 75 * time.Second + goKeepaliveDefaultCount = 9 + goKeepaliveMaxCount = 127 +) + +func (c *GoConn) SetKeepAlive(keepalive bool) error { + c.access.Lock() + c.keepaliveEnabled = keepalive + c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) + return nil +} + +func (c *GoConn) SetKeepAlivePeriod(d time.Duration) error { + c.access.Lock() + c.setKeepaliveIdle(d) + c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) + return nil +} + +func (c *GoConn) SetKeepAliveConfig(config net.KeepAliveConfig) error { + c.access.Lock() + c.keepaliveEnabled = config.Enable + c.setKeepaliveIdle(config.Idle) + switch { + case config.Interval == 0: + c.keepaliveInterval = goKeepaliveDefaultInterval + case config.Interval > 0: + c.keepaliveInterval = config.Interval + } + switch { + case config.Count == 0: + c.keepaliveCount = goKeepaliveDefaultCount + case config.Count > 0: + c.keepaliveCount = uint8(min(config.Count, goKeepaliveMaxCount)) + } + c.access.Unlock() + c.engine.postMessage(&c.keepaliveMessage) + return nil +} + +func (c *GoConn) setKeepaliveIdle(idle time.Duration) { + switch { + case idle == 0: + c.keepaliveIdle = goKeepaliveDefaultIdle + case idle > 0: + c.keepaliveIdle = idle + } +} + +func (c *GoConn) SetNoDelay(noDelay bool) error { + c.access.Lock() + c.nagle = !noDelay + c.access.Unlock() + return nil +} + +func (c *GoConn) SetLinger(sec int) error { + c.access.Lock() + c.linger = sec + c.access.Unlock() + return nil +} + +func (c *GoConn) SetReadBuffer(size int) error { + c.access.Lock() + c.receiveCapacityMax = uint64(min(max(size, goSlabSize), goReceiveCapacityMax)) + c.access.Unlock() + return nil +} + +func (c *GoConn) SetWriteBuffer(size int) error { + c.access.Lock() + c.transmitCapacity = uint64(min(max(size, goSlabSize), goTransmitCapacityMax)) + c.access.Unlock() + return nil +} diff --git a/stack_go_dial.go b/stack_go_dial.go new file mode 100644 index 00000000..a6777163 --- /dev/null +++ b/stack_go_dial.go @@ -0,0 +1,339 @@ +package tun + +import ( + "context" + "hash/maphash" + "math/rand/v2" + "net" + "net/netip" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" +) + +const ( + goSynRetransmit = time.Second + goSynAttempts = 6 + goEphemeralPortBase = 32768 + goEphemeralPortSpan = 60999 - goEphemeralPortBase + 1 + goEphemeralAttempts = 1024 +) + +var errGoInvalidAddress = E.New("go: invalid address") + +func (s *Go) DialTCP(ctx context.Context, source netip.Addr, destination netip.AddrPort) (*GoConn, error) { + source = source.Unmap() + destination = netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !source.IsValid() || !destination.IsValid() || destination.Port() == 0 || source.Is4() != destination.Addr().Is4() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(destination) + if err != nil { + return nil, err + } + conn := new(GoConn) + conn.initialize(engine, flowKey{}, M.SocksaddrFromNetIP(destination), M.SocksaddrFrom(source, 0)) + conn.socket = true + conn.ipVersion = goIPVersion(source) + conn.state = goTCPSynSent + conn.phase = goPhaseEngaged + conn.localWindowShift = goLocalWindowShift + conn.timestampsEnabled = true + conn.sackPermitted = true + conn.receiveCapacity = min(goReceiveCapacityBase, conn.receiveWindowBound()) + conn.dialMessage = goMessage{kind: goMessageConnDial, conn: conn} + engine.postMessage(&conn.dialMessage) + err = conn.awaitHandshake(ctx, nil) + if err != nil { + return nil, err + } + return conn, nil +} + +func goIPVersion(addr netip.Addr) uint8 { + if addr.Is4() { + return 4 + } + return 6 +} + +func (s *Go) selectEngine(destination netip.AddrPort) (*goEngine, error) { + s.access.Lock() + defer s.access.Unlock() + if s.closed.Load() { + return nil, net.ErrClosed + } + return s.engines[maphash.Comparable(s.engines[0].sequenceSeed, destination)%uint64(len(s.engines))], nil +} + +func goAllocatePort(available func(port uint16) bool) (uint16, bool) { + offset := rand.IntN(goEphemeralPortSpan) + for attempt := range goEphemeralAttempts { + port := uint16(goEphemeralPortBase + (offset+attempt)%goEphemeralPortSpan) + if available(port) { + return port, true + } + } + return 0, false +} + +func (e *goEngine) handleDial(conn *GoConn) { + if conn.dead { + return + } + if len(e.flows) >= e.flowCapacity && !e.evictFlow() { + e.detachConn(conn, E.Cause(syscall.ENOBUFS, "go: flow table full"), goDeathImmediate) + return + } + key := flowKey{protocol: uint8(header.TCPProtocolNumber), source: conn.peer.AddrPort()} + port, allocated := goAllocatePort(func(port uint16) bool { + key.destination = netip.AddrPortFrom(conn.local.Addr, port) + return e.flows[key] == nil && (len(e.stack.engines) == 1 || e.stack.directory.lookup(key) == nil) && e.lookupListener(key.destination) == nil + }) + if !allocated { + e.detachConn(conn, E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted"), goDeathImmediate) + return + } + now := e.coarseTime.Load() + conn.key = key + conn.local.Port = port + conn.sendISN = e.initialSequence(key, now) + conn.lastActivity = now + conn.buildHandshake(e.localMSS(conn.ipVersion), true, true, header.TCPFlagSyn, 0) + conn.keyed = true + e.stack.directory.insert(key, conn) + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.detachConn(conn, E.Cause(err, "go: send SYN"), goDeathImmediate) + return + } + conn.handshakeAttempts = 0 + conn.handshakeStamp = now + conn.handshakeDeadline = now + int64(goSynRetransmit) + e.rearmTimer(conn) +} + +func (e *goEngine) retransmitSyn(conn *GoConn, now int64) { + conn.handshakeAttempts++ + if conn.handshakeAttempts >= goSynAttempts { + conn.handshakeDeadline = 0 + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: connect timeout"), goDeathImmediate) + return + } + conn.buildHandshake(e.localMSS(conn.ipVersion), true, true, header.TCPFlagSyn, 0) + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN")) + } + conn.handshakeDeadline = now + int64(goSynRetransmit)<= 0 { + conn.peerWindowShift = uint8(synOptions.WS) + } else { + conn.localWindowShift = 0 + } + if synOptions.TS { + conn.tsRecent.Store(synOptions.TSVal) + effectiveMSS -= goTimestampOptionLength + } else { + conn.timestampsEnabled = false + } + conn.effectiveMSS.Store(uint32(effectiveMSS)) + e.initializeSession(conn, now, tcpHdr.WindowSize()) + if conn.handshakeAttempts == 0 { + roundTrip := max((now-conn.handshakeStamp)/int64(time.Microsecond), 1) + e.ackUpdateRoundTrip(conn, 0, roundTrip, -1, roundTrip, &e.rateSample, 0, false, now) + } + e.establishConn(conn) + e.rearmTimer(conn) + e.sendAck(conn) +} + +func (e *goEngine) inputICMPError(parsed *forwardPacket) { + inner, ok := parsed.icmpErrorInner() + if !ok { + return + } + embedded, ok := parseEmbedded(inner) + if !ok { + return + } + mtu, tooBig := goICMPPacketTooBig(parsed) + if tooBig { + e.inputPacketTooBig(&embedded, mtu) + return + } + err := goICMPError(parsed) + if err == nil { + return + } + switch embedded.protocol { + case uint8(header.TCPProtocolNumber): + conn := e.flows[embedded.flowKey().reversed()] + if conn == nil || conn.dead || conn.state != goTCPSynSent { + return + } + if len(embedded.payload) < header.TCPAckNumOffset || header.TCP(embedded.payload).SequenceNumber() != conn.sendISN { + return + } + e.detachConn(conn, E.Cause(err, "go: connect"), goDeathImmediate) + case uint8(header.UDPProtocolNumber): + socket := e.udpSockets[embedded.source] + if socket == nil || !socket.connected || socket.remote != embedded.destination { + return + } + socket.deliverError(err) + } +} + +func goICMPError(parsed *forwardPacket) error { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + icmpHdr := header.ICMPv4(parsed.transport) + switch icmpHdr.Type() { + case header.ICMPv4DstUnreachable: + switch icmpHdr.Code() { + case header.ICMPv4NetUnreachable, header.ICMPv4NetUnreachableForTos: + return syscall.ENETUNREACH + case header.ICMPv4HostUnreachable, header.ICMPv4HostUnreachableForTos: + return syscall.EHOSTUNREACH + case header.ICMPv4ProtoUnreachable: + return syscall.EPROTO + case header.ICMPv4PortUnreachable: + return syscall.ECONNREFUSED + case header.ICMPv4NetProhibited, header.ICMPv4HostProhibited: + return syscall.EACCES + default: + return syscall.EHOSTUNREACH + } + case header.ICMPv4TimeExceeded: + return syscall.EHOSTUNREACH + case header.ICMPv4ParamProblem: + return syscall.EPROTO + default: + return nil + } + case uint8(header.ICMPv6ProtocolNumber): + icmpHdr := header.ICMPv6(parsed.transport) + switch icmpHdr.Type() { + case header.ICMPv6DstUnreachable: + switch icmpHdr.Code() { + case header.ICMPv6NetworkUnreachable: + return syscall.ENETUNREACH + case header.ICMPv6Prohibited: + return syscall.EACCES + case header.ICMPv6PortUnreachable: + return syscall.ECONNREFUSED + default: + return syscall.EHOSTUNREACH + } + case header.ICMPv6TimeExceeded: + return syscall.EHOSTUNREACH + case header.ICMPv6ParamProblem: + return syscall.EPROTO + default: + return nil + } + default: + return nil + } +} + +func goICMPPacketTooBig(parsed *forwardPacket) (int, bool) { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + icmpHdr := header.ICMPv4(parsed.transport) + if icmpHdr.Type() != header.ICMPv4DstUnreachable || icmpHdr.Code() != header.ICMPv4FragmentationNeeded { + return 0, false + } + return int(icmpHdr.MTU()), true + case uint8(header.ICMPv6ProtocolNumber): + icmpHdr := header.ICMPv6(parsed.transport) + if icmpHdr.Type() != header.ICMPv6PacketTooBig { + return 0, false + } + return int(icmpHdr.MTU()), true + default: + return 0, false + } +} + +func (e *goEngine) inputPacketTooBig(embedded *embeddedPacket, mtu int) { + if embedded.protocol != uint8(header.TCPProtocolNumber) || mtu == 0 { + return + } + conn := e.flows[embedded.flowKey().reversed()] + if conn == nil || conn.dead || conn.state < goTCPEstablished || len(embedded.payload) < header.TCPAckNumOffset { + return + } + offset := conn.sendOffset(header.TCP(embedded.payload).SequenceNumber()) + if offset < int64(conn.sendUnacked.Load()) || offset >= int64(conn.sendNext()) { + return + } + if conn.ipVersion == 4 { + mtu = max(mtu, header.IPv4MinimumProcessableDatagramSize) + } else { + mtu = max(mtu, header.IPv6MinimumMTU) + } + mss := mtu - goNetworkHeaderLength(conn.ipVersion) - header.TCPMinimumSize + if conn.timestampsEnabled { + mss -= goTimestampOptionLength + } + if mss >= int(conn.effectiveMSS.Load()) { + return + } + e.drainDescriptors(conn) + lost := false + start := conn.sendUnacked.Load() + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.endOffset-start > uint64(mss) && descriptor.flags&(goDescriptorSacked|goDescriptorLost) == 0 { + e.markLost(conn, descriptor) + lost = true + } + start = descriptor.endOffset + } + conn.effectiveMSS.Store(uint32(mss)) + e.refreshPacketsOut(conn) + if lost { + e.enterCWR(conn) + if conn.congestionState < goCongestionRecovery { + e.enterRecovery(conn) + } + e.xmitRetransmitQueue(conn) + } + e.rearmRetransmit(conn) + conn.publishPermit(conn.sendUnacked.Load()) +} diff --git a/stack_go_engine.go b/stack_go_engine.go new file mode 100644 index 00000000..20472ffd --- /dev/null +++ b/stack_go_engine.go @@ -0,0 +1,1078 @@ +package tun + +import ( + "errors" + "hash/maphash" + "math" + "net" + "net/netip" + "os" + "runtime" + "slices" + "sync" + "sync/atomic" + "time" + + tcpip "github.com/sagernet/sing-tun/gtcpip" + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +const ( + goWaitIndefinite = time.Duration(-1) + goShutdownBudget = 250 * time.Millisecond + goEngineLoadWindow = 100 * time.Millisecond + goEngineBusyPercent = 70 +) + +const ( + goMessageStackClose uint8 = iota + goMessageStackReset + goMessageConnDial + goMessageConnEngage + goMessageConnClose + goMessageConnReadShut + goMessageConnRetransmit + goMessageConnWindow + goMessageConnTransmit + goMessageConnBlocked + goMessageConnDropped + goMessageConnPacing + goMessageConnThrottle + goMessageConnSplice + goMessageConnKeepalive + goMessageConnTransmitted + goMessagePacketSplice + goMessagePacketClose + goMessageUDPOpen + goMessageUDPClose + goMessageListenOpen + goMessageListenClose + goMessageInject +) + +const goInjectQueueLimit = 512 + +type goInjectedFrame struct { + next *goInjectedFrame + buffer *buf.Buffer + meta ForwardFrameMeta +} + +type goInjectStack struct { + head atomic.Pointer[goInjectedFrame] +} + +func (s *goInjectStack) push(frame *goInjectedFrame) { + for { + head := s.head.Load() + frame.next = head + if s.head.CompareAndSwap(head, frame) { + return + } + } +} + +func (s *goInjectStack) popAll() *goInjectedFrame { + head := s.head.Swap(nil) + var ordered *goInjectedFrame + for head != nil { + next := head.next + head.next = ordered + ordered = head + head = next + } + return ordered +} + +type goMessage struct { + next *goMessage + conn *GoConn + packet *GoPacketConn + socket *GoUDPConn + listener *GoListener + queued atomic.Bool + kind uint8 +} + +type goControlStack struct { + head atomic.Pointer[goMessage] +} + +func (s *goControlStack) push(message *goMessage) { + for { + head := s.head.Load() + message.next = head + if s.head.CompareAndSwap(head, message) { + return + } + } +} + +const ( + goEngineRunning uint32 = iota + goEngineParked + goEngineExited +) + +type goEngine struct { + stack *Go + platformIO goPlatformIO + dispatcher *ForwardDispatcher + dispatchStage *ForwardStage + wheel goWheel + controlStack goControlStack + injectStack goInjectStack + injectCount atomic.Int32 + injectMessage goMessage + droppedInjectFrames goDropCounter + engineState atomic.Uint32 + exitReapPending atomic.Bool + exitReapAccess sync.Mutex + coarseTime atomic.Int64 + epoch time.Time + frames []goFrame + closeMessage goMessage + resetMessage goMessage + delayedAckTickNode goWheelNode + reassemblyTickNode goWheelNode + dispatchTickNode goWheelNode + reassemblyEntries []goReassemblyEntry + udpUserData goUDPUserData + udpNat *UDPNat + transmitFrame [1][]byte + exitSignal chan struct{} + wokeHandlerThisBurst bool + tunPending bool + transmitWritable []*goBlockedWriters + readBuffersHeld bool + idleSince int64 + parkedNanos int64 + loadWindowStart int64 + ackCoalescing bool + inlineTransmitBudget int + socketEvents []goSocketEvent + spliceSlots []goSpliceSlot + spliceFree []uint32 + spliceDirtyList *GoConn + spliceSegments [][]byte + spliceIovecs []goIOVector + packetSlots goReadSlots + packetReceiveBatch int + packetMessages [goPacketBatchSize]goPacketMessage + packetFrames [goPacketBatchSize]goUDPFrame + packetFrameCount int + packetUploads [goPacketBatchSize]goPacketUpload + packetUploadCount int + packetDirtyList *goSplicePacket + packetIO goPacketBatchIO + packetFlows map[flowKey]*GoPacketConn + packetReadOptions atomic.Pointer[N.ReadWaitOptions] + udpSockets map[netip.AddrPort]*GoUDPConn + tcpListeners map[netip.AddrPort]*GoListener + + flows map[flowKey]*GoConn + flowCapacity int + slabPool *goSlabPool + descriptorPool goDescriptorPool + sequenceSeed maphash.Seed + ackList *GoConn + releasing goBlockedWriters + dyingList *GoConn + reclaimPending bool + resetBurst int + controlIdent uint32 + controlScratch [goHeaderScratchSize]byte + controlSegments [][]byte + sackScratch [goMaxSackBlocks]goSackBlock + sackRaw [goMaxSackBlocks]goRawSackBlock + rateSample goRateSample + ackSample goAckSample + + gsoSlots goReadSlots + gsoSegments [goGSOMaxSegments][]byte + gsoSizes [goGSOMaxSegments]int + gsoFrame goFrame +} + +func (e *goEngine) singleFrame(packet []byte) [][]byte { + e.transmitFrame[0] = packet + return e.transmitFrame[:] +} + +func newGoEngine(stack *Go, platformIO goPlatformIO, engineCount int) *goEngine { + engine := &goEngine{ + stack: stack, + platformIO: platformIO, + dispatcher: stack.dispatcher, + epoch: time.Now(), + frames: make([]goFrame, goReadBatch), + reassemblyEntries: make([]goReassemblyEntry, goReassemblyEntries), + exitSignal: make(chan struct{}), + flows: stack.directory.flows, + flowCapacity: max(goFlowCapacity/engineCount, 1024), + slabPool: newGoSlabPool(stack.memoryPressure, max(goSlabPoolLowWater/engineCount, 8)), + sequenceSeed: maphash.MakeSeed(), + controlSegments: make([][]byte, 0, 8), + socketEvents: make([]goSocketEvent, goSocketEventBatch), + spliceSegments: make([][]byte, 0, 8), + spliceIovecs: make([]goIOVector, 0, 8), + packetFlows: make(map[flowKey]*GoPacketConn, goReadBatch), + packetReceiveBatch: goReceiveBatchMin, + udpSockets: stack.directory.udpSockets, + tcpListeners: stack.directory.listeners, + } + if engineCount > 1 { + engine.flows = make(map[flowKey]*GoConn) + engine.udpSockets = make(map[netip.AddrPort]*GoUDPConn) + engine.tcpListeners = make(map[netip.AddrPort]*GoListener) + } + if stack.dispatcher != nil { + engine.dispatchStage = stack.dispatcher.NewStage(&goWriteback{platformIO: platformIO}) + } + engine.closeMessage.kind = goMessageStackClose + engine.resetMessage.kind = goMessageStackReset + engine.injectMessage.kind = goMessageInject + engine.packetReadOptions.Store(new(N.ReadWaitOptions)) + engine.delayedAckTickNode.expire = func(now int64) { engine.drainAckList(now, false) } + engine.reassemblyTickNode.expire = engine.expireReassemblyTick + engine.dispatchTickNode.expire = func(int64) { engine.dispatchStage.Flush() } + engine.refreshCoarseTime() + engine.wheel.currentTick = engine.coarseTime.Load() / goWheelTick + return engine +} + +func (e *goEngine) run() { + defer close(e.exitSignal) + defer e.exit() + e.loadWindowStart = e.coarseTime.Load() + var readRetry ReadRetry + for { + parkStart := int64(time.Since(e.epoch)) + tunReadable, eventCount, err := e.park() + e.parkedNanos += e.coarseTime.Load() - parkStart + e.updateLoad(e.coarseTime.Load()) + e.drainControlQueue() + if e.stack.closed.Load() { + return + } + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + for _, waiters := range e.transmitWritable { + e.releaseBlockedWriters(waiters) + } + clear(e.transmitWritable) + e.transmitWritable = e.transmitWritable[:0] + if err != nil { + if !errors.Is(err, os.ErrClosed) { + e.stack.logger.Error(E.Cause(err, "go: engine wait")) + } + return + } + e.resetBurst = 0 + e.inlineTransmitBudget = goEngineInlineBurst + e.platformIO.flush() + if tunReadable || e.tunPending { + err = e.processBurst() + if err != nil { + if e.stack.closed.Load() || E.IsClosedOrCanceled(err) { + return + } + if !IsRecoverableReadError(err) { + e.stack.logger.Error(E.Cause(err, "go: engine read")) + return + } + e.stack.logger.Debug(E.Cause(err, "go: engine read")) + readRetry.Wait(err) + } else { + readRetry.Reset() + } + } + e.dispatchSocketEvents(eventCount) + e.flushSpliceDirty() + e.wheel.advance(e.coarseTime.Load()) + e.drainAckList(e.coarseTime.Load(), true) + if e.dispatchStage != nil { + e.dispatchStage.Flush() + remaining, pending := e.dispatchStage.sweepDue() + if pending && e.dispatchTickNode.slot == nil { + e.wheel.schedule(&e.dispatchTickNode, e.coarseTime.Load()+int64(remaining)) + } + } + e.flushPacketUploads() + e.flushPacketFrames() + e.platformIO.flush() + e.reapDying() + if e.reclaimPending { + e.reclaimPending = false + e.reclaim() + } + if e.wokeHandlerThisBurst { + e.wokeHandlerThisBurst = false + runtime.Gosched() + } + } +} + +func (e *goEngine) exit() { + e.shutdown() + e.releaseReadBuffers() + e.packetSlots.release() + e.gsoSlots.release() + e.engineState.Store(goEngineExited) + e.releaseControlQueue() + e.releaseInjected() + e.slabPool.close() + if e.dyingList != nil { + e.exitReapPending.Store(true) + e.reapAfterExit() + } +} + +func (e *goEngine) reapAfterExit() { + if !e.exitReapPending.Load() { + return + } + e.exitReapAccess.Lock() + defer e.exitReapAccess.Unlock() + e.refreshCoarseTime() + e.reapDying() + e.exitReapPending.Store(e.dyingList != nil) +} + +func (e *goEngine) updateLoad(now int64) { + elapsed := now - e.loadWindowStart + if elapsed < int64(goEngineLoadWindow) { + return + } + busy := elapsed - e.parkedNanos + e.ackCoalescing = busy*100 >= elapsed*goEngineBusyPercent && len(e.flows) >= 2 + e.parkedNanos = 0 + e.loadWindowStart = now +} + +func (e *goEngine) park() (bool, int, error) { + clear(e.packetFlows) + deadline, scheduled := e.wheel.nextDeadline() + e.engineState.Store(goEngineParked) + if e.controlStack.head.Load() != nil { + e.engineState.Store(goEngineRunning) + e.refreshCoarseTime() + return false, 0, nil + } + timeout := goWaitIndefinite + if scheduled { + timeout = max(time.Duration(deadline-e.coarseTime.Load()), 0) + } + if e.tunPending { + timeout = 0 + } + tunReadable, eventCount, err := e.platformIO.wait(0, e.socketEvents) + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + if timeout != 0 && !tunReadable && eventCount == 0 && len(e.transmitWritable) == 0 && err == nil && e.controlStack.head.Load() == nil { + tunReadable, eventCount, err = e.platformIO.wait(e.idleTimeout(timeout), e.socketEvents) + } + if tunReadable || eventCount > 0 { + e.readBuffersHeld = true + e.idleSince = -1 + } + e.engineState.Store(goEngineRunning) + e.refreshCoarseTime() + return tunReadable, eventCount, err +} + +func (e *goEngine) idleTimeout(timeout time.Duration) time.Duration { + if !e.readBuffersHeld { + return timeout + } + now := e.coarseTime.Load() + if e.idleSince < 0 { + e.idleSince = now + } + releaseAt := e.idleSince + int64(goReadBufferIdle) + if now >= releaseAt || e.slabPool.pressureLevel() != MemoryPressureNone { + e.releaseReadBuffers() + return timeout + } + if timeout == goWaitIndefinite || time.Duration(releaseAt-now) < timeout { + return time.Duration(releaseAt - now) + } + return timeout +} + +func (e *goEngine) releaseReadBuffers() { + e.readBuffersHeld = false + e.flushPacketUploads() + e.flushPacketFrames() + clear(e.frames) + e.udpUserData.packet = nil + clear(e.transmitFrame[:]) + clear(e.spliceSegments[:cap(e.spliceSegments)]) + clear(e.spliceIovecs[:cap(e.spliceIovecs)]) + clear(e.controlSegments[:cap(e.controlSegments)]) + e.packetSlots.sleep() + e.gsoSlots.sleep() + clear(e.gsoSegments[:]) + e.gsoFrame = goFrame{} + clear(e.packetMessages[:]) + clear(e.packetFrames[:]) + clear(e.packetUploads[:]) + e.packetIO.reset() + e.platformIO.releaseReadBuffers() + e.slabPool.purge() +} + +func (e *goEngine) postMessage(message *goMessage) { + if !message.queued.CompareAndSwap(false, true) { + return + } + e.controlStack.push(message) + switch e.engineState.Load() { + case goEngineParked: + e.platformIO.wake() + case goEngineExited: + e.releaseControlQueue() + } +} + +func (e *goEngine) releaseControlQueue() { + message := e.controlStack.head.Swap(nil) + for message != nil { + next := message.next + message.next = nil + message.queued.Store(false) + e.releasePending(message) + message = next + } +} + +func (e *goEngine) drainControlQueue() { + message := e.controlStack.head.Swap(nil) + if message != nil { + e.readBuffersHeld = true + e.idleSince = -1 + } + for message != nil { + next := message.next + message.next = nil + message.queued.Store(false) + e.handleMessage(message) + message = next + } +} + +func (e *goEngine) handleMessage(message *goMessage) { + switch message.kind { + case goMessageStackReset: + e.reclaimPending = true + case goMessageConnDial: + e.handleDial(message.conn) + case goMessageConnEngage: + e.handleEngage(message.conn) + case goMessageConnClose: + e.handleCloseRequest(message.conn) + case goMessageConnReadShut: + e.handleReadShut(message.conn) + case goMessageConnRetransmit: + e.handleRetransmitArm(message.conn) + case goMessageConnWindow: + e.handleWindowUpdate(message.conn) + case goMessageConnTransmit: + e.handleTransmitRequest(message.conn) + case goMessageConnBlocked: + e.handleTransmitBlocked(message.conn) + case goMessageConnDropped: + e.handleDroppedFrames(message.conn) + case goMessageConnPacing: + e.handlePacingRequest(message.conn) + case goMessageConnThrottle: + if !message.conn.dead { + message.conn.wakeTransmitter() + } + case goMessageConnSplice: + e.handleSpliceEngage(message.conn) + case goMessageConnKeepalive: + e.handleKeepaliveUpdate(message.conn) + case goMessageConnTransmitted: + e.handleTransmitted(message.conn) + case goMessagePacketSplice: + e.handlePacketSpliceEngage(message.packet) + case goMessagePacketClose: + e.handlePacketSpliceClose(message.packet) + case goMessageUDPOpen: + e.handleUDPOpen(message.socket) + case goMessageUDPClose: + e.handleUDPClose(message.socket) + case goMessageListenOpen: + e.handleListenOpen(message.listener) + case goMessageListenClose: + e.handleListenClose(message.listener) + case goMessageInject: + e.processInjected() + } +} + +func (e *goEngine) inject(packet []byte, meta ForwardFrameMeta) { + if e.injectCount.Add(1) > goInjectQueueLimit { + e.injectCount.Add(-1) + e.droppedInjectFrames.record(e.stack.logger, "frames handed over between queues") + return + } + options := e.packetReadOptions.Load() + frame := &goInjectedFrame{buffer: options.NewBufferSize(len(packet)), meta: meta} + common.Must1(frame.buffer.Write(packet)) + options.PostReturn(frame.buffer) + e.injectStack.push(frame) + e.postMessage(&e.injectMessage) +} + +func (e *goEngine) processInjected() { + for frame := e.injectStack.popAll(); frame != nil; { + next := frame.next + e.injectCount.Add(-1) + frame.buffer = e.packetReadOptions.Load().Copy(frame.buffer) + e.processFrame(&goFrame{buffer: frame.buffer, meta: frame.meta}) + e.flushPacketUploads() + frame.buffer.Release() + frame = next + } +} + +func (e *goEngine) releaseInjected() { + for frame := e.injectStack.popAll(); frame != nil; { + next := frame.next + e.injectCount.Add(-1) + frame.buffer.Release() + frame = next + } +} + +func (e *goEngine) handleEngage(conn *GoConn) { + if conn.dead || conn.phase != goPhaseJudged { + return + } + conn.access.Lock() + conn.phase = goPhaseEngaged + conn.access.Unlock() + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.sendReset(conn) + e.detachConn(conn, E.Cause(err, "go: send SYN-ACK"), goDeathAbortLinger) + return + } + conn.handshakeAttempts = 0 + conn.handshakeDeadline = e.coarseTime.Load() + int64(goSynAckRetransmit) + e.rearmTimer(conn) +} + +func (e *goEngine) shutdown() { + e.closeAllPacketSplices() + e.closeAllUDPSockets() + e.closeAllListeners() + for index := range e.reassemblyEntries { + e.reassemblyEntries[index].discard() + } + e.releaseReadBuffers() + unreset := e.closeAllFlows() + for conn := e.dyingList; conn != nil; conn = conn.dyingNext { + e.spliceDetach(conn, net.ErrClosed) + conn.access.Lock() + conn.drainable = false + conn.access.Unlock() + } + deadline := e.coarseTime.Load() + int64(goShutdownBudget) + for len(unreset) > 0 { + e.releaseReadBuffers() + remaining := deadline - e.refreshCoarseTime() + if remaining <= 0 { + break + } + waiters, err := e.platformIO.armTransmitWritable(nil) + if err != nil || waiters == nil { + time.Sleep(time.Millisecond) + } else { + e.platformIO.wait(min(time.Duration(remaining), 10*time.Millisecond), nil) + e.transmitWritable = e.platformIO.takeTransmitWritable(e.transmitWritable) + if len(e.transmitWritable) == 0 { + time.Sleep(time.Millisecond) + } + clear(e.transmitWritable) + e.transmitWritable = e.transmitWritable[:0] + } + kept := unreset[:0] + for _, conn := range unreset { + if !e.sendReset(conn) { + kept = append(kept, conn) + } + } + unreset = kept + } + for { + e.reapDying() + if e.dyingList == nil || e.refreshCoarseTime() >= deadline { + return + } + time.Sleep(time.Millisecond) + } +} + +func (e *goEngine) processBurst() error { + remaining := goReadBatch + budget := goEngineBurstBytes + e.tunPending = false + for remaining > 0 && budget > 0 { + count, drained, err := e.platformIO.readBurst(e.frames[:remaining], *e.packetReadOptions.Load()) + if err != nil { + return err + } + for index := range count { + budget -= e.frames[index].buffer.Len() + e.processFrame(&e.frames[index]) + } + e.flushPacketUploads() + if drained || count == 0 { + return nil + } + remaining -= count + } + e.tunPending = true + return nil +} + +func (e *goEngine) processFrame(frame *goFrame) { + if frame.meta.gsoType == goUDPGSOType { + e.processUDPSegments(frame) + return + } + parsed, ok := parseForwardPacket(frame.buffer.Bytes()) + if !ok { + return + } + if e.stack.validateChecksum && parsed.ipVersion == 4 && !header.IPv4(parsed.network).IsChecksumValid() { + return + } + if e.dropNonUnicast(&parsed) { + return + } + if parsed.fragment { + if parsed.ipVersion == 4 { + e.reassembleIPv4(frame.buffer.Bytes(), &parsed) + } else { + e.reassembleIPv6(frame.buffer.Bytes(), &parsed) + } + return + } + e.processParsed(frame.buffer, frame.meta, &parsed) +} + +func (e *goEngine) processParsed(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + if parsed.protocol == uint8(header.UDPProtocolNumber) { + if len(parsed.transport) < header.UDPMinimumSize { + return + } + length := int(header.UDP(parsed.transport).Length()) + if length < header.UDPMinimumSize || length > len(parsed.transport) { + return + } + parsed.transport = parsed.transport[:length] + } + if e.stack.validateChecksum && !goValidateTransportChecksum(parsed) { + return + } + packet := buffer.Bytes() + if e.handleLoopbackHairpin(packet, meta, parsed) { + return + } + if parsed.protocol == uint8(header.UDPProtocolNumber) && parsed.hasFlow { + if socket := e.udpSockets[parsed.destination]; socket != nil { + e.inputUDPSocket(socket, parsed) + return + } + if len(e.stack.engines) > 1 { + if owner := e.stack.directory.lookupUDPSocket(parsed.destination); owner != nil && owner.engine != e { + owner.engine.inject(packet, meta) + return + } + } + writer := e.packetFlows[parsed.flowKey()] + if writer != nil && writer.splice != nil { + conn := writer.conn.Load() + if conn != nil && !conn.isClosed() && conn.service.state.Load() == udpNatStateStarted { + e.packetSpliceInput(writer, buffer, parsed) + return + } + } + } + establishedTCP := parsed.protocol == uint8(header.TCPProtocolNumber) && parsed.hasFlow && !parsed.isPureTCPSyn() + if establishedTCP { + conn := e.flows[parsed.flowKey()] + if conn != nil { + e.inputTCP(conn, parsed) + return + } + } else if parsed.hasFlow && parsed.isPureTCPSyn() && e.lookupListener(parsed.destination) != nil { + e.demuxL4(buffer, meta, parsed) + return + } + if e.dispatchStage != nil && e.dispatchStage.DispatchParsed(packet, meta, parsed) { + return + } + if establishedTCP && len(e.stack.engines) > 1 { + owner := e.stack.directory.lookup(parsed.flowKey()) + if owner != nil && owner.engine != e { + owner.engine.inject(packet, meta) + return + } + } + e.demuxL4(buffer, meta, parsed) +} + +func goValidateTransportChecksum(parsed *forwardPacket) bool { + transport := parsed.transport + switch parsed.protocol { + case uint8(header.TCPProtocolNumber): + if len(transport) < header.TCPMinimumSize { + return false + } + case uint8(header.UDPProtocolNumber): + if len(transport) < header.UDPMinimumSize { + return false + } + udpHeader := header.UDP(transport) + length := int(udpHeader.Length()) + if length < header.UDPMinimumSize || length > len(transport) { + return false + } + if udpHeader.Checksum() == 0 { + return parsed.ipVersion == 4 + } + transport = transport[:length] + case uint8(header.ICMPv4ProtocolNumber): + return len(transport) >= header.ICMPv4MinimumSize && checksum.Checksum(transport, 0) == 0xffff + case uint8(header.ICMPv6ProtocolNumber): + if len(transport) < header.ICMPv6MinimumSize { + return false + } + default: + return true + } + sourceAddress, destinationAddress := parsed.addressSlices() + partial := header.PseudoHeaderChecksum(tcpip.TransportProtocolNumber(parsed.protocol), sourceAddress, destinationAddress, uint16(len(transport))) + return checksum.Checksum(transport, partial) == 0xffff +} + +func (e *goEngine) demuxL4(buffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + switch parsed.protocol { + case uint8(header.TCPProtocolNumber): + e.demuxTCP(buffer.Bytes(), parsed) + case uint8(header.UDPProtocolNumber): + e.demuxUDP(buffer, meta, parsed) + case uint8(header.ICMPv4ProtocolNumber), uint8(header.ICMPv6ProtocolNumber): + if parsed.isICMPError() { + e.inputICMPError(parsed) + return + } + e.answerEcho(buffer.Bytes(), parsed) + } +} + +func (e *goEngine) dropNonUnicast(parsed *forwardPacket) bool { + destination := parsed.destination.Addr() + if parsed.ipVersion == 4 && destination == e.stack.broadcastAddr { + return true + } + return !destination.IsGlobalUnicast() +} + +func (e *goEngine) handleLoopbackHairpin(packet []byte, meta ForwardFrameMeta, parsed *forwardPacket) bool { + if parsed.protocol != uint8(header.TCPProtocolNumber) || len(parsed.transport) < header.TCPMinimumSize { + return false + } + destination := parsed.destination.Addr() + var loopbackAddresses []netip.Addr + if parsed.ipVersion == 4 { + loopbackAddresses = e.stack.inet4LoopbackAddress + } else { + loopbackAddresses = e.stack.inet6LoopbackAddress + } + if !slices.Contains(loopbackAddresses, destination) { + return false + } + tcpHdr := header.TCP(parsed.transport) + if parsed.ipVersion == 4 { + rewriteIPv4TCP(header.IPv4(parsed.network), tcpHdr, false, + destination, 0, false, + parsed.source.Addr(), 0, false) + } else { + rewriteIPv6TCP(header.IPv6(parsed.network), tcpHdr, false, + destination, 0, false, + parsed.source.Addr(), 0, false) + } + err := e.platformIO.writeFrame(e.singleFrame(packet), meta) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: hairpin write")) + } + return true +} + +func (e *goEngine) refreshCoarseTime() int64 { + now := int64(time.Since(e.epoch)) + e.coarseTime.Store(now) + return now +} + +const ( + goReassemblyTimeout = 3 * time.Second + goReassemblyHeadroom = 64 + goReassemblyCapacity = goReassemblyHeadroom + 65535 + goReassemblyMaxRanges = 8 +) + +type goReassemblyRange struct { + start int + end int +} + +type goReassemblyEntry struct { + active bool + headerSeen bool + ipVersion uint8 + protocol uint8 + source netip.Addr + destination netip.Addr + ident uint32 + deadline int64 + headerLength int + totalLength int + buffer *buf.Buffer + headroom int + ranges [goReassemblyMaxRanges]goReassemblyRange + rangeCount int +} + +func (r *goReassemblyEntry) discard() { + r.active = false + r.buffer.Release() + r.buffer = nil +} + +func (e *goEngine) reassembleIPv4(packet []byte, parsed *forwardPacket) { + ipHdr := header.IPv4(parsed.network) + headerLength := int(ipHdr.HeaderLength()) + if headerLength < header.IPv4MinimumSize || headerLength > len(packet) || headerLength > goReassemblyHeadroom { + return + } + entry := e.reassemblyEntry(4, uint8(ipHdr.TransportProtocol()), parsed.source.Addr(), parsed.destination.Addr(), uint32(ipHdr.ID())) + offset := int(ipHdr.FragmentOffset()) + if offset == 0 { + copy(entry.buffer.Range(entry.headroom-headerLength, entry.headroom), packet[:headerLength]) + entry.headerLength = headerLength + entry.headerSeen = true + } + e.reassemblyAdd(entry, offset, ipHdr.Payload(), ipHdr.More()) +} + +func (e *goEngine) reassembleIPv6(packet []byte, parsed *forwardPacket) { + ipHdr := header.IPv6(parsed.network) + if ipHdr.NextHeader() != header.IPv6FragmentHeader { + return + } + fragHdr := header.IPv6Fragment(ipHdr.Payload()) + if !fragHdr.IsValid() { + return + } + entry := e.reassemblyEntry(6, fragHdr.NextHeader(), parsed.source.Addr(), parsed.destination.Addr(), fragHdr.ID()) + if !entry.headerSeen { + copy(entry.buffer.Range(entry.headroom-header.IPv6MinimumSize, entry.headroom), packet[:header.IPv6MinimumSize]) + entry.headerLength = header.IPv6MinimumSize + entry.headerSeen = true + } + e.reassemblyAdd(entry, int(fragHdr.FragmentOffset())*8, fragHdr.Payload(), fragHdr.More()) +} + +func (e *goEngine) reassemblyEntry(ipVersion uint8, protocol uint8, source netip.Addr, destination netip.Addr, ident uint32) *goReassemblyEntry { + now := e.coarseTime.Load() + var free *goReassemblyEntry + var oldest *goReassemblyEntry + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if entry.active && entry.deadline <= now { + entry.discard() + } + if !entry.active { + if free == nil { + free = entry + } + continue + } + if entry.ipVersion == ipVersion && entry.protocol == protocol && entry.ident == ident && + entry.source == source && entry.destination == destination { + return entry + } + if oldest == nil || entry.deadline < oldest.deadline { + oldest = entry + } + } + entry := free + if entry == nil { + entry = oldest + } + options := e.packetReadOptions.Load() + bufferSize := options.FrontHeadroom + goReassemblyCapacity + options.RearHeadroom + if entry.buffer == nil || entry.buffer.RawCap() < bufferSize { + entry.buffer.Release() + entry.buffer = buf.NewSize(bufferSize) + } else { + entry.buffer.Reset() + } + entry.headroom = options.FrontHeadroom + goReassemblyHeadroom + entry.active = true + entry.headerSeen = false + entry.ipVersion = ipVersion + entry.protocol = protocol + entry.source = source + entry.destination = destination + entry.ident = ident + entry.deadline = now + int64(goReassemblyTimeout) + entry.headerLength = 0 + entry.totalLength = -1 + entry.rangeCount = 0 + if e.reassemblyTickNode.slot == nil { + e.wheel.schedule(&e.reassemblyTickNode, entry.deadline) + } + return entry +} + +func (e *goEngine) reassemblyAdd(entry *goReassemblyEntry, offset int, fragment []byte, more bool) { + if len(fragment) == 0 { + return + } + end := offset + len(fragment) + if end > 65535 { + entry.discard() + return + } + if !more { + if entry.totalLength >= 0 && entry.totalLength != end { + entry.discard() + return + } + entry.totalLength = end + } + if entry.totalLength >= 0 && end > entry.totalLength { + entry.discard() + return + } + copy(entry.buffer.Range(entry.headroom+offset, entry.headroom+end), fragment) + if !reassemblyMergeRanges(entry, offset, end) { + entry.discard() + return + } + if entry.headerSeen && entry.totalLength >= 0 && entry.rangeCount == 1 && + entry.ranges[0].start == 0 && entry.ranges[0].end == entry.totalLength { + e.completeReassembly(entry) + } +} + +func reassemblyMergeRanges(entry *goReassemblyEntry, start int, end int) bool { + count := entry.rangeCount + index := 0 + for index < count && entry.ranges[index].end < start { + index++ + } + if index == count || entry.ranges[index].start > end { + if count == goReassemblyMaxRanges { + return false + } + copy(entry.ranges[index+1:count+1], entry.ranges[index:count]) + entry.ranges[index] = goReassemblyRange{start: start, end: end} + entry.rangeCount = count + 1 + return true + } + mergedStart := min(start, entry.ranges[index].start) + mergedEnd := end + mergeEnd := index + for mergeEnd < count && entry.ranges[mergeEnd].start <= end { + mergedEnd = max(mergedEnd, entry.ranges[mergeEnd].end) + mergeEnd++ + } + entry.ranges[index] = goReassemblyRange{start: mergedStart, end: mergedEnd} + copy(entry.ranges[index+1:], entry.ranges[mergeEnd:count]) + entry.rangeCount = count - (mergeEnd - index) + 1 + return true +} + +func (e *goEngine) completeReassembly(entry *goReassemblyEntry) { + entry.active = false + buffer := entry.buffer + entry.buffer = nil + defer buffer.Release() + if entry.ipVersion == 4 && entry.headerLength+entry.totalLength > 65535 { + return + } + buffer.Resize(entry.headroom-entry.headerLength, entry.headerLength+entry.totalLength) + packet := buffer.Bytes() + if entry.ipVersion == 4 { + ipHdr := header.IPv4(packet) + ipHdr.SetTotalLength(uint16(len(packet))) + ipHdr.SetFlagsFragmentOffset(ipHdr.Flags()&^header.IPv4FlagMoreFragments, 0) + ipHdr.SetChecksum(0) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + } else { + ipHdr := header.IPv6(packet) + ipHdr.SetNextHeader(entry.protocol) + ipHdr.SetPayloadLength(uint16(entry.totalLength)) + } + parsed, ok := parseForwardPacket(packet) + if !ok || parsed.fragment { + return + } + e.processParsed(buffer, ForwardFrameMeta{}, &parsed) + e.flushPacketUploads() +} + +func (e *goEngine) expireReassemblyTick(now int64) { + next := int64(math.MaxInt64) + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + continue + } + if entry.deadline <= now { + entry.discard() + } else { + next = min(next, entry.deadline) + } + } + if next != math.MaxInt64 { + e.wheel.schedule(&e.reassemblyTickNode, next) + } +} + +func (e *goEngine) answerEcho(packet []byte, parsed *forwardPacket) { + switch parsed.protocol { + case uint8(header.ICMPv4ProtocolNumber): + if len(parsed.transport) < header.ICMPv4MinimumSize { + return + } + if !rewriteEchoReplyIPv4(header.IPv4(parsed.network), header.ICMPv4(parsed.transport)) { + return + } + case uint8(header.ICMPv6ProtocolNumber): + if len(parsed.transport) < header.ICMPv6MinimumSize { + return + } + if !rewriteEchoReplyIPv6(header.IPv6(parsed.network), header.ICMPv6(parsed.transport)) { + return + } + default: + return + } + err := e.platformIO.writeDatagram(packet, ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write echo reply")) + } +} diff --git a/stack_go_integration_cc_test.go b/stack_go_integration_cc_test.go new file mode 100644 index 00000000..d5ecbe58 --- /dev/null +++ b/stack_go_integration_cc_test.go @@ -0,0 +1,482 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "fmt" + "io" + "maps" + "net" + "runtime" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + N "github.com/sagernet/sing/common/network" +) + +func kernelTCPResult[T any](t *testing.T, result <-chan T, description string) T { + t.Helper() + select { + case value := <-result: + return value + case <-time.After(kernelTCPTimeout): + t.Fatalf("%s timed out", description) + var value T + return value + } +} + +func kernelTCPWrite(conn net.Conn, payload []byte, buffered bool) error { + if buffered { + buffer := buf.NewSize(len(payload)) + copy(buffer.Extend(len(payload)), payload) + return conn.(N.ExtendedWriter).WriteBuffer(buffer) + } + n, err := conn.Write(payload) + if err == nil && n != len(payload) { + return io.ErrShortWrite + } + return err +} + +func TestGoKernelTCPLifecycle(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + t.Run(congestion, func(algorithmTest *testing.T) { + fixture, traffic := newKernelTCPFixture(algorithmTest, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{}) + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, scenario := range []struct { + name string + size int + peerFirst bool + drain bool + }{ + {name: "empty"}, + {name: "empty_peer_first", peerFirst: true}, + {name: "pending_data", size: 16385}, + {name: "acknowledged_data", size: 16385, drain: true}, + {name: "peer_first", size: 16385, peerFirst: true}, + } { + algorithmTest.Run(fmt.Sprintf("ipv6=%v/%s/%s", ipv6, writer, scenario.name), func(test *testing.T) { + test.Parallel() + client, server, conn := traffic.pair(test, fixture, ipv6, writer) + if scenario.peerFirst { + err := kernelTransfer(client, server, kernelPayload(37, 11), false) + if err != nil { + test.Fatal(err) + } + } + payload := kernelPayload(scenario.size, 13) + if scenario.drain { + written := make(chan error, 1) + go func() { written <- kernelTCPWrite(server, payload, writer == "buffer") }() + received := make([]byte, len(payload)) + _, err := io.ReadFull(client, received) + if err != nil || !bytes.Equal(received, payload) { + test.Fatalf("stream before close: %v", err) + } + err = kernelTCPResult(test, written, "write before close") + if err != nil { + test.Fatal(err) + } + traffic.await(test, conn, "drain before close", func(flow kernelTCPFlow) bool { + return flow.acked >= uint64(len(payload)+1) && conn.sendUnacked.Load() == conn.sentTail.Load() + }) + payload = nil + } + err := kernelTransfer(server, client, payload, writer == "buffer") + if err != nil { + test.Fatal("server half-close:", err) + } + if !scenario.peerFirst { + err = kernelTransfer(client, server, kernelPayload(37, 17), false) + if err != nil { + test.Fatal("reply after half-close:", err) + } + } + traffic.await(test, conn, "graceful close", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.acked >= flow.fin && flow.peerFIN && conn.closed() + }) + }) + } + } + } + }) + } +} + +func TestGoKernelTCPSendBudget(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, phase := range []string{"ack", "control"} { + for _, overlap := range []int{0, goInitialWindow / 2, goInitialWindow - 1} { + for _, buffered := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/overlap=%d/buffer=%v", congestion, phase, overlap, buffered), func(test *testing.T) { + test.Parallel() + var stage atomic.Int32 + var window atomic.Uint32 + entered := make(chan struct{}) + resume := make(chan struct{}) + idle := make(chan struct{}) + release := make(chan struct{}) + checkpoint := func(current string, conn *GoConn) { + if current == phase && stage.CompareAndSwap(0, 1) { + close(entered) + <-resume + } + if current == "controlled" && stage.CompareAndSwap(1, 2) { + window.Store(conn.congestionWindow) + } + if (current == "idle" || current == "receive") && stage.CompareAndSwap(2, 3) { + close(idle) + <-release + } + } + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{checkpoint: checkpoint}) + resumeOnce := sync.OnceFunc(func() { close(resume) }) + releaseOnce := sync.OnceFunc(func() { close(release) }) + test.Cleanup(resumeOnce) + test.Cleanup(releaseOnce) + client, server := fixture.pair(test, false) + mss := int(server.effectiveMSS.Load()) + err := kernelTCPWrite(server, kernelPayload(mss, 19), buffered) + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(client, make([]byte, mss)) + if err != nil { + test.Fatal(err) + } + kernelTCPResult(test, entered, "ACK checkpoint") + overlapping := kernelPayload(overlap*mss, 23) + err = kernelTCPWrite(server, overlapping, buffered) + resumeOnce() + if err != nil { + test.Fatal(err) + } + kernelTCPResult(test, idle, "ACK completion") + allowance := int(window.Load()) * mss + payload := kernelPayload(2*allowance, 29) + err = kernelTCPWrite(server, payload, buffered) + if err != nil { + test.Fatal(err) + } + flow := traffic.snapshot(server.key) + if flow.failure != "" || flow.sent-flow.acked > uint64(allowance) { + test.Fatalf("send budget: window=%d bytes, unacknowledged wire data=%d bytes, failure=%q; recent TCP events: %+v", allowance, flow.sent-flow.acked, flow.failure, flow.events) + } + releaseOnce() + expected := slices.Concat(overlapping, payload) + received := make([]byte, len(expected)) + _, err = io.ReadFull(client, received) + if err != nil || !bytes.Equal(received, expected) { + test.Fatalf("stream after ACK backpressure: %v", err) + } + traffic.await(test, server, "send budget stream ACK processing", func(flow kernelTCPFlow) bool { + return flow.processed >= uint64(1+mss+len(expected)) + }) + }) + } + } + } + } +} + +func TestGoKernelTCPWaiting(t *testing.T) { + previous := runtime.GOMAXPROCS(2) + defer runtime.GOMAXPROCS(previous) + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, buffered := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/buffer=%v", congestion, buffered), func(test *testing.T) { + var stage atomic.Int32 + ready := make(chan struct{}) + checkpoint := func(phase string, _ *GoConn) { + if phase == "controlled" { + stage.CompareAndSwap(0, 1) + } + if phase == "idle" && stage.CompareAndSwap(1, 2) { + close(ready) + } + } + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{receiveDelay: 100 * time.Millisecond, checkpoint: checkpoint}) + client, server := fixture.pair(test, false) + size := goInitialWindow * int(server.effectiveMSS.Load()) + warmup := kernelPayload(size, 31) + payload := kernelPayload(2*size, 37) + readResult := make(chan error, 1) + go func() { + received := make([]byte, len(warmup)+len(payload)) + _, readErr := io.ReadFull(client, received) + if readErr == nil && !bytes.Equal(received, slices.Concat(warmup, payload)) { + readErr = io.ErrUnexpectedEOF + } + readResult <- readErr + }() + err := kernelTCPWrite(server, warmup, buffered) + if err != nil { + test.Fatal(err) + } + traffic.await(test, server, "warmup ACK", func(flow kernelTCPFlow) bool { return flow.acked >= uint64(size+1) }) + kernelTCPResult(test, ready, "warmup congestion update") + cpuStart := kernelProcessCPU(test) + start := time.Now() + err = kernelTCPWrite(server, payload, buffered) + if err != nil { + test.Fatal(err) + } + err = kernelTCPResult(test, readResult, "paced stream delivery") + if err != nil { + test.Fatal(err) + } + if server.congestion.pacing { + flow := traffic.snapshot(server.key) + var first, last time.Duration + var delivered, rate uint64 + for _, event := range flow.events { + if !event.outgoing || event.sequence < uint64(size+1) || event.end == event.sequence { + continue + } + if delivered == 0 { + first = event.at + } + last = event.at + delivered += event.end - event.sequence + rate = max(rate, event.rate) + } + allowed := rate*uint64(last-first+2*time.Millisecond)/uint64(time.Second) + 2*uint64(server.effectiveMSS.Load()) + if delivered == 0 || delivered > allowed { + test.Errorf("pacing envelope: sent=%d bytes over %s at up to %d bytes/s, allowed=%d; recent TCP events: %+v", delivered, last-first, rate, allowed, flow.events) + } + } + traffic.await(test, server, "paced stream ACK processing", func(flow kernelTCPFlow) bool { return flow.processed >= uint64(3*size+1) }) + elapsed := time.Since(start) + cpu := kernelProcessCPU(test) - cpuStart + if cpu > max(20*time.Millisecond, elapsed/3) { + test.Errorf("transfer waiting consumed CPU: stream=%d bytes elapsed=%s CPU=%s", len(payload), elapsed, cpu) + } + }) + } + } +} + +func kernelTCPRecoveryFilter(pattern string, start, end uint64, mss, round int) func(kernelTCPEvent) kernelTCPAction { + attempts := make(map[uint64]int) + highestACK := start + ackDropped := false + return func(event kernelTCPEvent) kernelTCPAction { + if !event.outgoing { + if event.flags != header.TCPFlagAck { + return kernelTCPAction{} + } + duplicate := event.ack == highestACK + highestACK = max(highestACK, event.ack) + if pattern == "duplicate_ack" && duplicate { + return kernelTCPAction{copies: 1 << (round + 1)} + } + if pattern == "ack_loss" && !ackDropped { + ackDropped = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + } + if event.sequence < start || event.sequence >= end || event.end <= event.sequence { + return kernelTCPAction{} + } + attempts[event.sequence]++ + if attempts[event.sequence] != 1 { + return kernelTCPAction{} + } + segment := (event.sequence - start) / uint64(mss) + switch pattern { + case "tail": + return kernelTCPAction{drop: event.end == end} + case "head": + return kernelTCPAction{drop: segment == 0} + case "burst", "duplicate_ack": + return kernelTCPAction{drop: segment < 2} + case "holes": + return kernelTCPAction{drop: segment%3 == 0} + case "reorder": + if segment%3 == 0 { + return kernelTCPAction{delay: 10 * time.Millisecond} + } + } + return kernelTCPAction{} + } +} + +func TestGoKernelTCPRecovery(t *testing.T) { + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, negotiation := range []struct { + name string + config kernelTCPConfig + }{ + {name: "sack_timestamps"}, + {name: "sack", config: kernelTCPConfig{disableTimestamps: true}}, + {name: "legacy", config: kernelTCPConfig{disableSACK: true, disableTimestamps: true}}, + } { + t.Run(congestion+"/"+negotiation.name, func(configTest *testing.T) { + configTest.Parallel() + fixture, traffic := newKernelTCPFixture(configTest, kernelStackConfig{mtu: 1500, congestion: congestion}, negotiation.config) + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, pattern := range []string{"tail", "head", "burst", "holes", "reorder", "duplicate_ack", "ack_loss", "fin"} { + configTest.Run(fmt.Sprintf("ipv6=%v/%s/%s", ipv6, writer, pattern), func(test *testing.T) { + test.Parallel() + client, sender, conn := traffic.pair(test, fixture, ipv6, writer) + mss := int(conn.effectiveMSS.Load()) + start := uint64(1) + for round := range 4 { + segments := 8 + if pattern == "tail" && round > 0 { + segments = 1 + } + length := segments*mss - max(round-1, 0)*mss/3 + end := start + uint64(length) + if round > 0 { + traffic.setFilter(conn, kernelTCPRecoveryFilter(pattern, start, end, mss, round)) + } + payload := kernelPayload(length, uint32(61+round)) + client.SetReadDeadline(time.Now().Add(15 * time.Second)) + sender.SetWriteDeadline(time.Now().Add(15 * time.Second)) + err := kernelTCPWrite(sender, payload, writer == "buffer") + if err != nil { + test.Fatal("write:", err) + } + received := make([]byte, length) + n, readErr := io.ReadFull(client, received) + if readErr != nil || !bytes.Equal(received, payload) { + flow := traffic.snapshot(conn.key) + test.Fatalf("round %d: delivered %d/%d bytes: %v; invariant=%q; recent TCP events: %+v", round, n, length, readErr, flow.failure, flow.events[max(0, len(flow.events)-16):]) + } + traffic.await(test, conn, "transfer acknowledgement", func(flow kernelTCPFlow) bool { return flow.processed >= end }) + traffic.setFilter(conn, nil) + start = end + } + if pattern == "fin" { + dropped := false + traffic.setFilter(conn, func(event kernelTCPEvent) kernelTCPAction { + if event.outgoing && event.flags&header.TCPFlagFin != 0 && !dropped { + dropped = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + }) + } + client.SetReadDeadline(time.Now().Add(kernelTCPTimeout)) + err := N.CloseWrite(sender) + if err != nil { + test.Fatal(err) + } + var extra [1]byte + _, err = client.Read(extra[:]) + if err != io.EOF { + test.Fatalf("stream end: %v", err) + } + flow := traffic.await(test, conn, "FIN acknowledgement", func(flow kernelTCPFlow) bool { return flow.fin != 0 && flow.acked >= flow.fin }) + if flow.faults == 0 { + test.Fatal("transfer completed without exercising the network fault") + } + }) + } + } + } + }) + } + } +} + +func TestGoKernelTCPTransmitLifetime(t *testing.T) { + devices := make(chan struct{}, 8) + for _, congestion := range slices.Sorted(maps.Keys(goCongestionRegistry)) { + for _, ipv6 := range []bool{false, true} { + for _, writer := range []string{"write", "buffer", "splice"} { + for _, shutdown := range []string{"peer_reset", "stack_close"} { + for _, shape := range []string{"first", "short", "partial", "multiple"} { + t.Run(fmt.Sprintf("%s/ipv6=%v/%s/%s/%s", congestion, ipv6, writer, shutdown, shape), func(test *testing.T) { + test.Parallel() + devices <- struct{}{} + test.Cleanup(func() { <-devices }) + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500, congestion: congestion}, kernelTCPConfig{}) + client, sender, conn := traffic.pair(test, fixture, ipv6, writer) + barrier := newKernelTCPBarrier(test) + mss := int(conn.effectiveMSS.Load()) + prefix := mss + length := 1 + switch shape { + case "first": + prefix = 0 + case "partial": + length = mss - 1 + case "multiple": + length = 2*mss + 1 + } + paused := false + traffic.setFilter(conn, func(event kernelTCPEvent) kernelTCPAction { + if !event.outgoing && event.flags == header.TCPFlagAck { + return kernelTCPAction{drop: true} + } + if event.outgoing && event.end > event.sequence && event.sequence >= uint64(1+prefix) && !paused { + paused = true + return kernelTCPAction{pause: barrier} + } + return kernelTCPAction{} + }) + if prefix > 0 { + err := kernelTCPWrite(sender, kernelPayload(prefix, 71), writer == "buffer") + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(client, make([]byte, prefix)) + if err != nil { + test.Fatal(err) + } + } + written := make(chan error, 1) + go func() { written <- kernelTCPWrite(sender, kernelPayload(length, 73), writer == "buffer") }() + kernelTCPResult(test, barrier.entered, "device write") + closed := make(chan error, 1) + go func() { + if shutdown == "stack_close" { + closed <- fixture.stack.Close() + return + } + closeErr := client.SetLinger(0) + if closeErr == nil { + closeErr = client.Close() + } + closed <- closeErr + }() + if shutdown == "stack_close" { + traffic.await(test, conn, "stack shutdown started", func(kernelTCPFlow) bool { return fixture.stack.closed.Load() }) + } else { + err := kernelTCPResult(test, closed, "peer socket reset") + if err != nil { + test.Fatal(err) + } + } + flow := traffic.snapshot(conn.key) + if flow.writes == 0 || flow.released || flow.failure != "" { + test.Fatalf("shutdown did not overlap an active device write: writes=%d released=%v failure=%q", flow.writes, flow.released, flow.failure) + } + barrier.once.Do(func() { close(barrier.resume) }) + kernelTCPResult(test, written, "write cancellation") + if shutdown == "stack_close" { + err := kernelTCPResult(test, closed, "stack shutdown") + if err != nil { + test.Fatal(err) + } + } + traffic.await(test, conn, "connection reclamation", func(flow kernelTCPFlow) bool { return flow.released }) + }) + } + } + } + } + } +} diff --git a/stack_go_integration_cpu_unix_test.go b/stack_go_integration_cpu_unix_test.go new file mode 100644 index 00000000..70b3cbf6 --- /dev/null +++ b/stack_go_integration_cpu_unix_test.go @@ -0,0 +1,20 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func kernelProcessCPU(t *testing.T) time.Duration { + t.Helper() + var usage unix.Rusage + err := unix.Getrusage(unix.RUSAGE_SELF, &usage) + if err != nil { + t.Fatal(err) + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} diff --git a/stack_go_integration_cpu_windows_test.go b/stack_go_integration_cpu_windows_test.go new file mode 100644 index 00000000..ab3af2e2 --- /dev/null +++ b/stack_go_integration_cpu_windows_test.go @@ -0,0 +1,20 @@ +package tun + +import ( + "testing" + "time" + + "golang.org/x/sys/windows" +) + +func kernelProcessCPU(t *testing.T) time.Duration { + t.Helper() + var created, exited, kernel, user windows.Filetime + err := windows.GetProcessTimes(windows.CurrentProcess(), &created, &exited, &kernel, &user) + if err != nil { + t.Fatal(err) + } + kernelTicks := uint64(kernel.HighDateTime)<<32 | uint64(kernel.LowDateTime) + userTicks := uint64(user.HighDateTime)<<32 | uint64(user.LowDateTime) + return time.Duration(kernelTicks+userTicks) * 100 +} diff --git a/stack_go_integration_darwin_test.go b/stack_go_integration_darwin_test.go new file mode 100644 index 00000000..0db89016 --- /dev/null +++ b/stack_go_integration_darwin_test.go @@ -0,0 +1,211 @@ +//go:build darwin && !ios + +package tun + +import ( + "bytes" + "fmt" + "io" + "net" + "runtime" + "sync/atomic" + "testing" + "time" + + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +const ( + kernelConnectionRefused = unix.ECONNREFUSED + kernelConnectionReset = unix.ECONNRESET +) + +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, size), + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_SNDBUF, size), + ) +} + +func prepareKernelNetif(t *testing.T, options *Options) { + t.Helper() + socketFD, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, sysprotoControl) + if err != nil { + t.Fatal(err) + } + transferred := false + defer func() { + if !transferred { + unix.Close(socketFD) + } + }() + var controlInfo unix.CtlInfo + copy(controlInfo.Name[:], utunControlName) + err = unix.IoctlCtlInfo(socketFD, &controlInfo) + if err != nil { + t.Fatal(err) + } + var index int + _, err = fmt.Sscanf(options.Name, "utun%d", &index) + if err != nil { + t.Fatal(err) + } + address := &unix.SockaddrCtl{ID: controlInfo.Id, Unit: uint32(index + 1)} + err = unix.Bind(socketFD, address) + if err != nil { + t.Fatal(err) + } + err = unix.SetsockoptInt(socketFD, sysprotoControl, utunOptionEnableNetif, 1) + if err != nil { + t.Fatal(err) + } + err = unix.Connect(socketFD, address) + if err != nil { + t.Fatal(err) + } + options.FileDescriptor = socketFD + transferred = true +} + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + ipv4 := options.Inet4Address[0] + ipv6 := options.Inet6Address[0] + commands := [][]string{ + {"/sbin/ifconfig", options.Name, "mtu", fmt.Sprint(options.MTU)}, + {"/sbin/ifconfig", options.Name, "inet", ipv4.Addr().String(), ipv4.Addr().String(), "netmask", net.IP(net.CIDRMask(ipv4.Bits(), 32)).String(), "up"}, + {"/sbin/ifconfig", options.Name, "inet6", ipv6.String(), "-dad"}, + {"/sbin/route", "-n", "add", "-net", ipv4.Masked().String(), "-interface", options.Name}, + } + for _, command := range commands { + output, err := kernelCommand(command[0], command[1:]...) + if err != nil { + t.Fatalf("%v: %s: %v", command, output, err) + } + } +} + +func TestGoKernelDeviceBackpressure(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + for _, mtu := range []uint32{1500, 9000} { + for _, shutdown := range []bool{false, true} { + t.Run(fmt.Sprintf("mtu=%d/shutdown=%v", mtu, shutdown), func(test *testing.T) { + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: mtu, prepare: prepareKernelNetif}) + platform := fixture.stack.engines[0].platformIO.(*goDarwinIO) + if !platform.netif { + test.Fatal("utun netif was not enabled") + } + observed := make(chan struct{}, 1) + stop := make(chan struct{}) + defer close(stop) + go func() { + ticker := time.NewTicker(50 * time.Microsecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if platform.gateBlocked.Load() { + observed <- struct{}{} + return + } + case <-stop: + return + } + } + }() + payload := kernelPayload(32<<20, 223) + written := make(chan error, 8) + read := make(chan error, 8) + start := make(chan struct{}) + var clients []net.Conn + for index := range 8 { + var client, server net.Conn + splice := index%2 != 0 + if splice { + client, server, _, _ = fixture.splicePair(test, index%4 >= 2) + } else { + client, server = fixture.pair(test, index%4 >= 2) + } + clients = append(clients, client) + go func() { + <-start + var accepted atomic.Int64 + written <- kernelFlowWrite(server, payload, !splice, &accepted) + }() + go func() { + <-start + data, err := io.ReadAll(io.LimitReader(client, int64(len(payload)+1))) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("device backpressure payload mismatch") + } + read <- err + }() + } + close(start) + select { + case <-observed: + case <-time.After(3 * time.Second): + test.Fatal("real netif transmit gate never became blocked") + } + if shutdown { + closed := make(chan error, 1) + go func() { closed <- fixture.stack.Close() }() + select { + case err := <-closed: + if err != nil { + test.Fatal("close during device backpressure:", err) + } + case <-time.After(time.Second): + test.Fatal("close during device backpressure exceeded 1 second") + } + } + interrupted := 0 + limit := time.After(12 * time.Second) + if shutdown { + limit = time.After(time.Second) + } + for range 8 { + select { + case err := <-written: + if shutdown { + if E.IsTimeout(err) { + test.Error("device-blocked I/O timed out after close:", err) + } + if err != nil { + interrupted++ + } + } else if err != nil { + test.Error("transfer after device backpressure:", err) + } + case <-limit: + test.Fatal("device-blocked I/O did not complete") + } + } + if shutdown && interrupted == 0 { + test.Error("all writers finished before shutdown") + } + if shutdown { + for _, client := range clients { + client.Close() + } + } + for range 8 { + select { + case err := <-read: + if !shutdown && err != nil { + test.Error("read after device backpressure:", err) + } + case <-limit: + test.Fatal("device backpressure reader did not complete") + } + } + if platform.leakedClusters.total.Load() != 0 { + test.Fatal("utun reported ENOSPC while applying device backpressure") + } + }) + } + } +} diff --git a/stack_go_integration_flow_linux_test.go b/stack_go_integration_flow_linux_test.go new file mode 100644 index 00000000..44b36a36 --- /dev/null +++ b/stack_go_integration_flow_linux_test.go @@ -0,0 +1,444 @@ +//go:build linux && !android + +package tun + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "runtime" + "slices" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +func TestGoKernelWindowClamp(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + cases := []struct { + window int + mss int + loss bool + }{ + {window: 2048}, + {window: 4096, mss: 512}, + {window: 16384, loss: true}, + } + for _, scenario := range cases { + t.Run(fmt.Sprintf("window=%d/mss=%d/loss=%v", scenario.window, scenario.mss, scenario.loss), func(configTest *testing.T) { + size := 256 << 10 + config := kernelStackConfig{mtu: 9000, socketBuffer: 4096, upstreamSocketBuffer: 4096, gso: scenario.loss, multiQueue: scenario.loss} + config.dialer = net.Dialer{Control: func(_, _ string, rawConn syscall.RawConn) error { + var optionErr error + err := rawConn.Control(func(descriptor uintptr) { + optionErr = unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, 4096) + if optionErr != nil { + return + } + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_WINDOW_CLAMP, scenario.window) + if optionErr != nil || scenario.mss == 0 { + return + } + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_MAXSEG, scenario.mss) + }) + return E.Errors(err, optionErr) + }} + if scenario.loss { + size = 1 << 20 + config.upstreamSocketBuffer = 64 << 10 + config.configure = func(test *testing.T, options Options) { + configureKernelNetem(test, options, "delay", "2ms") + } + } + var traffic *kernelTCPHarness + config.handshake = func(conn *GoConn) error { + if scenario.loss { + traffic.setFilter(conn, kernelBidirectionalDataLoss()) + } + return conn.HandshakeSuccess() + } + fixture, traffic := newKernelTCPFixture(configTest, config, kernelTCPConfig{}) + if scenario.loss { + configTest.Cleanup(func() { + traffic.access.Lock() + defer traffic.access.Unlock() + for key, flow := range traffic.flows { + if flow.faults != 2 { + configTest.Errorf("flow %v observed %d targeted data drops, expected 2", key, flow.faults) + } + } + }) + } + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + configTest.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + kernelBackpressure(test, fixture, ipv6, mode, size) + }) + } + } + }) + } +} + +func TestGoKernelReceiveWindowUpdateLoss(t *testing.T) { + fixture, traffic := newKernelTCPFixture(t, kernelStackConfig{mtu: 1500, socketBuffer: 4096}, kernelTCPConfig{}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, server := fixture.pair(test, ipv6) + payload := kernelPayload(512<<10, 131) + written := make(chan error, 1) + go func() { + _, err := client.Write(payload) + if err == nil { + err = client.CloseWrite() + } + written <- err + }() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if server.receiveEdge.Load()-server.receiveAvailable.Load() < 1<= 1< 0 + } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: windowUpdate(event)} + }) + data := make([]byte, len(payload)) + _, err := io.ReadFull(server, data[:available]) + if err != nil { + test.Fatal(err) + } + traffic.await(test, server, "drop Go receive-window update", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && windowUpdate(event) }) + }) + traffic.setFilter(server, nil) + server.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = io.ReadFull(server, data[available:]) + if err != nil { + test.Fatal("recover Go receive window:", err) + } + if !bytes.Equal(data, payload) { + test.Fatal("receive window recovery corrupted the stream") + } + _, err = server.Read(data[:1]) + if err != io.EOF { + test.Fatalf("stream end: %v", err) + } + err = kernelTCPResult(test, written, "upload after receive-window update loss") + if err != nil { + test.Fatal(err) + } + }) + } +} + +func TestGoKernelSACKReneging(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + fixture, traffic := newKernelTCPFixture(test, kernelStackConfig{mtu: 1500}, kernelTCPConfig{}) + client, server := fixture.pair(test, ipv6) + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: event.outgoing && event.sequence == 1 && event.end > event.sequence} + }) + mss := int(server.effectiveMSS.Load()) + payload := kernelPayload(8*mss, 137) + _, err := server.Write(payload) + if err != nil { + test.Fatal(err) + } + retained := goSackBlock{start: uint64(mss + 1), end: uint64(len(payload) + 1)} + traffic.await(test, server, "kernel SACK of queued out-of-order data", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { + return !event.outgoing && slices.Contains(event.sacks, retained) + }) + }) + err = client.SetReadBuffer(512) + if err != nil { + test.Fatal(err) + } + traffic.setFilter(server, nil) + traffic.await(test, server, "kernel reneging on its SACK interval", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { + return !event.outgoing && event.ack >= retained.start && event.ack < retained.end + }) + }) + err = client.SetReadBuffer(64 << 10) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + data := make([]byte, len(payload)) + n, err := io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("SACK reneging: received=%d/%d: %v", n, len(payload), err) + } + err = server.CloseWrite() + if err != nil { + test.Fatal(err) + } + _, err = client.Read(data[:1]) + if err != io.EOF { + test.Fatalf("stream end after SACK reneging: %v", err) + } + }) + } +} + +func TestGoKernelZeroWindowHalfClose(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + config := kernelStackConfig{mtu: 9000, socketBuffer: 512} + config.dialer = net.Dialer{Control: func(_, _ string, rawConn syscall.RawConn) error { + var optionErr error + err := rawConn.Control(func(descriptor uintptr) { + optionErr = unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, 512) + if optionErr == nil { + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_WINDOW_CLAMP, 1024) + } + }) + return E.Errors(err, optionErr) + }} + fixture, traffic := newKernelTCPFixture(test, config, kernelTCPConfig{}) + client, server := fixture.pair(test, ipv6) + payload := kernelPayload(int(server.sendPermit.Load()&^goPermitWindowBit-1), 139) + _, err := server.Write(payload) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendUnacked.Load() == uint64(len(payload)+1) && server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() { + break + } + time.Sleep(time.Millisecond) + } + if server.sendUnacked.Load() != uint64(len(payload)+1) || server.sendPermit.Load()&^goPermitWindowBit != server.sendUnacked.Load() { + test.Fatalf("window did not close after acknowledging all data: length=%d unacked=%d permit=%d", len(payload), server.sendUnacked.Load(), server.sendPermit.Load()&^goPermitWindowBit) + } + finAcknowledgement := func(event kernelTCPEvent) bool { + return !event.outgoing && event.flags&header.TCPFlagAck != 0 && event.ack == uint64(len(payload)+2) + } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: finAcknowledgement(event)} + }) + err = server.CloseWrite() + if err != nil { + test.Fatal(err) + } + traffic.await(test, server, "FIN sent with zero receive window", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.peerWindow == 0 + }) + err = client.SetReadBuffer(64 << 10) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("half-close data: %v", err) + } + traffic.await(test, server, "drop FIN acknowledgement at a closed receive window", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && finAcknowledgement(event) }) + }) + traffic.setFilter(server, nil) + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = client.Read(data[:1]) + if err != io.EOF { + test.Fatalf("zero-window FIN: %v", err) + } + traffic.await(test, server, "recover lost FIN acknowledgement", func(flow kernelTCPFlow) bool { + return flow.fin != 0 && flow.acked >= flow.fin + }) + err = client.CloseWrite() + if err != nil { + test.Fatal(err) + } + _, err = server.Read(data[:1]) + if err != io.EOF { + test.Fatalf("peer FIN: %v", err) + } + }) + } +} + +func TestGoKernelWindowLimitedRecovery(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 9000}) + client, server := fixture.pair(test, ipv6) + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + output, err := kernelCommand("tc", "qdisc", "add", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") + if err != nil { + test.Fatalf("delay ACKs: %s: %v", output, err) + } + payload := kernelPayload(8<<20, 223) + initial := 4 * int(server.effectiveMSS.Load()) + _, err = server.Write(payload[:initial]) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() > server.sendUnacked.Load()+2*uint64(server.effectiveMSS.Load()) { + break + } + time.Sleep(time.Millisecond) + } + if server.sendPermit.Load()&^goPermitWindowBit != server.sendUnacked.Load() || server.sentTail.Load() <= server.sendUnacked.Load()+2*uint64(server.effectiveMSS.Load()) { + test.Fatal("window did not shrink below outstanding data") + } + completed := make(chan kernelIOResult, 1) + server.SetWriteDeadline(time.Now().Add(2 * time.Second)) + go func() { + n, writeErr := server.Write(payload[initial:]) + completed <- kernelIOResult{n: n + initial, err: writeErr} + }() + deadline = time.Now().Add(time.Second) + for server.writerWaiting.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if server.writerWaiting.Load() == 0 { + test.Fatal("write did not block at the closed window") + } + server.SetWriteDeadline(time.Now()) + var accepted int + select { + case result := <-completed: + if !errors.Is(result.err, os.ErrDeadlineExceeded) || result.n <= 0 || result.n >= len(payload) { + test.Fatalf("interrupt blocked write: %+v", result) + } + accepted = result.n + case <-time.After(time.Second): + test.Fatal("write did not react to deadline") + } + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, "root") + if err != nil { + test.Fatalf("restore ACKs: %s: %v", output, err) + } + err = client.SetReadBuffer(4 << 20) + if err != nil { + test.Fatal(err) + } + rawConn, err := client.SyscallConn() + if err != nil { + test.Fatal(err) + } + var optionErr error + err = rawConn.Control(func(descriptor uintptr) { + optionErr = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_TCP, unix.TCP_WINDOW_CLAMP, 6144) + }) + if err != nil || optionErr != nil { + test.Fatalf("limit receive window: %v %v", err, optionErr) + } + server.SetWriteDeadline(time.Now().Add(4 * time.Second)) + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + go func() { + n, writeErr := server.Write(payload[accepted:]) + if writeErr == nil { + writeErr = server.CloseWrite() + } + completed <- kernelIOResult{n: n + accepted, err: writeErr} + }() + data, err := io.ReadAll(client) + if err != nil || !bytes.Equal(data, payload) { + test.Fatalf("window-limited recovery: received=%d/%d accepted=%d: %v", len(data), len(payload), accepted, err) + } + select { + case result := <-completed: + if result.err != nil || result.n != len(payload) { + test.Fatalf("resume blocked write: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("write remained blocked after recovery") + } + }) + } +} + +func TestGoKernelZeroWindow(t *testing.T) { + fixture, traffic := newKernelTCPFixture(t, kernelStackConfig{mtu: 9000}, kernelTCPConfig{}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, server := fixture.pair(test, ipv6) + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + output, err := kernelCommand("tc", "qdisc", "replace", "dev", fixture.options.Name, "root", "netem", "delay", "50ms") + if err != nil { + test.Fatalf("tc qdisc replace: %s: %v", output, err) + } + payload := kernelPayload(32768, 107) + _, err = server.Write(payload) + if err != nil { + test.Fatal(err) + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if server.sendPermit.Load()&^goPermitWindowBit == server.sendUnacked.Load() && server.sentTail.Load() == server.bufferedTail.Load() { + break + } + time.Sleep(time.Millisecond) + } + unacked := server.sendUnacked.Load() + if server.sendPermit.Load()&^goPermitWindowBit != unacked || server.sentTail.Load() != uint64(len(payload)+1) || unacked <= 1 || unacked >= server.sentTail.Load() { + test.Fatalf("window did not close with only outstanding data: unacked=%d sent=%d buffered=%d permit=%d", unacked, server.sentTail.Load(), server.bufferedTail.Load(), server.sendPermit.Load()&^goPermitWindowBit) + } + windowUpdate := func(event kernelTCPEvent) bool { + return !event.outgoing && event.flags == header.TCPFlagAck && event.window > 0 + } + traffic.setFilter(server, func(event kernelTCPEvent) kernelTCPAction { + return kernelTCPAction{drop: windowUpdate(event)} + }) + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, "root") + if err != nil { + test.Fatalf("tc qdisc del: %s: %v", output, err) + } + err = client.SetReadBuffer(4 << 20) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(client, data[:unacked-1]) + if err != nil { + test.Fatal(err) + } + traffic.await(test, server, "drop kernel receive-window update", func(flow kernelTCPFlow) bool { + return slices.ContainsFunc(flow.events, func(event kernelTCPEvent) bool { return event.dropped && windowUpdate(event) }) + }) + traffic.setFilter(server, nil) + client.SetReadDeadline(time.Now().Add(4 * time.Second)) + _, err = io.ReadFull(client, data[unacked-1:]) + if err != nil { + test.Fatal("recover lost window update:", err) + } + if !bytes.Equal(data, payload) { + test.Fatal("zero-window recovery corrupted the stream") + } + }) + } +} diff --git a/stack_go_integration_lifecycle_linux_test.go b/stack_go_integration_lifecycle_linux_test.go new file mode 100644 index 00000000..e815f63f --- /dev/null +++ b/stack_go_integration_lifecycle_linux_test.go @@ -0,0 +1,327 @@ +//go:build linux && !android + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "regexp" + "runtime" + "testing" + "time" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelHandshakeLoss(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, phase := range []string{"syn_ack", "final_ack"} { + for _, cancelHandshake := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/phase=%s/cancel=%v", ipv6, phase, cancelHandshake), func(test *testing.T) { + test.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + observed := make(chan *GoConn, 1) + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500, ctx: ctx, handshake: func(conn *GoConn) error { + observed <- conn + return conn.HandshakeSuccess() + }}) + direction, parent, flags := "ingress", "ffff:", "0x12" + if phase == "final_ack" { + direction, parent, flags = "root", "1:", "0x10" + } + protocol, offset := "ip", "33" + if ipv6 { + protocol, offset = "ipv6", "53" + } + command := []string{"qdisc", "add", "dev", fixture.options.Name, direction} + if direction == "root" { + command = append(command, "handle", "1:", "prio") + } + output, err := kernelCommand("tc", command...) + if err != nil { + test.Fatalf("handshake qdisc: %s: %v", output, err) + } + test.Cleanup(func() { kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, direction) }) + output, err = kernelCommand("tc", "filter", "add", "dev", fixture.options.Name, "parent", parent, "protocol", protocol, + "u32", "match", "u8", flags, "0x12", "at", offset, "action", "drop") + if err != nil { + test.Fatalf("handshake filter: %s: %v", output, err) + } + port := uint16(20000 + fixture.port.Add(1)) + accepted := make(chan kernelAccept, 1) + fixture.access.Lock() + fixture.tcp[port] = accepted + fixture.access.Unlock() + type dialResult struct { + conn net.Conn + err error + } + dialed := make(chan dialResult, 1) + go func() { + conn, dialErr := fixture.dialer.DialContext(ctx, "tcp", fixture.address(ipv6, port)) + dialed <- dialResult{conn: conn, err: dialErr} + }() + select { + case conn := <-observed: + test.Cleanup(func() { conn.Close() }) + case <-time.After(time.Second): + test.Fatal("SYN did not reach the application") + } + select { + case result := <-accepted: + test.Fatalf("handshake completed while packets were dropped: %v", result.err) + case <-time.After(250 * time.Millisecond): + } + output, err = kernelCommand("tc", "-s", "filter", "show", "dev", fixture.options.Name, "parent", parent) + if err != nil || !regexp.MustCompile(`dropped [1-9][0-9]*`).Match(output) { + test.Fatalf("no handshake packet was dropped: %s: %v", output, err) + } + if cancelHandshake { + cancel() + } else { + output, err = kernelCommand("tc", "qdisc", "del", "dev", fixture.options.Name, direction) + if err != nil { + test.Fatalf("restore handshake traffic: %s: %v", output, err) + } + } + var result kernelAccept + select { + case result = <-accepted: + if cancelHandshake { + if !errors.Is(result.err, context.Canceled) { + test.Fatalf("cancel pending handshake: %v", result.err) + } + } else if result.err != nil { + test.Fatal("recover handshake:", result.err) + } + case <-time.After(2 * time.Second): + test.Fatal("handshake did not react to recovery/cancellation") + } + select { + case peer := <-dialed: + if peer.conn != nil { + defer peer.conn.Close() + } + if !cancelHandshake { + if peer.err != nil { + test.Fatal(peer.err) + } + peer.conn.SetDeadline(time.Now().Add(3 * time.Second)) + err = kernelTransfer(result.conn, peer.conn, kernelPayload(32769, 233), true) + if err != nil { + test.Fatal("stream after handshake loss:", err) + } + } + case <-time.After(time.Second): + test.Fatal("native dial remained pending") + } + }) + } + } + } +} + +func TestGoKernelLinkInterruption(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + t.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500}) + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + runTC := func(args ...string) []byte { + test.Helper() + output, err := kernelCommand("tc", args...) + if err != nil { + test.Fatalf("tc %v: %s: %v", args, output, err) + } + return output + } + for phase := range 3 { + runTC("qdisc", "add", "dev", fixture.options.Name, "root", "netem", "loss", "100%") + runTC("qdisc", "add", "dev", fixture.options.Name, "ingress") + runTC("filter", "add", "dev", fixture.options.Name, "parent", "ffff:", "protocol", "all", + "u32", "match", "u32", "0", "0", "action", "drop") + written := make(chan error, 2) + read := make(chan error, 2) + for direction := range 2 { + sender, receiver := client, server + if direction == 1 { + sender, receiver = server, client + } + payload := kernelPayload(256<<10|123, uint32(239+phase*7+direction)) + go func() { + var err error + if direction == 1 && mode == "buffer" { + buffer := buf.NewSize(len(payload) + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload)) + err = sender.(N.ExtendedWriter).WriteBuffer(buffer) + } else { + _, err = sender.Write(payload) + } + if err == nil && phase == 2 { + err = N.CloseWrite(sender) + } + written <- err + }() + go func() { + data := make([]byte, len(payload)) + _, err := io.ReadFull(receiver, data) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("link recovery payload mismatch") + } + if err == nil && phase == 2 { + _, err = receiver.Read(data[:1]) + if err == io.EOF { + err = nil + } else { + err = E.New("link recovery stream end: ", err) + } + } + read <- err + }() + } + select { + case err := <-read: + test.Fatalf("phase %d: stream completed during link outage: %v", phase, err) + case <-time.After(250 * time.Millisecond): + } + dropPattern := regexp.MustCompile(`dropped [1-9][0-9]*`) + deadline := time.Now().Add(2 * time.Second) + for { + bothDropped := true + for _, output := range [][]byte{ + runTC("-s", "qdisc", "show", "dev", fixture.options.Name), + runTC("-s", "filter", "show", "dev", fixture.options.Name, "parent", "ffff:"), + } { + if dropPattern.Match(output) { + continue + } + if time.Now().After(deadline) { + test.Fatalf("phase %d: link outage did not drop traffic: %s", phase, output) + } + bothDropped = false + } + if bothDropped { + break + } + select { + case err := <-read: + test.Fatalf("phase %d: stream completed during link outage: %v", phase, err) + case <-time.After(20 * time.Millisecond): + } + } + runTC("qdisc", "del", "dev", fixture.options.Name, "root") + runTC("qdisc", "del", "dev", fixture.options.Name, "ingress") + for range 2 { + err := kernelTCPResult(test, written, "write through link outage") + if err != nil { + test.Fatal("write through link outage:", err) + } + err = kernelTCPResult(test, read, "read after link recovery") + if err != nil { + test.Fatal("read after link recovery:", err) + } + } + } + }) + } + } +} + +func configureKernelNetem(t *testing.T, options Options, impairment ...string) { + t.Helper() + ifbName := "ifb-" + options.Name + impairment = append(impairment, "limit", "4096") + commands := [][]string{ + {"ip", "link", "add", ifbName, "type", "ifb"}, + {"ip", "link", "set", ifbName, "up"}, + append([]string{"tc", "qdisc", "add", "dev", options.Name, "root", "netem"}, impairment...), + {"tc", "qdisc", "add", "dev", options.Name, "ingress"}, + {"tc", "filter", "add", "dev", options.Name, "parent", "ffff:", "protocol", "all", "u32", "match", "u32", "0", "0", "action", "mirred", "egress", "redirect", "dev", ifbName}, + append([]string{"tc", "qdisc", "add", "dev", ifbName, "root", "netem"}, impairment...), + } + for index, command := range commands { + output, err := kernelCommand(command[0], command[1:]...) + if err != nil { + t.Fatalf("%v: %s: %v", command, output, err) + } + if index == 0 { + t.Cleanup(func() { kernelCommand("ip", "link", "delete", ifbName) }) + } + } +} + +func kernelBidirectionalDataLoss() func(kernelTCPEvent) kernelTCPAction { + var dropped [2]bool + return func(event kernelTCPEvent) kernelTCPAction { + direction := 0 + if event.outgoing { + direction = 1 + } + if event.length > 0 && !dropped[direction] { + dropped[direction] = true + return kernelTCPAction{drop: true} + } + return kernelTCPAction{} + } +} + +func TestGoKernelBidirectionalLoss(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + for _, gso := range []bool{false, true} { + for _, multiQueue := range []bool{false, true} { + t.Run(fmt.Sprintf("gso=%v/mq=%v", gso, multiQueue), func(offloadTest *testing.T) { + fixture, traffic := newKernelTCPFixture(offloadTest, kernelStackConfig{ + mtu: 1500, gso: gso, multiQueue: multiQueue, + configure: func(test *testing.T, options Options) { + configureKernelNetem(test, options, "delay", "2ms", "1ms", "loss", "1%", "reorder", "10%", "50%") + }, + }, kernelTCPConfig{}) + for _, ipv6 := range []bool{false, true} { + for _, splice := range []bool{false, true} { + offloadTest.Run(fmt.Sprintf("ipv6=%v/splice=%v", ipv6, splice), func(flowTest *testing.T) { + flowTest.Parallel() + writer := "write" + if splice { + writer = "splice" + } + client, server, conn := traffic.pair(flowTest, fixture, ipv6, writer) + traffic.setFilter(conn, kernelBidirectionalDataLoss()) + payload := kernelPayload(1<<20, 43) + response := kernelPayload(1<<20, 47) + result := make(chan error, 1) + go func() { result <- kernelTransfer(client, server, payload, false) }() + err := kernelTransfer(server, client, response, !splice) + if err != nil { + flowTest.Error("download:", err) + } + err = kernelTCPResult(flowTest, result, "upload through bidirectional loss") + if err != nil { + flowTest.Error("upload:", err) + } + observed := traffic.snapshot(conn.key) + if observed.faults != 2 { + flowTest.Fatalf("bidirectional transfer observed %d targeted data drops, expected 2", observed.faults) + } + }) + } + } + }) + } + } +} diff --git a/stack_go_integration_lifecycle_test.go b/stack_go_integration_lifecycle_test.go new file mode 100644 index 00000000..28f3fd76 --- /dev/null +++ b/stack_go_integration_lifecycle_test.go @@ -0,0 +1,493 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "runtime" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelHandshake(t *testing.T) { + for _, ipv6 := range []bool{false, true} { + for _, action := range []string{"accept", "reject", "close"} { + t.Run(fmt.Sprintf("ipv6=%v/action=%s", ipv6, action), func(test *testing.T) { + test.Parallel() + observed := make(chan *GoConn, 1) + var observe sync.Once + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + fixture := newKernelStackFixture(test, kernelStackConfig{mtu: 1500, dialer: net.Dialer{Timeout: 5 * time.Second}, handshake: func(conn *GoConn) error { + observe.Do(func() { observed <- conn }) + test.Cleanup(func() { conn.Close() }) + <-release + switch action { + case "reject": + return conn.HandshakeFailure(kernelConnectionRefused) + case "close": + return conn.Close() + default: + return conn.HandshakeSuccess() + } + }}) + test.Cleanup(unblock) + port := uint16(20000 + fixture.port.Add(1)) + fixture.access.Lock() + fixture.tcp[port] = make(chan kernelAccept, 8) + fixture.access.Unlock() + type dialResult struct { + conn net.Conn + err error + } + dialed := make(chan dialResult, 1) + go func() { + conn, err := fixture.dialer.Dial("tcp", fixture.address(ipv6, port)) + dialed <- dialResult{conn: conn, err: err} + }() + var server *GoConn + select { + case server = <-observed: + case <-time.After(time.Second): + test.Fatal("connection did not reach the application") + } + select { + case result := <-dialed: + if result.conn != nil { + result.conn.Close() + } + test.Fatalf("dial completed before application decision: %v", result.err) + case <-time.After(80 * time.Millisecond): + } + unblock() + select { + case result := <-dialed: + if action != "accept" { + if result.conn != nil { + result.conn.Close() + } + if !errors.Is(result.err, kernelConnectionRefused) { + test.Fatalf("application %s: %v", action, result.err) + } + return + } + if result.err != nil { + test.Fatal(result.err) + } + defer result.conn.Close() + result.conn.SetDeadline(time.Now().Add(3 * time.Second)) + err := kernelTransfer(server, result.conn, kernelPayload(32769, 157), true) + if err != nil { + test.Fatal("accepted stream:", err) + } + case <-time.After(6 * time.Second): + test.Fatal("application decision did not complete the dial") + } + }) + } + } +} + +func TestGoKernelHalfClose(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/close_read", ipv6), func(test *testing.T) { + test.Parallel() + client, server := fixture.pair(test, ipv6) + completed := make(chan kernelIOResult, 1) + go func() { + n, err := server.Read(make([]byte, 1)) + completed <- kernelIOResult{n: n, err: err} + }() + select { + case result := <-completed: + test.Fatalf("read did not wait: %+v", result) + case <-time.After(40 * time.Millisecond): + } + err := server.CloseRead() + if err != nil { + test.Fatal(err) + } + select { + case result := <-completed: + if result.n != 0 || result.err != io.EOF { + test.Fatalf("CloseRead: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("CloseRead did not wake reader") + } + uploaded := make(chan error, 1) + go func() { + _, writeErr := client.Write(kernelPayload(1<<20, 173)) + if writeErr == nil { + writeErr = client.CloseWrite() + } + uploaded <- writeErr + }() + err = kernelTransfer(server, client, kernelPayload(256<<10, 179), true) + if err != nil { + test.Fatal("write after CloseRead:", err) + } + select { + case err = <-uploaded: + if err != nil { + test.Fatal("peer upload after CloseRead:", err) + } + case <-time.After(time.Second): + test.Fatal("discarded input blocked the peer") + } + }) + t.Run(fmt.Sprintf("ipv6=%v/upstream_reset", ipv6), func(test *testing.T) { + test.Parallel() + client, upstream, _, _ := fixture.splicePair(test, ipv6) + marker := kernelPayload(1024, 181) + _, err := upstream.Write(marker) + if err != nil { + test.Fatal(err) + } + data := make([]byte, len(marker)) + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, marker) { + test.Fatal("upstream data before reset:", err) + } + upstream.SetLinger(0) + upstream.Close() + client.SetReadDeadline(time.Now().Add(time.Second)) + _, err = client.Read(data[:1]) + if !errors.Is(err, kernelConnectionReset) { + test.Fatalf("upstream reset: %v", err) + } + }) + } +} + +func TestGoKernelClose(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + t.Run("reset", func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + client, conn := fixture.pair(scenarioTest, ipv6) + client.SetLinger(0) + client.Close() + _, err := conn.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + scenarioTest.Errorf("reset ipv6=%v: %v", ipv6, err) + } + } + }) + t.Run("drain_after_fin", func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + client, conn := fixture.pair(scenarioTest, ipv6) + payload := kernelPayload(32769, 41) + _, err := client.Write(payload) + if err != nil { + scenarioTest.Fatal(err) + } + client.CloseWrite() + conn.CloseWrite() + _, err = io.ReadAll(client) + if err != nil { + scenarioTest.Fatal(err) + } + data, err := io.ReadAll(conn) + if err != nil || !bytes.Equal(data, payload) { + scenarioTest.Fatalf("drain ipv6=%v bytes=%d: %v", ipv6, len(data), err) + } + } + }) + for _, splice := range []bool{false, true} { + t.Run(fmt.Sprintf("busy_shutdown/splice=%v", splice), func(scenarioTest *testing.T) { + fixture := newKernelStackFixture(scenarioTest, kernelStackConfig{ + mtu: 1500, multiQueue: runtime.GOOS == "linux", + upstreamSocketBuffer: 4096, + dialer: net.Dialer{Control: func(_, _ string, rawConn syscall.RawConn) error { + var optionErr error + err := rawConn.Control(func(descriptor uintptr) { + optionErr = kernelSetSocketBuffers(descriptor, 4096) + }) + return E.Errors(err, optionErr) + }}, + }) + payload := kernelPayload(32749, 33) + type pendingResult struct { + connection int + operation string + kernelIOResult + } + completed := make(chan pendingResult, 32) + reading := make(chan kernelIOResult, 16) + var progress [16]atomic.Int64 + for index := range 16 { + var client, conn net.Conn + if splice { + client, conn, _, _ = fixture.splicePair(scenarioTest, index%2 != 0) + } else { + client, conn = fixture.pair(scenarioTest, index%2 != 0) + } + client.SetDeadline(time.Now().Add(2 * time.Second)) + conn.SetDeadline(time.Time{}) + go func() { + for { + n, err := conn.Write(payload) + progress[index].Add(int64(n)) + if err != nil { + completed <- pendingResult{connection: index, operation: "write", kernelIOResult: kernelIOResult{n: n, err: err}} + return + } + } + }() + go func() { + var probe [1]byte + n, err := io.ReadFull(conn, probe[:]) + if err == nil && probe[0] != 0x71 { + err = E.New("shutdown read probe corrupted") + } + reading <- kernelIOResult{n: n, err: err} + n, err = conn.Read(probe[:]) + completed <- pendingResult{connection: index, operation: "read", kernelIOResult: kernelIOResult{n: n, err: err}} + }() + _, err := client.Write([]byte{0x71}) + if err != nil { + scenarioTest.Fatal("start blocked reader:", err) + } + var probe [1]byte + _, err = io.ReadFull(client, probe[:]) + if err != nil || probe[0] != payload[0] { + scenarioTest.Fatalf("start blocked writer: byte=%x error=%v", probe[0], err) + } + } + for range 16 { + select { + case result := <-reading: + if result.n != 1 || result.err != nil { + scenarioTest.Fatalf("start blocked reader: %+v", result) + } + case <-time.After(time.Second): + scenarioTest.Fatal("reader did not consume peer probe") + } + } + var previousProgress [16]int64 + stalledAt := time.Now() + backpressureDeadline := time.NewTimer(time.Second) + defer backpressureDeadline.Stop() + observations := time.NewTicker(10 * time.Millisecond) + defer observations.Stop() + for time.Since(stalledAt) < 40*time.Millisecond { + select { + case result := <-completed: + scenarioTest.Fatalf("I/O completed before stack shutdown: %+v", result) + case <-observations.C: + for index := range progress { + accepted := progress[index].Load() + if accepted != previousProgress[index] { + previousProgress[index] = accepted + stalledAt = time.Now() + } + } + case <-backpressureDeadline.C: + scenarioTest.Fatal("writers did not encounter backpressure within 1 second") + } + } + closed := make(chan error, 1) + go func() { closed <- fixture.stack.Close() }() + select { + case err := <-closed: + if err != nil { + scenarioTest.Fatal(err) + } + case <-time.After(time.Second): + scenarioTest.Fatal("stack close did not complete within 1 second") + } + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for range 32 { + select { + case result := <-completed: + if !(E.IsClosed(result.err) || errors.Is(result.err, kernelConnectionReset)) || E.IsTimeout(result.err) || + result.operation == "read" && result.n != 0 || + result.operation == "write" && result.n >= len(payload) { + scenarioTest.Errorf("pending I/O after stack shutdown: %+v", result) + } + case <-deadline.C: + scenarioTest.Fatal("pending I/O did not unblock after stack shutdown") + } + } + }) + } +} + +type kernelIOResult struct { + n int + err error +} + +func kernelCommand(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + command := exec.CommandContext(ctx, name, args...) + command.WaitDelay = time.Second + return command.CombinedOutput() +} + +func TestGoKernelDeadline(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000}) + for _, ipv6 := range []bool{false, true} { + for _, operation := range []string{"read", "wait_read", "write"} { + for _, change := range []string{"advance", "extend", "clear", "close"} { + t.Run(fmt.Sprintf("ipv6=%v/operation=%s/change=%s", ipv6, operation, change), func(test *testing.T) { + test.Parallel() + client, server := fixture.pair(test, ipv6) + writing := operation == "write" + payload := []byte{0x71} + setDeadline := server.SetReadDeadline + if writing { + payload = kernelPayload(8<<20, 151) + setDeadline = server.SetWriteDeadline + err := client.SetReadBuffer(4096) + if err != nil { + test.Fatal(err) + } + } + var waiter N.ReadWaiter + if operation == "wait_read" { + var created bool + waiter, created = bufio.CreateReadWaiter(server) + if !created { + test.Fatal("missing read waiter") + } + waiter.InitializeReadWaiter(N.ReadWaitOptions{FrontHeadroom: 91, RearHeadroom: 73}) + } + read := func() kernelIOResult { + if waiter != nil { + buffer, err := waiter.WaitReadBuffer() + result := kernelIOResult{err: err} + if buffer != nil { + result.n = buffer.Len() + if !bytes.Equal(buffer.Bytes(), payload) { + result.err = E.New("read waiter payload mismatch") + } + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + return result + } + data := make([]byte, len(payload)) + n, err := io.ReadFull(server, data) + if err == nil && !bytes.Equal(data, payload) { + err = E.New("read payload mismatch") + } + return kernelIOResult{n: n, err: err} + } + original := time.Now().Add(300 * time.Millisecond) + if change == "advance" || change == "close" { + original = time.Now().Add(4 * time.Second) + } + setDeadline(original) + completed := make(chan kernelIOResult, 1) + go func() { + if writing { + n, err := server.Write(payload) + completed <- kernelIOResult{n: n, err: err} + } else { + completed <- read() + } + }() + select { + case result := <-completed: + test.Fatalf("I/O did not wait: %+v", result) + case <-time.After(40 * time.Millisecond): + } + switch change { + case "advance": + setDeadline(time.Now()) + case "extend": + setDeadline(time.Now().Add(4 * time.Second)) + case "clear": + setDeadline(time.Time{}) + case "close": + server.Close() + } + accepted := 0 + if change == "advance" || change == "close" { + select { + case result := <-completed: + accepted = result.n + if accepted < 0 || accepted >= len(payload) { + test.Fatalf("blocked I/O accepted %d/%d bytes", accepted, len(payload)) + } + if change == "advance" && !errors.Is(result.err, os.ErrDeadlineExceeded) { + test.Fatalf("updated deadline: %v", result.err) + } + if change == "close" && (!E.IsClosed(result.err) || E.IsTimeout(result.err)) { + test.Fatalf("close pending I/O: %v", result.err) + } + case <-time.After(time.Second): + test.Fatal("pending I/O did not react to deadline/close") + } + if change == "close" { + return + } + setDeadline(time.Now().Add(4 * time.Second)) + go func() { + if writing { + n, err := server.Write(payload[accepted:]) + completed <- kernelIOResult{n: n + accepted, err: err} + } else { + completed <- read() + } + }() + } else { + select { + case result := <-completed: + test.Fatalf("old deadline still affected I/O: %+v", result) + case <-time.After(time.Until(original.Add(40 * time.Millisecond))): + } + } + if writing { + client.SetReadBuffer(4 << 20) + data := make([]byte, len(payload)) + n, err := io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + select { + case result := <-completed: + test.Logf("writer after deadline change: accepted=%d: %v", result.n, result.err) + default: + } + test.Fatalf("write after deadline change: received=%d/%d accepted_before_change=%d: %v", n, len(payload), accepted, err) + } + } else { + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + } + select { + case result := <-completed: + if result.err != nil || result.n != len(payload) { + test.Fatalf("completed I/O: %+v", result) + } + case <-time.After(time.Second): + test.Fatal("I/O remained blocked after peer resumed") + } + }) + } + } + } +} diff --git a/stack_go_integration_linux_test.go b/stack_go_integration_linux_test.go new file mode 100644 index 00000000..6f5022b8 --- /dev/null +++ b/stack_go_integration_linux_test.go @@ -0,0 +1,65 @@ +//go:build linux && !android + +package tun + +import ( + "testing" + "time" + + "github.com/sagernet/netlink" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +const ( + kernelConnectionRefused = unix.ECONNREFUSED + kernelConnectionReset = unix.ECONNRESET +) + +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_RCVBUF, size), + unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_SNDBUF, size), + ) +} + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + link, err := netlink.LinkByName(options.Name) + if err != nil { + t.Fatal(err) + } + for _, prefix := range append(options.Inet4Address, options.Inet6Address...) { + address, parseErr := netlink.ParseAddr(prefix.String()) + if parseErr != nil { + t.Fatal(parseErr) + } + address.Flags = unix.IFA_F_NODAD + err = netlink.AddrAdd(link, address) + if err != nil { + t.Fatal(err) + } + } + err = netlink.LinkSetUp(link) + if err != nil { + t.Fatal(err) + } + for _, prefix := range options.Inet6Address { + deadline := time.Now().Add(time.Second) + for { + routes, routeErr := netlink.RouteGet(prefix.Addr().AsSlice()) + if routeErr != nil { + t.Fatal(routeErr) + } + if len(routes) > 0 && routes[0].Type == unix.RTN_LOCAL { + break + } + if time.Now().After(deadline) { + t.Fatalf("IPv6 local route was not installed: %s", prefix.Addr()) + } + t.Log("waiting for IPv6 local route:", prefix.Addr()) + time.Sleep(time.Millisecond) + } + } +} diff --git a/stack_go_integration_listen_test.go b/stack_go_integration_listen_test.go new file mode 100644 index 00000000..9e0ffa13 --- /dev/null +++ b/stack_go_integration_listen_test.go @@ -0,0 +1,182 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/netip" + "syscall" + "testing" + "time" +) + +func TestGoListenTCP(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, noHandler: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("duplex", func(test *testing.T) { + listener, err := fixture.stack.ListenTCP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer listener.Close() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + dialer := net.Dialer{Timeout: 3 * time.Second} + kernel, err := dialer.Dial("tcp", listener.Addr().String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(10 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + if conn.LocalAddr().String() != listener.Addr().String() { + test.Fatalf("accepted local address %s, listener %s", conn.LocalAddr(), listener.Addr()) + } + if conn.RemoteAddr().String() != kernel.LocalAddr().String() { + test.Fatalf("accepted remote address %s, kernel %s", conn.RemoteAddr(), kernel.LocalAddr()) + } + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, kernel, kernelPayload(4<<20, 31), false) }() + err = kernelTransfer(kernel, conn, kernelPayload(4<<20, 47), false) + if err != nil { + test.Error("upload:", err) + } + err = <-result + if err != nil { + test.Error("download:", err) + } + }) + addressTest.Run("close", func(test *testing.T) { + listener, err := fixture.stack.ListenTCP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + dialer := net.Dialer{Timeout: 3 * time.Second} + kernel, err := dialer.Dial("tcp", listener.Addr().String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + listener.Close() + _, err = kernel.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + test.Fatalf("kernel read on a connection queued at a closed listener: %v", err) + } + _, err = listener.Accept() + if !errors.Is(err, net.ErrClosed) { + test.Fatalf("accept on a closed listener: %v", err) + } + _, err = dialer.Dial("tcp", listener.Addr().String()) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("connect to a closed listener: %v", err) + } + }) + }) + } +} + +func TestGoListenPrecedence(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + port := uint16(30000 + fixture.port.Add(1)) + local := netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), port) + listener, err := fixture.stack.ListenTCP(local) + if err != nil { + test.Fatal(err) + } + defer listener.Close() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = fixture.stack.ListenTCP(local) + if !errors.Is(err, syscall.EADDRINUSE) { + test.Fatalf("second listener on %s: %v", local, err) + } + kernel, err := fixture.dialer.Dial("tcp", local.String()) + if err != nil { + test.Fatal(err) + } + defer kernel.Close() + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + test.Fatal(err) + } + defer conn.Close() + _, err = kernel.Write([]byte("listener")) + if err != nil { + test.Fatal(err) + } + received := make([]byte, 8) + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, err = conn.Read(received) + if err != nil { + test.Fatal(err) + } + if string(received) != "listener" { + test.Fatalf("listener received %q", received) + } + }) + } +} + +func TestGoListenUDPPort(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + port := uint16(30000 + fixture.port.Add(1)) + local := netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), port) + socket, err := fixture.stack.ListenUDP(local) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + if socket.LocalAddr().String() != local.String() { + test.Fatalf("bound to %s, want %s", socket.LocalAddr(), local) + } + _, err = fixture.stack.ListenUDP(local) + if !errors.Is(err, syscall.EADDRINUSE) { + test.Fatalf("second socket on %s: %v", local, err) + } + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + payload := kernelPayload(1200, 21) + _, err = peer.WriteToUDPAddrPort(payload, local) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 2000) + n, source, err := socket.ReadFromUDPAddrPort(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("received %d bytes with corrupted payload", n) + } + _, err = socket.WriteToUDPAddrPort(storage[:n], source) + if err != nil { + test.Fatal(err) + } + n, _, err = peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("reply carried %d bytes with corrupted payload", n) + } + }) + } +} diff --git a/stack_go_integration_memory_darwin_test.go b/stack_go_integration_memory_darwin_test.go new file mode 100644 index 00000000..0d613123 --- /dev/null +++ b/stack_go_integration_memory_darwin_test.go @@ -0,0 +1,15 @@ +//go:build darwin && !ios + +package tun + +import ( + "net/netip" + "testing" +) + +func kernelAddUnreachableRoute(t *testing.T, prefix netip.Prefix) bool { + return false +} + +// net.inet.udp.maxdgram caps a datagram at 9216 bytes. +const kernelMaxDatagram = 9216 diff --git a/stack_go_integration_memory_linux_test.go b/stack_go_integration_memory_linux_test.go new file mode 100644 index 00000000..9c3eb2e0 --- /dev/null +++ b/stack_go_integration_memory_linux_test.go @@ -0,0 +1,29 @@ +//go:build linux && !android + +package tun + +import ( + "net" + "net/netip" + "testing" + + "github.com/sagernet/netlink" + + "golang.org/x/sys/unix" +) + +const kernelMaxDatagram = 60000 + +func kernelAddUnreachableRoute(t *testing.T, prefix netip.Prefix) bool { + t.Helper() + route := &netlink.Route{ + Dst: &net.IPNet{IP: prefix.Addr().AsSlice(), Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen())}, + Type: unix.RTN_UNREACHABLE, + } + err := netlink.RouteAdd(route) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { netlink.RouteDel(route) }) + return true +} diff --git a/stack_go_integration_memory_test.go b/stack_go_integration_memory_test.go new file mode 100644 index 00000000..305c2eb7 --- /dev/null +++ b/stack_go_integration_memory_test.go @@ -0,0 +1,565 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +func kernelListen(t *testing.T, fixture *kernelStackFixture, ipv6 bool) (*net.TCPListener, netip.AddrPort) { + t.Helper() + listener, err := net.ListenTCP("tcp", net.TCPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + return listener, listener.Addr().(*net.TCPAddr).AddrPort() +} + +func kernelAcceptTCP(t *testing.T, listener *net.TCPListener) *net.TCPConn { + t.Helper() + listener.SetDeadline(time.Now().Add(3 * time.Second)) + conn, err := listener.AcceptTCP() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn +} + +func TestGoMemoryDial(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("duplex", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + if conn.RemoteAddr().String() != address.String() { + test.Fatalf("remote address %s, want %s", conn.RemoteAddr(), address) + } + if conn.LocalAddr().String() != server.RemoteAddr().String() { + test.Fatalf("local address %s, kernel peer %s", conn.LocalAddr(), server.RemoteAddr()) + } + payload := kernelPayload(4<<20, 31) + response := kernelPayload(4<<20, 47) + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, server, payload, false) }() + err = kernelTransfer(server, conn, response, false) + if err != nil { + test.Error("download:", err) + } + err = <-result + if err != nil { + test.Error("upload:", err) + } + }) + addressTest.Run("close", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + server := kernelAcceptTCP(test, listener) + _, err = conn.Write([]byte("hello")) + if err != nil { + test.Fatal(err) + } + err = conn.Close() + if err != nil { + test.Fatal(err) + } + data, err := io.ReadAll(server) + if err != nil { + test.Fatal(err) + } + if string(data) != "hello" { + test.Fatalf("kernel received %q", data) + } + }) + addressTest.Run("refused", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + listener.Close() + started := time.Now() + _, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("dial to a closed port returned %v", err) + } + if time.Since(started) > time.Second { + test.Fatalf("refusal took %v", time.Since(started)) + } + }) + addressTest.Run("unreachable", func(test *testing.T) { + prefix := netip.MustParsePrefix("198.19.255.0/24") + destination := netip.MustParseAddrPort("198.19.255.254:9") + if ipv6 { + prefix = netip.MustParsePrefix("fd73:ab91:ffff::/48") + destination = netip.MustParseAddrPort("[fd73:ab91:ffff::254]:9") + } + if !kernelAddUnreachableRoute(test, prefix) { + test.Skip("unreachable routes are not configurable on this platform") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), destination) + if !E.IsMulti(err, syscall.EHOSTUNREACH, syscall.ENETUNREACH) { + test.Fatalf("dial to an unroutable address returned %v", err) + } + }) + addressTest.Run("cancel", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + fixture.bridge.pauseOutbound(true) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + _, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + fixture.bridge.pauseOutbound(false) + if !errors.Is(err, context.DeadlineExceeded) { + test.Fatalf("cancelled dial returned %v", err) + } + listener.SetDeadline(time.Now().Add(500 * time.Millisecond)) + conn, acceptErr := listener.Accept() + if acceptErr == nil { + conn.SetReadDeadline(time.Now().Add(time.Second)) + _, readErr := conn.Read(make([]byte, 1)) + conn.Close() + if !errors.Is(readErr, syscall.ECONNRESET) { + test.Fatalf("stale handshake was not reset: %v", readErr) + } + } + }) + addressTest.Run("retransmit", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + fixture.bridge.outboundDrop.Store(1) + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + if time.Since(started) < goSynRetransmit { + test.Fatalf("handshake completed in %v with the first SYN dropped", time.Since(started)) + } + server := kernelAcceptTCP(test, listener) + err = kernelTransfer(conn, server, kernelPayload(64<<10, 5), false) + if err != nil { + test.Fatal(err) + } + }) + addressTest.Run("stall", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + payload := kernelPayload(2<<20, 61) + fixture.bridge.pauseOutbound(true) + result := make(chan error, 1) + go func() { result <- kernelTransfer(conn, server, payload, false) }() + select { + case err = <-result: + test.Fatalf("transfer finished through a paused link: %v", err) + case <-time.After(300 * time.Millisecond): + } + fixture.bridge.pauseOutbound(false) + err = <-result + if err != nil { + test.Fatal(err) + } + }) + }) + } +} + +func TestGoMemoryUDP(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("listen", func(test *testing.T) { + socket, err := fixture.stack.ListenUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + storage := make([]byte, 65535) + for index, size := range []int{1, 1200, 9000, kernelMaxDatagram} { + payload := kernelPayload(size, uint32(index+1)) + _, err = peer.WriteToUDPAddrPort(payload, socket.LocalAddr().(*net.UDPAddr).AddrPort()) + if err != nil { + test.Fatal(err) + } + n, source, readErr := socket.ReadFrom(storage) + if readErr != nil { + test.Fatalf("size=%d: %v", size, readErr) + } + if source.String() != peer.LocalAddr().String() { + test.Fatalf("size=%d: source %s, want %s", size, source, peer.LocalAddr()) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("size=%d: received %d bytes with corrupted payload", size, n) + } + _, err = socket.WriteTo(payload, source) + if err != nil { + test.Fatal(err) + } + n, _, readErr = peer.ReadFrom(storage) + if readErr != nil { + test.Fatalf("size=%d: reply: %v", size, readErr) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("size=%d: reply carried %d bytes with corrupted payload", size, n) + } + } + }) + addressTest.Run("connected", func(test *testing.T) { + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + stranger, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer stranger.Close() + socket, err := fixture.stack.DialUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0), peer.LocalAddr().(*net.UDPAddr).AddrPort()) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = socket.Write([]byte("ping")) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 64) + n, source, err := peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if string(storage[:n]) != "ping" || source.String() != socket.LocalAddr().String() { + test.Fatalf("kernel received %q from %s", storage[:n], source) + } + _, err = stranger.WriteTo([]byte("stranger"), source) + if err != nil { + test.Fatal(err) + } + _, err = peer.WriteTo([]byte("pong"), source) + if err != nil { + test.Fatal(err) + } + n, err = socket.Read(storage) + if err != nil { + test.Fatal(err) + } + if string(storage[:n]) != "pong" { + test.Fatalf("connected socket received %q", storage[:n]) + } + socket.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + _, err = socket.Read(storage) + if !errors.Is(err, os.ErrDeadlineExceeded) { + test.Fatalf("datagram from another peer leaked into a connected socket: %v", err) + } + }) + addressTest.Run("refused", func(test *testing.T) { + closedPeer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + address := closedPeer.LocalAddr().(*net.UDPAddr).AddrPort() + closedPeer.Close() + socket, err := fixture.stack.DialUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0), address) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + socket.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = socket.Write([]byte("anyone")) + if err != nil { + test.Fatal(err) + } + _, err = socket.Read(make([]byte, 64)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("datagram to a closed port returned %v", err) + } + }) + }) + } +} + +func TestGoMemoryMTUUpdate(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, server := fixture.pair(test, ipv6) + payload := kernelPayload(4<<20, 79) + result := make(chan error, 1) + go func() { result <- kernelTransfer(server, client, payload, false) }() + time.Sleep(20 * time.Millisecond) + fixture.memoryTun.UpdateMTU(576) + err := <-result + if err != nil { + test.Fatal(err) + } + fixture.bridge.outboundLargest.Store(0) + client, server = fixture.pair(test, ipv6) + err = kernelTransfer(server, client, kernelPayload(1<<20, 83), false) + if err != nil { + test.Fatal(err) + } + largest := fixture.bridge.outboundLargest.Load() + if largest > 576 { + test.Fatalf("stack emitted a %d byte packet after the MTU dropped to 576", largest) + } + fixture.memoryTun.UpdateMTU(1500) + }) + } +} + +func TestGoMemoryNoHandler(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true, noHandler: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Run("tcp", func(test *testing.T) { + dialer := net.Dialer{Timeout: 2 * time.Second} + _, err := dialer.Dial("tcp", fixture.address(ipv6, 8080)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("connect to a handler-less stack returned %v", err) + } + }) + addressTest.Run("udp", func(test *testing.T) { + peer, err := net.Dial("udp", fixture.address(ipv6, 8080)) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(2 * time.Second)) + _, err = peer.Write([]byte("anyone")) + if err != nil { + test.Fatal(err) + } + _, err = peer.Read(make([]byte, 64)) + if !errors.Is(err, syscall.ECONNREFUSED) { + test.Fatalf("datagram to a handler-less stack returned %v", err) + } + }) + addressTest.Run("dial", func(test *testing.T) { + listener, address := kernelListen(test, fixture, ipv6) + conn, err := fixture.stack.DialTCP(context.Background(), fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + test.Fatal(err) + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + server := kernelAcceptTCP(test, listener) + err = kernelTransfer(server, conn, kernelPayload(256<<10, 11), false) + if err != nil { + test.Fatal(err) + } + }) + }) + } +} + +func TestGoMemoryOutboundReply(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, conn, destination := fixture.packetPair(test, ipv6) + time.Sleep(100 * time.Millisecond) + payload := kernelPayload(64, 97) + options := N.NewReadWaitOptions(nil, conn) + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + started := time.Now() + err := conn.WritePacket(packet, destination) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(time.Second)) + received := make([]byte, 1500) + n, err := client.Read(received) + elapsed := time.Since(started) + if err != nil || !bytes.Equal(received[:n], payload) { + test.Fatalf("reply: %x, %v", received[:n], err) + } + if elapsed > 100*time.Millisecond { + test.Fatalf("reply reached the kernel after %v", elapsed) + } + }) + } +} + +func TestGoMemoryOutboundBound(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + client, conn, destination := fixture.packetPair(t, false) + err := client.SetReadBuffer(4 << 20) + if err != nil { + t.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + options := N.NewReadWaitOptions(nil, conn) + payload := kernelPayload(100, 113) + sendReply := func() error { + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + return conn.WritePacket(packet, destination) + } + capacity := fixture.memoryTun.outbound.datagram.capacity + for range 4 * capacity { + err = sendReply() + if err != nil { + t.Fatal(err) + } + } + fixture.bridge.pauseOutbound(false) + received := make([]byte, 1500) + delivered := 0 + for { + client.SetReadDeadline(time.Now().Add(time.Second)) + _, err = client.Read(received) + if err != nil { + break + } + delivered++ + } + if delivered < capacity || delivered > capacity+fixture.memoryTun.batchSize { + t.Fatalf("%d replies delivered after the outbound handler resumed, want between %d and %d", delivered, capacity, capacity+fixture.memoryTun.batchSize) + } + marker := kernelPayload(7, 131) + packet := options.NewBufferSize(len(marker)) + packet.Write(marker) + options.PostReturn(packet) + err = conn.WritePacket(packet, destination) + if err != nil { + t.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(time.Second)) + n, err := client.Read(received) + if err != nil || !bytes.Equal(received[:n], marker) { + t.Fatalf("reply after backlog: %x, %v", received[:n], err) + } +} + +func TestGoMemoryControlPriority(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + _, conn, destination := fixture.packetPair(t, false) + listener, address := kernelListen(t, fixture, false) + fixture.bridge.pauseOutbound(true) + options := N.NewReadWaitOptions(nil, conn) + payload := kernelPayload(100, 113) + for range 4 * fixture.memoryTun.outbound.datagram.capacity { + packet := options.NewBufferSize(len(payload)) + packet.Write(payload) + options.PostReturn(packet) + err := conn.WritePacket(packet, destination) + if err != nil { + t.Fatal(err) + } + } + dialed := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + dialConn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(false).Next(), address) + if err == nil { + dialConn.Close() + } + dialed <- err + }() + time.Sleep(100 * time.Millisecond) + resumed := time.Now() + fixture.bridge.pauseOutbound(false) + server := kernelAcceptTCP(t, listener) + server.Close() + err := <-dialed + if err != nil { + t.Fatal(err) + } + if elapsed := time.Since(resumed); elapsed > 500*time.Millisecond { + t.Fatalf("handshake completed %v after the link resumed, the SYN waited behind the datagram backlog", elapsed) + } +} + +func TestGoMemoryPathMTU(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1408, memoryLink: true}) + var tooBig atomic.Int32 + fixture.bridge.filterOutbound(func(packet []byte) bool { + if header.IPVersion(packet) != header.IPv6Version || len(packet) <= header.IPv6MinimumMTU { + return true + } + reply, built := BuildICMPError(packet, ICMPErrorPacketTooBig, netip.Addr{}, header.IPv6MinimumMTU, 0) + if built { + tooBig.Add(1) + fixture.memoryTun.WritePackets([][]byte{reply}) + } + return false + }) + t.Cleanup(func() { fixture.bridge.filterOutbound(nil) }) + for _, dialed := range []bool{true, false} { + t.Run(fmt.Sprintf("dialed=%v", dialed), func(test *testing.T) { + tooBig.Store(0) + conn, kernel, _ := openConnection(test, fixture, true, dialed) + conn.SetDeadline(time.Now().Add(3 * time.Second)) + kernel.SetDeadline(time.Now().Add(3 * time.Second)) + payload := kernelPayload(32<<10, 151) + written := make(chan error, 1) + go func() { + _, writeErr := conn.Write(payload) + written <- writeErr + }() + received := make([]byte, len(payload)) + _, err := io.ReadFull(kernel, received) + if err != nil { + test.Fatalf("transfer over a %d byte path stalled after %d Packet Too Big messages: %v", header.IPv6MinimumMTU, tooBig.Load(), err) + } + err = <-written + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(received, payload) { + test.Fatal("payload corrupted") + } + if tooBig.Load() == 0 { + test.Fatal("no segment exceeded the path MTU") + } + }) + } +} diff --git a/stack_go_integration_options_test.go b/stack_go_integration_options_test.go new file mode 100644 index 00000000..2126f8a5 --- /dev/null +++ b/stack_go_integration_options_test.go @@ -0,0 +1,458 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "sync" + "syscall" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" +) + +func dialKernel(t *testing.T, fixture *kernelStackFixture, ipv6 bool) (*GoConn, *net.TCPConn) { + t.Helper() + listener, address := kernelListen(t, fixture, ipv6) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := fixture.stack.DialTCP(ctx, fixture.kernelAddress(ipv6).Next(), address) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, kernelAcceptTCP(t, listener) +} + +func openConnection(t *testing.T, fixture *kernelStackFixture, ipv6 bool, dialed bool) (*GoConn, *net.TCPConn, uint16) { + t.Helper() + if dialed { + conn, kernel := dialKernel(t, fixture, ipv6) + return conn, kernel, conn.LocalAddr().(*net.TCPAddr).AddrPort().Port() + } + kernel, conn := fixture.pair(t, ipv6) + return conn, kernel, kernel.RemoteAddr().(*net.TCPAddr).AddrPort().Port() +} + +func connectionCases(t *testing.T, fixture *kernelStackFixture, run func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16)) { + for _, ipv6 := range []bool{false, true} { + for _, dialed := range []bool{true, false} { + t.Run(fmt.Sprintf("ipv6=%v,dialed=%v", ipv6, dialed), func(test *testing.T) { + conn, kernel, port := openConnection(test, fixture, ipv6, dialed) + run(test, conn, kernel, port) + }) + } + } +} + +type wireSegment struct { + outbound bool + flags header.TCPFlags + sequence uint32 + ack uint32 + window uint32 + payload int +} + +func (s wireSegment) probe() bool { + return s.outbound && s.payload == 0 && s.flags == header.TCPFlagAck +} + +type wireRecorder struct { + access sync.Mutex + port uint16 + segments []wireSegment +} + +func recordWire(t *testing.T, fixture *kernelStackFixture, port uint16) *wireRecorder { + recorder := &wireRecorder{port: port} + fixture.bridge.observe( + func(packet []byte) { recorder.record(packet, false) }, + func(packet []byte) { recorder.record(packet, true) }, + ) + t.Cleanup(func() { fixture.bridge.observe(nil, nil) }) + return recorder +} + +func (r *wireRecorder) record(packet []byte, outbound bool) { + var transport []byte + switch header.IPVersion(packet) { + case 4: + ipHdr := header.IPv4(packet) + if ipHdr.Protocol() != uint8(header.TCPProtocolNumber) { + return + } + transport = ipHdr.Payload() + case 6: + ipHdr := header.IPv6(packet) + if ipHdr.NextHeader() != uint8(header.TCPProtocolNumber) { + return + } + transport = ipHdr.Payload() + default: + return + } + tcpHdr := header.TCP(transport) + if outbound && tcpHdr.SourcePort() != r.port || !outbound && tcpHdr.DestinationPort() != r.port { + return + } + segment := wireSegment{ + outbound: outbound, + flags: tcpHdr.Flags(), + sequence: tcpHdr.SequenceNumber(), + ack: tcpHdr.AckNumber(), + payload: len(transport) - int(tcpHdr.DataOffset()), + } + if outbound { + segment.window = uint32(tcpHdr.WindowSize()) << goLocalWindowShift + } + r.access.Lock() + r.segments = append(r.segments, segment) + r.access.Unlock() +} + +func (r *wireRecorder) take() []wireSegment { + r.access.Lock() + defer r.access.Unlock() + segments := r.segments + r.segments = nil + return segments +} + +func TestGoConnKeepalive(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("vanished", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetKeepAliveConfig(net.KeepAliveConfig{Enable: true, Idle: 500 * time.Millisecond, Interval: 500 * time.Millisecond, Count: 2}) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + fixture.bridge.pauseInbound(true) + defer fixture.bridge.pauseInbound(false) + start := time.Now() + _, err = conn.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ETIMEDOUT) { + test.Fatalf("read after unanswered probes: %v", err) + } + if elapsed := time.Since(start); elapsed > 4*time.Second { + test.Fatalf("keepalive gave up after %v", elapsed) + } + probes := 0 + reset := false + for deadline := time.Now().Add(time.Second); !reset && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + for _, segment := range recorder.take() { + switch { + case segment.probe(): + probes++ + case segment.outbound && segment.flags&header.TCPFlagRst != 0: + reset = true + } + } + } + if probes != 2 || !reset { + test.Fatalf("sent %d probes and reset=%v before giving up on a peer that stopped answering", probes, reset) + } + }) + }) + t.Run("answered", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetKeepAliveConfig(net.KeepAliveConfig{Enable: true, Idle: 100 * time.Millisecond, Interval: 100 * time.Millisecond, Count: 2}) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + time.Sleep(2 * time.Second) + probes := 0 + answered := 0 + var awaiting []uint32 + for _, segment := range recorder.take() { + switch { + case segment.probe(): + probes++ + awaiting = append(awaiting, segment.sequence+1) + case !segment.outbound && segment.payload == 0 && len(awaiting) > 0 && segment.ack == awaiting[0]: + answered++ + awaiting = awaiting[1:] + } + } + if probes < 2 || answered != probes { + test.Fatalf("kernel answered %d of %d probes sent while idle for 2s", answered, probes) + } + err = kernelTransfer(conn, kernel, kernelPayload(1<<20, 5), false) + if err != nil { + test.Fatal("after probes:", err) + } + }) + }) +} + +func TestGoConnNoDelay(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, noDelay := range []bool{true, false} { + t.Run(fmt.Sprintf("nodelay=%v", noDelay), func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetNoDelay(noDelay) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + fixture.bridge.pauseInbound(true) + defer fixture.bridge.pauseInbound(false) + for _, letter := range []byte("abcdef") { + _, err = conn.Write([]byte{letter}) + if err != nil { + test.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + } + time.Sleep(100 * time.Millisecond) + dataSegments := make(map[uint32]bool) + for _, segment := range recorder.take() { + if segment.outbound && segment.payload > 0 { + dataSegments[segment.sequence] = true + } + } + if noDelay && len(dataSegments) != 6 { + test.Fatalf("sent %d data segments for 6 writes without delay", len(dataSegments)) + } + if !noDelay && len(dataSegments) != 1 { + test.Fatalf("sent %d data segments while the first byte was unacknowledged", len(dataSegments)) + } + fixture.bridge.pauseInbound(false) + received := make([]byte, 6) + _, err = io.ReadFull(kernel, received) + if err != nil { + test.Fatal(err) + } + if string(received) != "abcdef" { + test.Fatalf("kernel received %q", received) + } + if !noDelay { + coalesced := false + for _, segment := range recorder.take() { + if segment.outbound && segment.payload == 5 { + coalesced = true + } + } + if !coalesced { + test.Fatal("held bytes were not sent as one segment once the acknowledgement arrived") + } + } + }) + }) + } +} + +func TestGoConnLinger(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("abort", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + _, err := conn.Write([]byte("hello")) + if err != nil { + test.Fatal(err) + } + _, err = io.ReadFull(kernel, make([]byte, 5)) + if err != nil { + test.Fatal(err) + } + err = conn.SetLinger(0) + if err != nil { + test.Fatal(err) + } + conn.Close() + _, err = kernel.Read(make([]byte, 1)) + if !errors.Is(err, syscall.ECONNRESET) { + test.Fatalf("kernel read after linger 0 close: %v", err) + } + }) + }) + t.Run("prompt", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetLinger(3) + if err != nil { + test.Fatal(err) + } + _, err = conn.Write([]byte("x")) + if err != nil { + test.Fatal(err) + } + start := time.Now() + conn.Close() + if elapsed := time.Since(start); elapsed > time.Second { + test.Fatalf("close waited %v although the peer acknowledged the FIN", elapsed) + } + received, err := io.ReadAll(kernel) + if err != nil { + test.Fatal(err) + } + if string(received) != "x" { + test.Fatalf("kernel received %q", received) + } + }) + }) + t.Run("expire", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetLinger(1) + if err != nil { + test.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + defer fixture.bridge.pauseOutbound(false) + _, err = conn.Write([]byte("x")) + if err != nil { + test.Fatal(err) + } + start := time.Now() + conn.Close() + elapsed := time.Since(start) + if elapsed < time.Second || elapsed > 3*time.Second { + test.Fatalf("close waited %v with the FIN undeliverable and linger 1s", elapsed) + } + fixture.bridge.pauseOutbound(false) + received, err := io.ReadAll(kernel) + if err != nil { + test.Fatal(err) + } + if string(received) != "x" { + test.Fatalf("kernel received %q", received) + } + }) + }) +} + +func TestGoConnBuffers(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + t.Run("read", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetReadBuffer(goSlabSize) + if err != nil { + test.Fatal(err) + } + recorder := recordWire(test, fixture, port) + payload := kernelPayload(4<<20, 11) + err = kernelTransfer(kernel, conn, payload, false) + if err != nil { + test.Fatal(err) + } + var first *wireSegment + var largest uint32 + for _, segment := range recorder.take() { + if !segment.outbound { + continue + } + if first == nil { + first = &segment + } + if segment.ack-first.ack < uint32(len(payload)/2) { + continue + } + largest = max(largest, segment.window) + } + if largest == 0 || largest > goSlabSize { + test.Fatalf("advertised a %d byte window in the second half of the transfer with a %d byte read buffer", largest, goSlabSize) + } + }) + }) + t.Run("write", func(scenario *testing.T) { + connectionCases(scenario, fixture, func(test *testing.T, conn *GoConn, kernel *net.TCPConn, port uint16) { + err := conn.SetWriteBuffer(goSlabSize) + if err != nil { + test.Fatal(err) + } + fixture.bridge.pauseOutbound(true) + defer fixture.bridge.pauseOutbound(false) + conn.SetWriteDeadline(time.Now().Add(300 * time.Millisecond)) + n, err := conn.Write(kernelPayload(4<<20, 13)) + if !errors.Is(err, os.ErrDeadlineExceeded) { + test.Fatalf("write against a stalled link: %v", err) + } + if n != goSlabSize { + test.Fatalf("buffered %d bytes with a %d byte write buffer", n, goSlabSize) + } + }) + }) +} + +func TestGoUDPReadBuffer(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500, memoryLink: true}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + socket, err := fixture.stack.ListenUDP(netip.AddrPortFrom(fixture.kernelAddress(ipv6).Next(), 0)) + if err != nil { + test.Fatal(err) + } + defer socket.Close() + err = socket.SetReadBuffer(8192) + if err != nil { + test.Fatal(err) + } + peer, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(netip.AddrPortFrom(fixture.kernelAddress(ipv6), 0))) + if err != nil { + test.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + target := socket.LocalAddr().(*net.UDPAddr).AddrPort() + for index := range 64 { + _, err = peer.WriteToUDPAddrPort(kernelPayload(1000, uint32(index+1)), target) + if err != nil { + test.Fatal(err) + } + } + time.Sleep(200 * time.Millisecond) + storage := make([]byte, 2000) + received := 0 + for { + socket.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + n, _, readErr := socket.ReadFromUDPAddrPort(storage) + if readErr != nil { + if !errors.Is(readErr, os.ErrDeadlineExceeded) { + test.Fatal(readErr) + } + break + } + if !bytes.Equal(storage[:n], kernelPayload(1000, uint32(received+1))) { + test.Fatalf("datagram %d out of order or corrupted", received) + } + received++ + } + if received == 0 || received*1000 > 8192 { + test.Fatalf("queued %d datagrams of 1000 bytes with an 8192 byte read buffer", received) + } + payload := kernelPayload(1000, 99) + _, err = peer.WriteToUDPAddrPort(payload, target) + if err != nil { + test.Fatal(err) + } + socket.SetReadDeadline(time.Now().Add(3 * time.Second)) + n, source, err := socket.ReadFromUDPAddrPort(storage) + if err != nil { + test.Fatal("after draining the queue:", err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("datagram after draining carried %d bytes with corrupted payload", n) + } + _, err = socket.WriteToUDPAddrPort(storage[:n], source) + if err != nil { + test.Fatal(err) + } + n, _, err = peer.ReadFrom(storage) + if err != nil { + test.Fatal(err) + } + if !bytes.Equal(storage[:n], payload) { + test.Fatalf("reply carried %d bytes with corrupted payload", n) + } + }) + } +} diff --git a/stack_go_integration_packet_flow_test.go b/stack_go_integration_packet_flow_test.go new file mode 100644 index 00000000..8a9bd492 --- /dev/null +++ b/stack_go_integration_packet_flow_test.go @@ -0,0 +1,355 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "net/netip" + "runtime" + "sync" + "testing" + "time" + + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +func TestGoKernelPacketFragments(t *testing.T) { + configs := []kernelStackConfig{{mtu: 1500}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("gso=%v/mq=%v/memory=%v", config.gso, config.multiQueue, config.memoryLink), func(configTest *testing.T) { + fixture := newKernelStackFixture(configTest, config) + for _, ipv6 := range []bool{false, true} { + configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + const sessions = 4 + for phase := range 4 { + start := make(chan struct{}) + release := sync.OnceFunc(func() { close(start) }) + test.Cleanup(release) + completed := make(chan error, sessions) + for index := range sessions { + client, conn, destination := fixture.packetPair(test, ipv6) + payload := kernelPayload(16385, uint32(phase*sessions+index+1)) + go func() { + defer client.Close() + defer conn.Close() + <-start + client.SetReadDeadline(time.Now().Add(time.Second)) + options := N.NewReadWaitOptions(nil, conn) + buffer := options.NewBufferSize(len(payload)) + copy(buffer.Extend(len(payload)), payload) + options.PostReturn(buffer) + err := conn.WritePacket(buffer, destination) + if err != nil { + completed <- E.Cause(err, "write fragments for session ", index) + return + } + storage := make([]byte, 65535) + n, err := client.Read(storage) + if err == nil && !bytes.Equal(storage[:n], payload) { + err = E.New("reassembled ", n, "/", len(payload), " bytes with corrupted payload") + } + if err != nil { + err = E.Cause(err, "receive fragments for session ", index) + } + completed <- err + }() + } + release() + for range sessions { + err := kernelTCPResult(test, completed, "fragmented UDP delivery") + if err != nil { + test.Errorf("phase=%d: %v", phase, err) + } + } + if test.Failed() { + return + } + } + }) + } + }) + } +} + +func TestGoKernelPacketRead(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + for _, batch := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v/batch=%v", ipv6, batch), func(test *testing.T) { + test.Parallel() + client, conn, destination := fixture.packetPair(test, ipv6) + waiter, created := bufio.CreatePacketReadWaiter(conn) + if !created { + test.Fatal("missing packet read waiter") + } + batchWaiter, batchCreated := bufio.CreatePacketBatchReadWaiter(conn) + if !batchCreated { + test.Fatal("missing packet batch read waiter") + } + for phase, options := range []N.ReadWaitOptions{ + {BatchSize: 4}, + {FrontHeadroom: 91, RearHeadroom: 73, BatchSize: 4}, + {FrontHeadroom: 17, RearHeadroom: 127, BatchSize: 3}, + } { + expected := make(map[string]bool) + sizes := []int{0, 1, 1452, 1473, 8193, 1024, 13, 64} + for index, size := range sizes { + payload := kernelPayload(size, uint32(193+phase*17+index)) + expected[string(payload)] = true + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + } + time.Sleep(40 * time.Millisecond) + waiter.InitializeReadWaiter(options) + largestBatch := 0 + for received := 0; received < len(sizes); { + var buffers []*buf.Buffer + var destinations []M.Socksaddr + var err error + if batch { + buffers, destinations, err = batchWaiter.WaitReadPackets() + } else { + var buffer *buf.Buffer + var packetDestination M.Socksaddr + buffer, packetDestination, err = waiter.WaitReadPacket() + if buffer != nil { + buffers = []*buf.Buffer{buffer} + destinations = []M.Socksaddr{packetDestination} + } + } + if err != nil { + buf.ReleaseMulti(buffers) + test.Fatal(err) + } + largestBatch = max(largestBatch, len(buffers)) + if len(buffers) == 0 || len(buffers) != len(destinations) || len(buffers) > options.BatchSize { + buf.ReleaseMulti(buffers) + test.Fatal("incorrect packet batch size") + } + for index, buffer := range buffers { + if !expected[string(buffer.Bytes())] || destinations[index] != destination { + buf.ReleaseMulti(buffers[index:]) + test.Fatalf("phase %d: unexpected/duplicate packet or destination", phase) + } + delete(expected, string(buffer.Bytes())) + buffer.ExtendHeader(options.FrontHeadroom) + buffer.Extend(options.RearHeadroom) + buffer.Release() + } + received += len(buffers) + } + if batch && largestBatch <= 1 { + test.Fatal("queued datagrams were never read as a batch") + } + } + options := N.NewReadWaitOptions(nil, conn) + writer := bufio.NewPacketBatchWriter(conn) + var buffers []*buf.Buffer + var destinations []M.Socksaddr + expected := make(map[string]bool) + for index, size := range []int{0, 1, 1453, 8193} { + payload := kernelPayload(size, uint32(211+index)) + expected[string(payload)] = true + buffer := options.NewBufferSize(len(payload)) + _, _ = buffer.Write(payload) + options.PostReturn(buffer) + buffers = append(buffers, buffer) + destinations = append(destinations, destination) + } + err := writer.WritePacketBatch(buffers, destinations) + if err != nil { + test.Fatal(err) + } + storage := make([]byte, 65535) + for len(expected) > 0 { + n, readErr := client.Read(storage) + if readErr != nil || !expected[string(storage[:n])] { + var missing []int + for packet := range expected { + missing = append(missing, len(packet)) + } + test.Fatalf("batch write data: length=%d missing=%v: %v", n, missing, readErr) + } + delete(expected, string(storage[:n])) + } + }) + } + } +} + +func TestGoKernelPacketMapping(t *testing.T) { + queues := []bool{false} + if runtime.GOOS == "linux" { + queues = append(queues, true) + } + for _, multiQueue := range queues { + for _, mapping := range []NATMapping{NATMappingEndpointIndependent, NATMappingAddressDependent, NATMappingAddressAndPortDependent} { + for _, filtering := range []NATFiltering{NATFilteringEndpointIndependent, NATFilteringAddressDependent, NATFilteringAddressAndPortDependent} { + t.Run(fmt.Sprintf("mq=%v/mapping=%d/filtering=%d", multiQueue, mapping, filtering), func(configTest *testing.T) { + fixture := newKernelStackFixture(configTest, kernelStackConfig{mtu: 1500, multiQueue: multiQueue, udpMapping: mapping, udpFiltering: filtering}) + for _, ipv6 := range []bool{false, true} { + configTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + address := fixture.kernelAddress(ipv6) + client, err := net.ListenUDP("udp", &net.UDPAddr{IP: address.AsSlice()}) + if err != nil { + test.Fatal(err) + } + test.Cleanup(func() { client.Close() }) + client.SetDeadline(time.Now().Add(3 * time.Second)) + port := uint16(20000 + 8*fixture.port.Add(1)) + peers := []M.Socksaddr{ + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port+1)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next().Next(), port+2)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next(), port+3)), + M.SocksaddrFromNetIP(netip.AddrPortFrom(address.Next().Next().Next(), port+4)), + } + accepted := make(chan N.PacketConn, 4) + fixture.access.Lock() + for _, peer := range peers { + fixture.udp[peer.Port] = accepted + } + fixture.access.Unlock() + var first N.PacketConn + for index, peer := range peers[:3] { + payload := []byte{byte(index + 1)} + _, err = client.WriteToUDPAddrPort(payload, peer.AddrPort()) + if err != nil { + test.Fatal(err) + } + conn := first + if index == 0 || mapping == NATMappingAddressAndPortDependent || (mapping == NATMappingAddressDependent && index == 2) { + select { + case conn = <-accepted: + test.Cleanup(func() { conn.Close() }) + case <-time.After(time.Second): + test.Fatal("missing independent UDP session") + } + } + if index == 0 { + first = conn + } + conn.SetReadDeadline(time.Now().Add(time.Second)) + buffer := buf.NewPacket() + destination, readErr := conn.ReadPacket(buffer) + matches := bytes.Equal(buffer.Bytes(), payload) + buffer.Release() + if readErr != nil || !matches || destination != peer { + test.Fatalf("mapping peer %d: destination=%v: %v", index, destination, readErr) + } + } + for index, peer := range peers { + allowed := filtering == NATFilteringEndpointIndependent || index == 0 + if filtering == NATFilteringAddressDependent && (index == 1 || index == 3) { + allowed = true + } + if index == 1 && mapping != NATMappingAddressAndPortDependent { + allowed = true + } + if index == 2 && mapping == NATMappingEndpointIndependent { + allowed = true + } + payload := []byte{byte(index + 31)} + options := N.NewReadWaitOptions(nil, first) + buffer := options.NewBufferSize(len(payload)) + _, _ = buffer.Write(payload) + options.PostReturn(buffer) + err = first.WritePacket(buffer, peer) + if err != nil { + test.Fatal(err) + } + client.SetReadDeadline(time.Now().Add(75 * time.Millisecond)) + if allowed { + client.SetReadDeadline(time.Now().Add(time.Second)) + } + data := make([]byte, 16) + n, source, readErr := client.ReadFromUDPAddrPort(data) + if allowed { + if readErr != nil || !bytes.Equal(data[:n], payload) || source != peer.AddrPort() { + test.Fatalf("allowed return peer %d: source=%v: %v", index, source, readErr) + } + } else { + var timeout net.Error + if !errors.As(readErr, &timeout) || !timeout.Timeout() { + test.Fatalf("uncontacted return peer %d was not filtered: source=%v: %v", index, source, readErr) + } + } + } + }) + } + }) + } + } + } +} + +func TestGoKernelPacketLifetime(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + test.Parallel() + _, idle, _ := fixture.packetPair(test, ipv6) + client, active, _ := fixture.packetPair(test, ipv6) + for _, conn := range []*UDPNatConn{idle, active} { + if !conn.SetTimeout(180 * time.Millisecond) { + test.Fatal("cannot set UDP timeout") + } + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + } + idleClosed := make(chan error, 1) + go func() { + buffer, _, err := idle.WaitReadPacket() + if buffer != nil { + buffer.Release() + } + idleClosed <- err + }() + buffer := buf.NewPacket() + defer buffer.Release() + for index := range 8 { + time.Sleep(50 * time.Millisecond) + payload := []byte{byte(index + 1)} + _, err := client.Write(payload) + if err != nil { + test.Fatal(err) + } + buffer.Reset() + _, err = active.ReadPacket(buffer) + if err != nil || !bytes.Equal(buffer.Bytes(), payload) { + test.Fatalf("active UDP session expired: %v", err) + } + } + select { + case err := <-idleClosed: + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("independent idle session: %v", err) + } + case <-time.After(300 * time.Millisecond): + test.Fatal("traffic on another session kept idle session alive") + } + active.SetReadDeadline(time.Now().Add(600 * time.Millisecond)) + _, err := active.ReadPacket(buffer) + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("UDP session did not expire after traffic stopped: %v", err) + } + }) + } +} diff --git a/stack_go_integration_packet_offload_linux_test.go b/stack_go_integration_packet_offload_linux_test.go new file mode 100644 index 00000000..37c5aaf2 --- /dev/null +++ b/stack_go_integration_packet_offload_linux_test.go @@ -0,0 +1,185 @@ +//go:build linux && !android + +package tun + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "net/netip" + "testing" + "time" + "unsafe" + + "github.com/sagernet/sing/common/buf" + M "github.com/sagernet/sing/common/metadata" + "github.com/sagernet/sing/protocol/socks" + + "golang.org/x/sys/unix" +) + +func TestGoKernelPacketOffload(t *testing.T) { + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator, epoch: time.Now()} + buf.DefaultAllocator = allocator + t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) + for _, multiQueue := range []bool{false, true} { + t.Run(fmt.Sprintf("mq=%v", multiQueue), func(queueTest *testing.T) { + fixture := newKernelStackFixture(queueTest, kernelStackConfig{mtu: 1500, gso: true, multiQueue: multiQueue, prepareStack: allocator.prepareStack}) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"direct", "socks", "socks-fqdn"} { + queueTest.Run(fmt.Sprintf("ipv6=%v/%s", ipv6, mode), func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + if ipv6 { + network = "udp6" + address = net.IPv6loopback + } + peer, err := net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + defer peer.Close() + peer.SetReadBuffer(1 << 20) + upstream, err := net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + scenarioTest.Fatal(err) + } + defer upstream.Close() + upstream.SetReadBuffer(1 << 20) + rawUpstream, err := upstream.SyscallConn() + if err != nil { + scenarioTest.Fatal(err) + } + var ( + monitorDescriptor int + monitorError error + ) + err = rawUpstream.Control(func(descriptor uintptr) { + monitorDescriptor, monitorError = unix.FcntlInt(descriptor, unix.F_DUPFD_CLOEXEC, 0) + }) + if err != nil || monitorError != nil { + scenarioTest.Fatalf("duplicate socket for GRO inspection: %v, %v", err, monitorError) + } + defer unix.Close(monitorDescriptor) + for _, socket := range []*net.UDPConn{client, peer} { + rawConn, rawError := socket.SyscallConn() + if rawError != nil { + scenarioTest.Fatal(rawError) + } + controlError := rawConn.Control(func(descriptor uintptr) { + err = unix.SetsockoptInt(int(descriptor), unix.IPPROTO_UDP, unix.UDP_GRO, 1) + }) + if controlError != nil || err != nil { + scenarioTest.Fatalf("enable peer GRO: %v, %v", controlError, err) + } + } + options := SplicePacketOptions{NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}} + var prefix []byte + if mode != "direct" { + target := M.ParseSocksaddr("192.0.2.1:53") + prefix = []byte{0, 0, 0, 1, 192, 0, 2, 1, 0, 53} + if ipv6 { + target = M.ParseSocksaddr("[2001:db8::1]:53") + prefix = append([]byte{0, 0, 0, 4}, netip.MustParseAddr("2001:db8::1").AsSlice()...) + prefix = append(prefix, 0, 53) + } + if mode == "socks-fqdn" { + target = M.ParseSocksaddr("batch.example:53") + prefix = append([]byte{0, 0, 0, 3, byte(len(target.Fqdn))}, target.Fqdn...) + prefix = append(prefix, 0, 53) + } + associate := socks.NewAssociatePacketConn(upstream, target, upstream) + options.Offload, _ = associate.CreatePacketOffload() + options.FrontHeadroom = associate.FrontHeadroom() + options.NAT.Destination = target + } + if !conn.Splice(&kernelPacketSocket{UDPConn: upstream}, options) { + scenarioTest.Fatal("UDP splice refused") + } + for cycle := range 2 { + deadline := time.Now().Add(3 * time.Second) + client.SetDeadline(deadline) + conn.SetReadDeadline(deadline) + peer.SetDeadline(deadline) + payloads := make([][]byte, 20) + wirePackets := make([][]byte, len(payloads)) + for index := range payloads { + size := 1200 + if index == len(payloads)-1 { + size = 133 + } + payloads[index] = kernelPayload(size, uint32(index+cycle*31+53)) + wirePackets[index] = append(bytes.Clone(prefix), payloads[index]...) + } + _, _, err = client.WriteMsgUDP(bytes.Join(payloads, nil), kernelUDPSegmentControl(1200), nil) + if err != nil { + scenarioTest.Fatal("kernel GSO upload:", err) + } + source := kernelReadUDPSegments(scenarioTest, peer, wirePackets) + gro, inspectError := unix.GetsockoptInt(monitorDescriptor, unix.IPPROTO_UDP, unix.UDP_GRO) + if inspectError != nil || gro != 1 { + scenarioTest.Fatalf("splice socket GRO disabled: %d, %v", gro, inspectError) + } + _, _, err = peer.WriteMsgUDP(bytes.Join(wirePackets, nil), kernelUDPSegmentControl(1200+len(prefix)), source) + if err != nil { + scenarioTest.Fatal("peer GSO download:", err) + } + kernelReadUDPSegments(scenarioTest, client, payloads) + allocator.waitIdle(scenarioTest) + time.Sleep(10 * time.Millisecond) + } + }) + } + } + }) + } +} + +func kernelUDPSegmentControl(size int) []byte { + control := make([]byte, unix.CmsgSpace(2)) + header := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + header.Level = unix.IPPROTO_UDP + header.Type = unix.UDP_SEGMENT + header.SetLen(unix.CmsgLen(2)) + binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):], uint16(size)) + return control +} + +func kernelReadUDPSegments(t *testing.T, socket *net.UDPConn, expected [][]byte) *net.UDPAddr { + t.Helper() + data := make([]byte, 65535) + control := make([]byte, 128) + var source *net.UDPAddr + segmented := false + for index := 0; index < len(expected); { + length, controlLength, flags, sender, err := socket.ReadMsgUDP(data, control) + if err != nil || flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 { + t.Fatalf("receive segment %d: flags=%d, %v", index, flags, err) + } + source = sender + messages, err := unix.ParseSocketControlMessage(control[:controlLength]) + if err != nil { + t.Fatal(err) + } + segmentSize := length + for _, message := range messages { + if message.Header.Level == unix.IPPROTO_UDP && message.Header.Type == unix.UDP_GRO { + segmentSize = int(binary.NativeEndian.Uint32(message.Data)) + segmented = segmented || segmentSize < length + } + } + for offset := 0; offset < length; offset += segmentSize { + packet := data[offset:min(offset+segmentSize, length)] + if index >= len(expected) || !bytes.Equal(packet, expected[index]) { + t.Fatalf("segment %d corrupted: length=%d", index, len(packet)) + } + index++ + } + } + if !segmented { + t.Fatal("kernel received no UDP GSO/GRO aggregate") + } + return source +} diff --git a/stack_go_integration_packet_test.go b/stack_go_integration_packet_test.go new file mode 100644 index 00000000..a5146874 --- /dev/null +++ b/stack_go_integration_packet_test.go @@ -0,0 +1,772 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + std_bufio "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/protocol/socks" +) + +func TestGoKernelPacketWriteAfterClose(t *testing.T) { + configs := []kernelStackConfig{{mtu: 1500}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}) + } + for _, config := range configs { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("gso=%v/memory=%v/ipv6=%v", config.gso, config.memoryLink, ipv6), func(t *testing.T) { + fixture := newKernelStackFixture(t, config) + client, conn, destination := fixture.packetPair(t, ipv6) + defer client.Close() + batchWriter, batchSupported := conn.CreatePacketBatchWriter() + err := fixture.stack.Close() + if err != nil { + t.Fatal(err) + } + options := N.NewReadWaitOptions(nil, conn) + newPacket := func() *buf.Buffer { + packet := options.NewBufferSize(64) + packet.Write([]byte("late UDP reply")) + options.PostReturn(packet) + return packet + } + err = conn.WritePacket(newPacket(), destination) + if !errors.Is(err, os.ErrClosed) { + t.Errorf("packet write after stack close: %v", err) + } + if batchSupported { + packets := []*buf.Buffer{newPacket(), newPacket()} + destinations := []M.Socksaddr{destination, destination} + err = batchWriter.WritePacketBatch(packets, destinations) + if !errors.Is(err, os.ErrClosed) { + t.Errorf("packet batch write after stack close: %v", err) + } + } + }) + } + } +} + +func TestGoKernelPacketTimeout(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(test *testing.T) { + client, conn, destination := fixture.packetPair(test, ipv6) + if !conn.SetTimeout(100 * time.Millisecond) { + test.Fatal("cannot set live UDP session timeout") + } + buffer, _, err := conn.WaitReadPacket() + if buffer != nil { + buffer.Release() + } + if !errors.Is(err, io.ErrClosedPipe) { + test.Fatalf("idle UDP session did not close: %v", err) + } + payload := kernelPayload(1200, 103) + _, err = client.Write(payload) + if err != nil { + test.Fatal(err) + } + fixture.access.Lock() + accepted := fixture.udp[destination.Port] + fixture.access.Unlock() + select { + case replacement := <-accepted: + defer replacement.Close() + replacement.SetReadDeadline(time.Now().Add(time.Second)) + options := N.NewReadWaitOptions(nil, replacement) + packet := options.NewBufferSize(1500) + _, err = replacement.ReadPacket(packet) + if err != nil || !bytes.Equal(packet.Bytes(), payload) { + packet.Release() + test.Fatalf("UDP session renewal lost its first packet: %v", err) + } + options.PostReturn(packet) + err = replacement.WritePacket(packet, destination) + if err != nil { + test.Fatal(err) + } + response := make([]byte, 1500) + n, readErr := client.Read(response) + if readErr != nil || !bytes.Equal(response[:n], payload) { + test.Fatalf("renewed UDP session response: %v", readErr) + } + case <-time.After(time.Second): + test.Fatal("next UDP packet did not create a new session") + } + }) + } +} + +func TestGoKernelPacketBuffer(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + client, conn, destination := fixture.packetPair(addressTest, ipv6) + for _, size := range []int{0, 1, 1473} { + for _, capacity := range []int{0, 1, 1024, 2048} { + addressTest.Run(fmt.Sprintf("size=%d/capacity=%d", size, capacity), func(packetTest *testing.T) { + payload := kernelPayload(size, uint32(size+71)) + _, err := client.Write(payload) + if err != nil { + packetTest.Fatal(err) + } + prefix := []byte("prefix") + buffer := buf.With(make([]byte, 16+len(prefix)+capacity+32)) + buffer.Resize(16, len(prefix)) + buffer.Reserve(32) + copy(buffer.Bytes(), prefix) + actualDestination, readErr := conn.ReadPacket(buffer) + if capacity == 0 { + if !errors.Is(readErr, io.ErrShortBuffer) { + packetTest.Fatalf("full buffer: %v", readErr) + } + } else if readErr != nil { + packetTest.Fatal(readErr) + } + if actualDestination != destination { + packetTest.Fatalf("destination: %v, want %v", actualDestination, destination) + } + if buffer.Start() != 16 || !bytes.Equal(buffer.Bytes(), append(prefix, payload[:min(size, capacity)]...)) { + packetTest.Fatalf("packet buffer: %x", buffer.Bytes()) + } + marker := kernelPayload(7, 83) + _, err = client.Write(marker) + if err != nil { + packetTest.Fatal(err) + } + buffer.Reset() + _, err = conn.ReadPacket(buffer) + if err != nil || !bytes.Equal(buffer.Bytes(), marker) { + packetTest.Fatalf("next datagram: %x: %v", buffer.Bytes(), err) + } + }) + } + } + }) + } +} + +func (f *kernelStackFixture) packetPair(t *testing.T, ipv6 bool) (*net.UDPConn, *UDPNatConn, M.Socksaddr) { + t.Helper() + port := uint16(20000 + f.port.Add(1)) + accepted := make(chan N.PacketConn, 1) + f.access.Lock() + f.udp[port] = accepted + f.access.Unlock() + destination := M.ParseSocksaddr(f.address(ipv6, port)) + client, err := net.DialUDP("udp", nil, destination.UDPAddr()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { client.Close() }) + err = client.SetWriteBuffer(1 << 20) + if err != nil { + t.Fatal(err) + } + client.SetDeadline(time.Now().Add(3 * time.Second)) + _, err = client.Write([]byte("ready")) + if err != nil { + t.Fatal(err) + } + select { + case conn := <-accepted: + t.Cleanup(func() { conn.Close() }) + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + buffer := buf.NewPacket() + _, err = conn.ReadPacket(buffer) + buffer.Release() + if err != nil { + t.Fatal(err) + } + return client, conn.(*UDPNatConn), destination + case <-time.After(3 * time.Second): + t.Fatal("UDP flow not accepted") + return nil, nil, M.Socksaddr{} + } +} + +type kernelPacketSocket struct { + *net.UDPConn + access sync.Mutex + owner io.Closer + closed bool +} + +func (s *kernelPacketSocket) Attach(owner io.Closer) (io.Closer, bool) { + s.access.Lock() + defer s.access.Unlock() + if s.closed || s.owner != nil { + return nil, false + } + s.owner = owner + return s.UDPConn, true +} + +func (s *kernelPacketSocket) Detach() { + s.access.Lock() + s.owner = nil + closed := s.closed + s.access.Unlock() + if closed { + s.UDPConn.Close() + } +} + +func (s *kernelPacketSocket) Close() error { + s.access.Lock() + s.closed = true + owner := s.owner + s.access.Unlock() + if owner != nil { + return owner.Close() + } + return s.UDPConn.Close() +} + +func TestGoKernelPacket(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + for _, splice := range []string{"none", "direct", "socks", "socks-fqdn"} { + addressTest.Run("splice="+splice, func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + maximumPayload := 65535 - header.IPv4MinimumSize - header.UDPMinimumSize + if ipv6 { + network = "udp6" + address = net.IPv6loopback + maximumPayload = 65535 - header.UDPMinimumSize + } + var ( + peer *net.UDPConn + socksPackets chan kernelSocksPacket + socksDestination M.Socksaddr + headerLength int + ) + switch splice { + case "direct": + var err error + peer, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { peer.Close() }) + upstream, err := net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { upstream.Close() }) + peer.SetWriteBuffer(1 << 20) + upstream.SetWriteBuffer(1 << 20) + peer.SetDeadline(time.Now().Add(3 * time.Second)) + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + SpliceOptions: SpliceOptions{OnClose: func(error) { upstream.Close() }}, + }) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + case "socks", "socks-fqdn": + if splice == "socks" { + socksDestination = M.ParseSocksaddrHostPort("192.0.2.1", 53) + if ipv6 { + socksDestination = M.ParseSocksaddrHostPort("2001:db8::1", 53) + } + } else { + socksDestination = M.ParseSocksaddrHostPort(strings.Repeat(strings.Repeat("a", 63)+".", 3)+strings.Repeat("b", 61), 53) + } + var serverAddress M.Socksaddr + serverAddress, socksPackets = newKernelSocksEchoServer(scenarioTest, address) + socksClient := socks.NewClient(N.SystemDialer, serverAddress, socks.Version5, "", "") + handshakeContext, cancelHandshake := context.WithTimeout(context.Background(), 3*time.Second) + associateConn, err := socksClient.ListenPacket(handshakeContext, socksDestination) + cancelHandshake() + if err != nil { + scenarioTest.Fatal(err) + } + scenarioTest.Cleanup(func() { associateConn.Close() }) + terminal, offload := N.UnwrapPacketOffload(associateConn) + upstream, isUDPConn := terminal.(*net.UDPConn) + if !isUDPConn || offload == nil { + scenarioTest.Fatalf("SOCKS UDP not unwrapped: %T", terminal) + } + upstream.SetWriteBuffer(1 << 20) + _, err = client.Write([]byte("cached")) + if err != nil { + scenarioTest.Fatal(err) + } + cached := N.NewPacketBuffer() + cached.Buffer, cached.Destination, err = conn.WaitReadPacket() + if err != nil { + N.PutPacketBuffer(cached) + scenarioTest.Fatal(err) + } + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: socksDestination}, + Cached: []*N.PacketBuffer{cached}, + Offload: offload, + FrontHeadroom: N.CalculateFrontHeadroom(associateConn), + RearHeadroom: N.CalculateRearHeadroom(associateConn), + SpliceOptions: SpliceOptions{OnClose: func(error) { associateConn.Close() }}, + }) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + headerLength = 3 + M.SocksaddrSerializer.AddrPortLen(socksDestination) + _, err = client.Write([]byte("ready")) + if err != nil { + scenarioTest.Fatal(err) + } + for _, expected := range []string{"cached", "ready"} { + select { + case packet := <-socksPackets: + if string(packet.payload) != expected || packet.destination != socksDestination { + scenarioTest.Fatalf("SOCKS initial packet: %q to %v", packet.payload, packet.destination) + } + case <-time.After(3 * time.Second): + scenarioTest.Fatal("SOCKS UDP session not established") + } + response := make([]byte, 16) + n, readErr := client.Read(response) + if readErr != nil || string(response[:n]) != expected { + scenarioTest.Fatalf("SOCKS initial response: %q: %v", response[:n], readErr) + } + } + } + sizes := []int{0, 1, 1472, 1473, 8193, 65507} + if ipv6 { + sizes = append(sizes, 1452, 1453, 65527) + } + if headerLength > 0 { + sizes = append(sizes, maximumPayload-headerLength) + } + for _, size := range sizes { + if headerLength+size > maximumPayload { + continue + } + scenarioTest.Run(fmt.Sprintf("size=%d", size), func(packetTest *testing.T) { + payload := kernelPayload(size, uint32(size+37)) + _, err := client.Write(payload) + if err != nil { + packetTest.Fatal("kernel upload:", err) + } + var received []byte + switch splice { + case "none": + options := N.NewReadWaitOptions(nil, conn) + buffer := options.NewBufferSize(65535) + _, err = conn.ReadPacket(buffer) + received = bytes.Clone(buffer.Bytes()) + if err != nil { + buffer.Release() + packetTest.Fatal("Go upload:", err) + } + options.PostReturn(buffer) + err = conn.WritePacket(buffer, destination) + case "direct": + data := make([]byte, 65535) + n, source, readErr := peer.ReadFromUDP(data) + if readErr != nil { + packetTest.Fatal("spliced upload:", readErr) + } + received = data[:n] + _, err = peer.WriteToUDP(payload, source) + default: + select { + case packet := <-socksPackets: + if packet.destination != socksDestination { + packetTest.Fatalf("SOCKS destination: %v, want %v", packet.destination, socksDestination) + } + received = packet.payload + case <-time.After(3 * time.Second): + packetTest.Fatal("spliced SOCKS upload timed out") + } + } + if !bytes.Equal(received, payload) { + packetTest.Fatalf("upload bytes=%d want=%d", len(received), size) + } + if err != nil { + packetTest.Fatal("download write:", err) + } + response := make([]byte, 65535) + n, readErr := client.Read(response) + if readErr != nil { + packetTest.Fatal("kernel download:", readErr) + } + if !bytes.Equal(response[:n], payload) { + packetTest.Fatalf("download bytes=%d want=%d", n, size) + } + }) + } + }) + } + addressTest.Run("tcp_after_udp", func(streamTest *testing.T) { + client, server := fixture.pair(streamTest, ipv6) + err := kernelTransfer(client, server, kernelPayload(64<<10, 41), false) + if err != nil { + streamTest.Fatal("TCP upload after UDP headroom negotiation:", err) + } + err = kernelTransfer(server, client, kernelPayload(64<<10, 43), true) + if err != nil { + streamTest.Fatal("TCP download after UDP headroom negotiation:", err) + } + }) + }) + } + }) + } +} + +type kernelSocksPacket struct { + payload []byte + destination M.Socksaddr +} + +type kernelSocksEchoServer struct { + packets chan kernelSocksPacket +} + +func newKernelSocksEchoServer(t *testing.T, address net.IP) (M.Socksaddr, chan kernelSocksPacket) { + t.Helper() + listener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: address}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + ctx := t.Context() + server := &kernelSocksEchoServer{packets: make(chan kernelSocksPacket, 16)} + go func() { + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + go func() { + stopContext := context.AfterFunc(ctx, func() { conn.Close() }) + handleErr := socks.HandleConnectionEx(ctx, conn, std_bufio.NewReader(conn), nil, server, server, 0, M.SocksaddrFromNet(conn.RemoteAddr()), nil) + if handleErr != nil { + stopContext() + conn.Close() + } + }() + } + }() + return M.SocksaddrFromNet(listener.Addr()), server.packets +} + +func (s *kernelSocksEchoServer) ListenPacket(listenConfig net.ListenConfig, ctx context.Context, network string, address string) (net.PacketConn, error) { + conn, err := listenConfig.ListenPacket(ctx, network, address) + if err != nil { + return nil, err + } + udpConn := conn.(*net.UDPConn) + udpConn.SetReadBuffer(1 << 20) + udpConn.SetWriteBuffer(1 << 20) + return conn, nil +} + +func (s *kernelSocksEchoServer) NewConnectionEx(ctx context.Context, conn net.Conn, source M.Socksaddr, destination M.Socksaddr, onClose N.CloseHandlerFunc) { + conn.Close() +} + +func (s *kernelSocksEchoServer) NewPacketConnectionEx(ctx context.Context, conn N.PacketConn, source M.Socksaddr, destination M.Socksaddr, onClose N.CloseHandlerFunc) { + go func() { + defer conn.Close() + frontHeadroom := N.CalculateFrontHeadroom(conn) + for { + buffer := buf.NewSize(frontHeadroom + 65535) + buffer.Resize(frontHeadroom, 0) + packetDestination, err := conn.ReadPacket(buffer) + if err != nil { + buffer.Release() + return + } + select { + case s.packets <- kernelSocksPacket{payload: bytes.Clone(buffer.Bytes()), destination: packetDestination}: + case <-ctx.Done(): + buffer.Release() + return + } + err = conn.WritePacket(buffer, packetDestination) + if err != nil { + return + } + } + }() +} + +type kernelPacketAllocator struct { + buf.Allocator + active atomic.Int64 + epoch time.Time + lastActivity atomic.Int64 +} + +type kernelPacketIO struct { + goPlatformIO + allocator *kernelPacketAllocator +} + +func (p *kernelPacketIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + readable, count, err := p.goPlatformIO.wait(timeout, events) + if readable || count > 0 { + now := time.Since(p.allocator.epoch).Nanoseconds() + for { + previous := p.allocator.lastActivity.Load() + if now <= previous || p.allocator.lastActivity.CompareAndSwap(previous, now) { + break + } + } + } + return readable, count, err +} + +func (a *kernelPacketAllocator) prepareStack(stack *Go) { + queues := stack.queueFactory + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + platforms, err := queues(current) + if err != nil { + return nil, err + } + for index, platform := range platforms { + platforms[index] = &kernelPacketIO{goPlatformIO: platform, allocator: a} + } + return platforms, nil + } +} + +func (a *kernelPacketAllocator) Get(size int) []byte { + buffer := a.Allocator.Get(size) + if buffer != nil { + a.active.Add(1) + } + return buffer +} + +func (a *kernelPacketAllocator) Put(buffer []byte) error { + err := a.Allocator.Put(buffer) + if err == nil { + a.active.Add(-1) + } + return err +} + +func (a *kernelPacketAllocator) waitIdle(t *testing.T) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + lastActivity := a.lastActivity.Load() + quiet := time.Since(a.epoch) - time.Duration(lastActivity) + active := a.active.Load() + unchanged := a.lastActivity.Load() == lastActivity + if quiet >= goReadBufferIdle && active == 0 && unchanged { + return + } + if quiet >= goReadBufferIdle+time.Second && active != 0 && unchanged { + t.Fatalf("connections retain %d pooled buffers after %s without native I/O", active, quiet) + } + if time.Now().After(deadline) { + t.Fatalf("idle wait deadline reached: last native I/O %s ago, %d pooled buffers retained", quiet, active) + } + time.Sleep(time.Millisecond) + } +} + +func TestGoKernelPacketBatchIdle(t *testing.T) { + allocator := &kernelPacketAllocator{Allocator: buf.DefaultAllocator, epoch: time.Now()} + buf.DefaultAllocator = allocator + t.Cleanup(func() { buf.DefaultAllocator = allocator.Allocator }) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + for _, config := range configs { + config.prepareStack = allocator.prepareStack + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"batch", "connected", "unconnected", "socks-fqdn"} { + configurationTest.Run(fmt.Sprintf("ipv6=%v/%s", ipv6, mode), func(scenarioTest *testing.T) { + client, conn, destination := fixture.packetPair(scenarioTest, ipv6) + var peer *net.UDPConn + var source *net.UDPAddr + var prefix []byte + if mode != "batch" { + network := "udp4" + address := net.IPv4(127, 0, 0, 1) + if ipv6 { + network = "udp6" + address = net.IPv6loopback + } + var err error + peer, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + if err != nil { + scenarioTest.Fatal(err) + } + defer peer.Close() + peer.SetReadBuffer(1 << 20) + peer.SetWriteBuffer(1 << 20) + var upstream *net.UDPConn + if mode != "unconnected" { + upstream, err = net.DialUDP(network, nil, peer.LocalAddr().(*net.UDPAddr)) + } else { + upstream, err = net.ListenUDP(network, &net.UDPAddr{IP: address}) + } + if err != nil { + scenarioTest.Fatal(err) + } + defer upstream.Close() + upstream.SetReadBuffer(1 << 20) + spliceOptions := SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + } + if mode == "socks-fqdn" { + domain := strings.Repeat(strings.Repeat("a", 63)+".", 3) + strings.Repeat("b", 61) + target := M.ParseSocksaddrHostPort(domain, 53) + prefix = append([]byte{0, 0, 0, 3, byte(len(domain))}, domain...) + prefix = append(prefix, 0, 53) + associate := socks.NewAssociatePacketConn(upstream, target, upstream) + spliceOptions.Offload, _ = associate.CreatePacketOffload() + spliceOptions.FrontHeadroom = associate.FrontHeadroom() + spliceOptions.NAT.Destination = target + } + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, spliceOptions) + if !accepted { + scenarioTest.Fatal("UDP splice refused") + } + } + for cycle := range 3 { + deadline := time.Now().Add(3 * time.Second) + client.SetDeadline(deadline) + conn.SetReadDeadline(deadline) + if peer != nil { + peer.SetDeadline(deadline) + } + payloads := make([][]byte, 24) + for index := range payloads { + size := 1200 + if index >= 16 { + size = []int{1, 0, 1453, 8193, 1201, 0, 3, 1200}[index-16] + } + payloads[index] = kernelPayload(size, uint32(cycle*31+index+1)) + _, err := client.Write(payloads[index]) + if err != nil { + scenarioTest.Fatalf("cycle %d upload packet %d: %v", cycle, index, err) + } + } + uploaded := make([]bool, len(payloads)) + for index := range payloads { + var actual []byte + if peer != nil { + data := make([]byte, 65535) + length, sender, err := peer.ReadFromUDP(data) + if err != nil { + scenarioTest.Fatalf("upload packet %d: %v", index, err) + } + source = sender + if length < len(prefix) || !bytes.Equal(data[:len(prefix)], prefix) { + scenarioTest.Fatal("SOCKS packet header corrupted") + } + actual = data[len(prefix):length] + } else { + buffer, _, err := conn.WaitReadPacket() + if err != nil { + scenarioTest.Fatal(err) + } + actual = bytes.Clone(buffer.Bytes()) + buffer.Release() + } + kernelCheckBatchPacket(scenarioTest, actual, payloads, uploaded, index, !config.multiQueue) + } + if peer != nil { + for _, payload := range payloads { + _, err := peer.WriteToUDP(append(bytes.Clone(prefix), payload...), source) + if err != nil { + scenarioTest.Fatal(err) + } + } + } else { + options := N.NewReadWaitOptions(nil, conn) + buffers := make([]*buf.Buffer, len(payloads)) + destinations := make([]M.Socksaddr, len(payloads)) + for index, payload := range payloads { + buffers[index] = options.NewBufferSize(len(payload)) + copy(buffers[index].Extend(len(payload)), payload) + options.PostReturn(buffers[index]) + destinations[index] = destination + } + err := bufio.NewPacketBatchWriter(conn).WritePacketBatch(buffers, destinations) + if err != nil { + scenarioTest.Fatal(err) + } + } + downloaded := make([]bool, len(payloads)) + for index := range payloads { + data := make([]byte, 65535) + length, err := client.Read(data) + if err != nil { + scenarioTest.Fatalf("download packet %d: %v", index, err) + } + kernelCheckBatchPacket(scenarioTest, data[:length], payloads, downloaded, index, !config.multiQueue) + } + allocator.waitIdle(scenarioTest) + if conn.isClosed() { + scenarioTest.Fatal("idle test closed the UDP session") + } + time.Sleep(10 * time.Millisecond) + } + }) + } + } + }) + } +} + +func kernelCheckBatchPacket(t *testing.T, packet []byte, expected [][]byte, received []bool, index int, ordered bool) { + t.Helper() + if ordered { + if !bytes.Equal(packet, expected[index]) { + t.Fatalf("packet %d corrupted or reordered: received %d bytes, expected %d", index, len(packet), len(expected[index])) + } + return + } + for candidate, payload := range expected { + if !received[candidate] && bytes.Equal(packet, payload) { + received[candidate] = true + return + } + } + t.Fatalf("packet %d corrupted or duplicated: received %d bytes", index, len(packet)) +} diff --git a/stack_go_integration_splice_test.go b/stack_go_integration_splice_test.go new file mode 100644 index 00000000..1aef4aac --- /dev/null +++ b/stack_go_integration_splice_test.go @@ -0,0 +1,187 @@ +//go:build (linux && !android) || (darwin && !ios) + +package tun + +import ( + "bytes" + "errors" + "io" + "net" + "testing" + "time" + + M "github.com/sagernet/sing/common/metadata" + "github.com/sagernet/sing/protocol/socks" +) + +func TestGoKernelSpliceOwnership(t *testing.T) { + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 1500}) + t.Run("tcp", func(streamTest *testing.T) { + client, conn := fixture.pair(streamTest, false) + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + streamTest.Fatal(err) + } + defer listener.Close() + upstream, err := net.DialTCP("tcp4", nil, listener.Addr().(*net.TCPAddr)) + if err != nil { + streamTest.Fatal(err) + } + defer upstream.Close() + peer, err := listener.AcceptTCP() + if err != nil { + streamTest.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + closed := make(chan error, 1) + owner := &kernelSocket{TCPConn: upstream} + accepted := conn.Splice(owner, SpliceOptions{ + OnClose: func(closeError error) { closed <- closeError }, + }) + if !accepted { + streamTest.Fatal("TCP splice refused") + } + payload := kernelPayload(4097, 193) + _, err = client.Write(payload) + if err != nil { + streamTest.Fatal(err) + } + data := make([]byte, len(payload)) + _, err = io.ReadFull(peer, data) + if err != nil || !bytes.Equal(data, payload) { + streamTest.Fatalf("spliced TCP upload: %v", err) + } + err = upstream.SetDeadline(time.Now().Add(-time.Second)) + if !errors.Is(err, net.ErrClosed) { + streamTest.Fatalf("original TCP connection survived transfer: %v", err) + } + _, err = peer.Write(payload) + if err != nil { + streamTest.Fatal(err) + } + _, err = io.ReadFull(client, data) + if err != nil || !bytes.Equal(data, payload) { + streamTest.Fatalf("spliced TCP download after original connection closed: %v", err) + } + owner.Close() + owner.Close() + select { + case <-closed: + case <-time.After(time.Second): + streamTest.Fatal("TCP splice did not close its socket") + } + _, err = peer.Read(data) + var networkError net.Error + if err == nil || errors.As(err, &networkError) && networkError.Timeout() { + streamTest.Fatalf("TCP peer survived splice close: %v", err) + } + }) + t.Run("udp", func(packetTest *testing.T) { + client, conn, destination := fixture.packetPair(packetTest, false) + peer, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + packetTest.Fatal(err) + } + defer peer.Close() + peer.SetDeadline(time.Now().Add(3 * time.Second)) + upstream, err := net.DialUDP("udp4", nil, peer.LocalAddr().(*net.UDPAddr)) + if err != nil { + packetTest.Fatal(err) + } + defer upstream.Close() + address := upstream.LocalAddr().(*net.UDPAddr) + closed := make(chan error, 1) + owner := &kernelPacketSocket{UDPConn: upstream} + accepted := conn.Splice(owner, SplicePacketOptions{ + NAT: PacketNAT{Origin: destination, Destination: M.SocksaddrFromNet(peer.LocalAddr())}, + SpliceOptions: SpliceOptions{OnClose: func(closeError error) { closed <- closeError }}, + }) + if !accepted { + packetTest.Fatal("UDP splice refused") + } + payload := kernelPayload(1023, 197) + _, err = client.Write(payload) + if err != nil { + packetTest.Fatal(err) + } + data := make([]byte, len(payload)) + n, source, err := peer.ReadFromUDP(data) + if err != nil || !bytes.Equal(data[:n], payload) { + packetTest.Fatalf("spliced UDP upload: %v", err) + } + err = upstream.SetDeadline(time.Now().Add(-time.Second)) + if !errors.Is(err, net.ErrClosed) { + packetTest.Fatalf("original UDP connection survived transfer: %v", err) + } + _, err = peer.WriteToUDP(payload, source) + if err != nil { + packetTest.Fatal(err) + } + n, err = client.Read(data) + if err != nil || !bytes.Equal(data[:n], payload) { + packetTest.Fatalf("spliced UDP download after original connection closed: %v", err) + } + owner.Close() + owner.Close() + select { + case <-closed: + case <-time.After(time.Second): + packetTest.Fatal("UDP splice did not close its socket") + } + rebound, err := net.ListenUDP("udp4", address) + if err != nil { + packetTest.Fatal("UDP endpoint survived splice close: ", err) + } + rebound.Close() + }) + t.Run("fallback", func(fallbackTest *testing.T) { + _, conn, _ := fixture.packetPair(fallbackTest, false) + upstream, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + fallbackTest.Fatal(err) + } + defer upstream.Close() + address := upstream.LocalAddr().(*net.UDPAddr) + peer, err := net.DialUDP("udp4", nil, upstream.LocalAddr().(*net.UDPAddr)) + if err != nil { + fallbackTest.Fatal(err) + } + defer peer.Close() + offload, _ := (&socks.AssociatePacketConn{}).CreatePacketOffload() + accepted := conn.Splice(&kernelPacketSocket{UDPConn: upstream}, SplicePacketOptions{Offload: offload}) + if accepted { + fallbackTest.Fatal("unconnected SOCKS socket unexpectedly accepted") + } + upstream.SetReadDeadline(time.Now().Add(time.Second)) + completed := make(chan error, 1) + go func() { + data := make([]byte, 1) + _, readError := upstream.Read(data) + completed <- readError + }() + select { + case err = <-completed: + fallbackTest.Fatalf("fallback reader did not wait: %v", err) + case <-time.After(20 * time.Millisecond): + } + _, err = peer.Write([]byte{1}) + if err != nil { + fallbackTest.Fatal(err) + } + select { + case err = <-completed: + if err != nil { + fallbackTest.Fatal("runtime reader after splice rejection: ", err) + } + case <-time.After(2 * time.Second): + fallbackTest.Fatal("runtime reader was not woken") + } + upstream.Close() + rebound, err := net.ListenUDP("udp4", address) + if err != nil { + fallbackTest.Fatal("rejected splice retained a duplicate socket: ", err) + } + rebound.Close() + }) +} diff --git a/stack_go_integration_tcp_test.go b/stack_go_integration_tcp_test.go new file mode 100644 index 00000000..9159152e --- /dev/null +++ b/stack_go_integration_tcp_test.go @@ -0,0 +1,530 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "fmt" + "net" + "slices" + "sync" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + N "github.com/sagernet/sing/common/network" +) + +const kernelTCPTimeout = 3 * time.Second + +type kernelTCPEvent struct { + at time.Duration + rate uint64 + outgoing bool + sequence uint64 + end uint64 + ack uint64 + window uint16 + length int + sacks []goSackBlock + flags header.TCPFlags + dropped bool + delayed bool + delivery bool + copies int +} + +type kernelTCPAction struct { + drop bool + copies int + delay time.Duration + pause *kernelTCPBarrier +} + +type kernelTCPBarrier struct { + entered chan struct{} + resume chan struct{} + once sync.Once +} + +func newKernelTCPBarrier(t *testing.T) *kernelTCPBarrier { + barrier := &kernelTCPBarrier{entered: make(chan struct{}), resume: make(chan struct{})} + t.Cleanup(func() { barrier.once.Do(func() { close(barrier.resume) }) }) + return barrier +} + +type kernelTCPConfig struct { + receiveDelay time.Duration + disableSACK bool + disableTimestamps bool + checkpoint func(string, *GoConn) +} + +type kernelTCPFlow struct { + conn *GoConn + isn uint32 + sent uint64 + acked uint64 + fin uint64 + peerFIN bool + peerWindow uint16 + failure string + events []kernelTCPEvent + processed uint64 + closed bool + released bool + writes int + faults uint64 + filter func(kernelTCPEvent) kernelTCPAction +} + +type kernelTCPHarness struct { + access sync.Mutex + flows map[flowKey]*kernelTCPFlow + changed chan struct{} + done chan struct{} + epoch time.Time + config kernelTCPConfig + workers sync.WaitGroup + stopped bool +} + +func newKernelTCPFixture(t *testing.T, config kernelStackConfig, tcpConfig kernelTCPConfig) (*kernelStackFixture, *kernelTCPHarness) { + t.Helper() + harness := &kernelTCPHarness{ + flows: make(map[flowKey]*kernelTCPFlow), changed: make(chan struct{}, 1), done: make(chan struct{}), + epoch: time.Now(), config: tcpConfig, + } + prepare := config.prepareStack + config.prepareStack = func(stack *Go) { + if prepare != nil { + prepare(stack) + } + queues := stack.queueFactory + stack.queueFactory = func(current *Go) ([]goPlatformIO, error) { + platforms, err := queues(current) + if err != nil { + return nil, err + } + for index, platform := range platforms { + platforms[index] = &kernelTCPIO{goPlatformIO: platform, harness: harness} + } + return platforms, nil + } + operations := *stack.congestion + initialize := operations.init + operations.init = func(conn *GoConn) { + if initialize != nil { + initialize(conn) + } + harness.access.Lock() + harness.flows[conn.key] = &kernelTCPFlow{conn: conn, isn: conn.sendISN} + harness.access.Unlock() + } + release := operations.release + operations.release = func(conn *GoConn) { + harness.access.Lock() + flow := harness.flows[conn.key] + flow.released = true + if flow.writes != 0 && flow.failure == "" { + flow.failure = fmt.Sprintf("connection resources released with %d device writes in progress", flow.writes) + } + harness.access.Unlock() + if release != nil { + release(conn) + } + } + checkpoint := tcpConfig.checkpoint + if checkpoint != nil { + acked := operations.packetsAcked + operations.packetsAcked = func(conn *GoConn, sample *goAckSample) { + if sample.packetsAcked > 0 { + checkpoint("ack", conn) + } + if acked != nil { + acked(conn, sample) + } + } + avoid := operations.congAvoid + if avoid != nil { + operations.congAvoid = func(conn *GoConn, packets uint32) { + checkpoint("control", conn) + avoid(conn, packets) + checkpoint("controlled", conn) + } + } + control := operations.congControl + if control != nil { + operations.congControl = func(conn *GoConn, flags goAckFlags, sample *goRateSample) { + checkpoint("control", conn) + control(conn, flags, sample) + checkpoint("controlled", conn) + } + } + } + stack.congestion = &operations + } + fixture := newKernelStackFixture(t, config) + t.Cleanup(func() { + harness.access.Lock() + harness.stopped = true + close(harness.done) + harness.access.Unlock() + harness.workers.Wait() + }) + return fixture, harness +} + +func (h *kernelTCPHarness) eventLocked(packet []byte, outgoing bool) (*kernelTCPFlow, kernelTCPEvent) { + parsed, valid := parseForwardPacket(packet) + if !valid || parsed.protocol != uint8(header.TCPProtocolNumber) { + return nil, kernelTCPEvent{} + } + key := parsed.flowKey() + if outgoing { + key = key.reversed() + } + flow := h.flows[key] + if flow == nil { + return nil, kernelTCPEvent{} + } + tcp := header.TCP(parsed.transport) + flags := tcp.Flags() + event := kernelTCPEvent{at: time.Since(h.epoch), outgoing: outgoing, flags: flags, window: tcp.WindowSize(), length: len(tcp.Payload())} + if outgoing { + event.rate = flow.conn.pacingRate.Load() + event.sequence = uint64(uint32(tcp.SequenceNumber() - flow.isn)) + event.end = event.sequence + uint64(len(tcp.Payload())) + if flags&(header.TCPFlagSyn|header.TCPFlagFin) != 0 { + event.end++ + } + } else { + event.ack = uint64(uint32(tcp.AckNumber() - flow.isn)) + for _, block := range header.ParseTCPOptions(tcp.Options()).SACKBlocks { + event.sacks = append(event.sacks, goSackBlock{ + start: uint64(uint32(block.Start) - flow.isn), + end: uint64(uint32(block.End) - flow.isn), + }) + } + } + return flow, event +} + +func (h *kernelTCPHarness) recordLocked(flow *kernelTCPFlow, event kernelTCPEvent) { + if event.outgoing { + if !event.delivery && event.end > event.sequence && event.end <= flow.processed && flow.peerWindow != 0 && flow.failure == "" { + flow.failure = fmt.Sprintf("retransmitted acknowledged sequence [%d,%d), processed ACK %d", event.sequence, event.end, flow.processed) + } + flow.sent = max(flow.sent, event.end) + if event.flags&header.TCPFlagFin != 0 { + flow.fin = event.end + } + } else if !event.dropped && !event.delayed { + flow.peerWindow = event.window + if event.flags&header.TCPFlagAck != 0 { + flow.acked = max(flow.acked, event.ack) + } + flow.peerFIN = flow.peerFIN || event.flags&header.TCPFlagFin != 0 + } + if !event.delivery && (event.dropped || event.delayed || event.copies > 1) { + flow.faults++ + } + if len(flow.events) == 128 { + copy(flow.events, flow.events[1:]) + flow.events = flow.events[:127] + } + flow.events = append(flow.events, event) + select { + case h.changed <- struct{}{}: + default: + } +} + +func (h *kernelTCPHarness) pair(t *testing.T, fixture *kernelStackFixture, ipv6 bool, writer string) (*net.TCPConn, net.Conn, *GoConn) { + t.Helper() + var client *net.TCPConn + var sender net.Conn + if writer == "splice" { + client, sender, _, _ = fixture.splicePair(t, ipv6) + } else { + client, sender = fixture.pair(t, ipv6) + } + key := flowKey{protocol: uint8(header.TCPProtocolNumber), source: client.LocalAddr().(*net.TCPAddr).AddrPort(), destination: client.RemoteAddr().(*net.TCPAddr).AddrPort()} + return client, sender, h.snapshot(key).conn +} + +func (h *kernelTCPHarness) setFilter(conn *GoConn, filter func(kernelTCPEvent) kernelTCPAction) { + h.access.Lock() + h.flows[conn.key].filter = filter + h.access.Unlock() +} + +func (h *kernelTCPHarness) inspectLocked(platform *kernelTCPIO) { + for _, flow := range h.flows { + conn := flow.conn + if conn.engine.platformIO != platform { + continue + } + flow.processed = conn.sendUnacked.Load() + flow.closed = conn.closed() + if flow.closed || flow.failure != "" { + continue + } + flight := conn.flight + if uint64(flight.sackedOut)+uint64(flight.lostOut) > uint64(flight.packetsOut) || + uint64(flight.sackedOut)+uint64(flight.retransmitOut) > uint64(flight.packetsOut) { + flow.failure = fmt.Sprintf("invalid flight accounting: %+v", flight) + } + if conn.sendReleased.Load() > min(flow.processed, conn.transmittedTail.Load()) { + flow.failure = "send buffers released before transmission and acknowledgement completed" + } + } +} + +func (h *kernelTCPHarness) snapshot(key flowKey) kernelTCPFlow { + h.access.Lock() + defer h.access.Unlock() + flow := *h.flows[key] + flow.events = slices.Clone(flow.events) + return flow +} + +func (h *kernelTCPHarness) await(t *testing.T, conn *GoConn, description string, ready func(kernelTCPFlow) bool) kernelTCPFlow { + t.Helper() + timer := time.NewTimer(kernelTCPTimeout) + defer timer.Stop() + tick := time.NewTicker(time.Millisecond) + defer tick.Stop() + for { + flow := h.snapshot(conn.key) + if flow.failure != "" { + t.Fatalf("%s: %s; recent TCP events: %+v", description, flow.failure, flow.events[max(0, len(flow.events)-16):]) + } + if ready(flow) { + return flow + } + select { + case <-h.changed: + case <-tick.C: + case <-timer.C: + t.Fatalf("%s timed out: sent=%d ACK=%d processed=%d FIN=%d peerFIN=%v closed=%v; recent TCP events: %+v", description, flow.sent, flow.acked, flow.processed, flow.fin, flow.peerFIN, conn.closed(), flow.events[max(0, len(flow.events)-16):]) + } + } +} + +type kernelTCPIO struct { + goPlatformIO + harness *kernelTCPHarness +} + +func (p *kernelTCPIO) close() error { + p.harness.access.Lock() + p.harness.inspectLocked(p) + p.harness.access.Unlock() + return p.goPlatformIO.close() +} + +func (p *kernelTCPIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + p.harness.access.Lock() + p.harness.inspectLocked(p) + p.harness.access.Unlock() + if p.harness.config.checkpoint != nil { + p.harness.config.checkpoint("idle", nil) + } + readable, count, err := p.goPlatformIO.wait(timeout, events) + if readable && p.harness.config.receiveDelay > 0 { + timer := time.NewTimer(p.harness.config.receiveDelay) + select { + case <-timer.C: + case <-p.harness.done: + } + timer.Stop() + } + return readable, count, err +} + +func (p *kernelTCPIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + if p.harness.config.checkpoint != nil { + p.harness.config.checkpoint("receive", nil) + } + n, drained, err := p.goPlatformIO.readBurst(frames, options) + p.harness.access.Lock() + kept := 0 + for index := range n { + frame := frames[index] + packet := frame.buffer.Bytes() + p.rewriteSynOptions(packet) + flow, event := p.harness.eventLocked(packet, false) + if flow != nil { + action := kernelTCPAction{} + if flow.filter != nil { + action = flow.filter(event) + } + event.dropped = action.drop + event.delayed = action.delay > 0 + event.copies = max(action.copies, 1) + p.harness.recordLocked(flow, event) + if action.drop { + continue + } + if action.delay > 0 { + p.delayLocked(flow, event, packet, frame.meta, nil, 0, action.delay) + continue + } + for copyIndex := 1; copyIndex < action.copies; copyIndex++ { + flow.conn.engine.inject(packet, frame.meta) + } + } + frames[kept] = frame + kept++ + } + p.harness.access.Unlock() + return kept, drained, err +} + +func (p *kernelTCPIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + return p.write(frame, meta, nil, 0) +} + +func (p *kernelTCPIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + return p.write([][]byte{packet}, meta, nil, 0) +} + +func (p *kernelTCPIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + return p.write(frame, meta, owner, segmentEnd) +} + +func (p *kernelTCPIO) write(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + packet := slices.Concat(frame...) + p.harness.access.Lock() + flow, event := p.harness.eventLocked(packet, true) + action := kernelTCPAction{} + if flow != nil { + flow.writes++ + if flow.filter != nil { + action = flow.filter(event) + } + } + if action.pause != nil { + p.harness.access.Unlock() + close(action.pause.entered) + select { + case <-action.pause.resume: + case <-p.harness.done: + } + p.harness.access.Lock() + } + var err error + event.copies = max(action.copies, 1) + switch { + case action.drop: + event.dropped = true + case action.delay > 0: + event.delayed = true + p.delayLocked(flow, event, packet, meta, owner, segmentEnd, action.delay) + default: + for range event.copies { + if owner != nil { + err = p.goPlatformIO.writeData(frame, meta, owner, segmentEnd) + } else { + err = p.goPlatformIO.writeFrame(frame, meta) + } + if err != nil { + break + } + } + } + if flow != nil { + if err == nil { + p.harness.recordLocked(flow, event) + } + flow.writes-- + } + if owner != nil { + p.goPlatformIO.flush() + } + p.harness.access.Unlock() + return err +} + +func (p *kernelTCPIO) delayLocked(flow *kernelTCPFlow, event kernelTCPEvent, packet []byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64, delay time.Duration) { + if p.harness.stopped { + return + } + packet = slices.Clone(packet) + p.harness.workers.Go(func() { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-p.harness.done: + return + case <-timer.C: + } + p.harness.access.Lock() + defer p.harness.access.Unlock() + if p.harness.stopped || flow.conn.closed() { + return + } + event.at = time.Since(p.harness.epoch) + event.delayed = false + event.delivery = true + for range event.copies { + if event.outgoing { + err := p.goPlatformIO.writeData([][]byte{packet}, meta, owner, segmentEnd) + p.goPlatformIO.flush() + if err != nil && flow.failure == "" { + flow.failure = fmt.Sprintf("deliver delayed packet: %v", err) + } + } else { + flow.conn.engine.inject(packet, meta) + } + } + p.harness.recordLocked(flow, event) + }) +} + +func (p *kernelTCPIO) rewriteSynOptions(packet []byte) { + if !p.harness.config.disableSACK && !p.harness.config.disableTimestamps { + return + } + parsed, valid := parseForwardPacket(packet) + if !valid || !parsed.isPureTCPSyn() { + return + } + tcp := header.TCP(parsed.transport) + options := tcp.Options() + for index := 0; index < len(options); { + kind := options[index] + if kind == header.TCPOptionEOL { + break + } + if kind == header.TCPOptionNOP { + index++ + continue + } + if index+1 >= len(options) { + break + } + length := int(options[index+1]) + if length < 2 || index+length > len(options) { + break + } + if kind == header.TCPOptionSACKPermitted && p.harness.config.disableSACK || kind == header.TCPOptionTS && p.harness.config.disableTimestamps { + for offset := index; offset < index+length; offset++ { + options[offset] = header.TCPOptionNOP + } + } + index += length + } + var source, destination []byte + if parsed.ipVersion == 4 { + ip := header.IPv4(packet) + source, destination = ip.SourceAddressSlice(), ip.DestinationAddressSlice() + } else { + ip := header.IPv6(packet) + source, destination = ip.SourceAddressSlice(), ip.DestinationAddressSlice() + } + pseudo := header.PseudoHeaderChecksum(header.TCPProtocolNumber, source, destination, uint16(len(tcp))) + tcp.SetChecksum(0) + tcp.SetChecksum(^checksum.Checksum(tcp, pseudo)) +} diff --git a/stack_go_integration_test.go b/stack_go_integration_test.go new file mode 100644 index 00000000..cf31f6d2 --- /dev/null +++ b/stack_go_integration_test.go @@ -0,0 +1,831 @@ +//go:build (linux && !android) || (darwin && !ios) || windows + +package tun + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io" + "net" + "net/netip" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +type kernelStackConfig struct { + stackFactory func(StackOptions) (Stack, error) + congestion string + prepareStack func(*Go) + mtu uint32 + gso bool + multiQueue bool + memoryLink bool + noHandler bool + prepare func(*testing.T, *Options) + configure func(*testing.T, Options) + ctx context.Context + handshake func(*GoConn) error + udpMapping NATMapping + udpFiltering NATFiltering + dialer net.Dialer + socketBuffer int + upstreamSocketBuffer int +} + +type kernelAccept struct { + conn *GoConn + err error +} + +type kernelStackFixture struct { + stack *Go + memoryTun *MemoryTun + bridge *memoryBridge + options Options + access sync.Mutex + tcp map[uint16]chan kernelAccept + udp map[uint16]chan N.PacketConn + port atomic.Uint32 + dialer net.Dialer + socketBuffer int + upstreamSocketBuffer int + handshake func(*GoConn) error +} + +var kernelInterfaceSequence atomic.Uint32 + +func newKernelStackFixture(t *testing.T, config kernelStackConfig) *kernelStackFixture { + t.Helper() + index := kernelInterfaceSequence.Add(1) + if index >= 1<<14 { + t.Fatal("test interface address range exhausted") + } + fixture := &kernelStackFixture{ + tcp: make(map[uint16]chan kernelAccept), + udp: make(map[uint16]chan N.PacketConn), + dialer: config.dialer, + socketBuffer: config.socketBuffer, + upstreamSocketBuffer: config.upstreamSocketBuffer, + handshake: config.handshake, + options: Options{ + Name: fmt.Sprintf("gotest%d", index), + MTU: config.mtu, + GSO: config.gso, + MultiQueue: config.multiQueue, + Inet4Address: []netip.Prefix{netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 18 + byte(index>>13), byte(index >> 5), byte(index<<3) | 1}), 29)}, + Inet6Address: []netip.Prefix{netip.MustParsePrefix(fmt.Sprintf("fd73:ab91:%x::1/64", index))}, + EXP_ExternalConfiguration: true, + EXP_MultiPendingPackets: true, + Logger: logger.NOP(), + }, + } + if fixture.dialer.Timeout == 0 { + fixture.dialer.Timeout = 3 * time.Second + } + if runtime.GOOS == "darwin" { + fixture.options.Name = fmt.Sprintf("utun%d", 200+index) + } + if config.prepare != nil { + config.prepare(t, &fixture.options) + } + device, err := New(fixture.options) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { device.Close() }) + configureKernelInterface(t, device, fixture.options) + if config.configure != nil { + config.configure(t, fixture.options) + } + err = device.Start() + if err != nil { + t.Fatal(err) + } + if config.ctx == nil { + config.ctx = context.Background() + } + var stackTun Tun + stackTun = device + if config.memoryLink { + fixture.bridge = newMemoryBridge(t, device, int(fixture.options.MTU)) + fixture.memoryTun = fixture.bridge.memoryTun + stackTun = fixture.memoryTun + } + var handler Handler = fixture + if config.noHandler { + handler = nil + } + stackOptions := StackOptions{ + Context: config.ctx, Tun: stackTun, TunOptions: fixture.options, + Handler: handler, Logger: logger.NOP(), UDPTimeout: time.Minute, ICMPTimeout: time.Minute, + UDPMapping: config.udpMapping, UDPFiltering: config.udpFiltering, + TCPCongestionControl: config.congestion, + } + var stack Stack + if config.stackFactory != nil { + stack, err = config.stackFactory(stackOptions) + if err != nil { + t.Fatal(err) + } + fixture.stack, _ = stack.(*Go) + } else { + fixture.stack, err = NewGo(stackOptions) + if err != nil { + t.Fatal(err) + } + stack = fixture.stack + } + if config.prepareStack != nil { + if fixture.stack == nil { + t.Fatal("prepareStack requires the Go stack") + } + config.prepareStack(fixture.stack) + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + return fixture +} + +type memoryBridge struct { + device Tun + memoryTun *MemoryTun + inbound bridgeGate + outbound bridgeGate + outboundLargest atomic.Int64 + outboundDrop atomic.Int32 + observerAccess sync.Mutex + inboundObserver func(packet []byte) + outboundObserver func(packet []byte) + outboundFilter func(packet []byte) bool + done chan struct{} +} + +type bridgeGate struct { + access sync.Mutex + resume *sync.Cond + paused bool +} + +func (g *bridgeGate) pass() { + g.access.Lock() + for g.paused { + g.resume.Wait() + } + g.access.Unlock() +} + +func (g *bridgeGate) pause(paused bool) { + g.access.Lock() + g.paused = paused + g.access.Unlock() + g.resume.Broadcast() +} + +func newMemoryBridge(t *testing.T, device Tun, mtu int) *memoryBridge { + bridge := &memoryBridge{device: device, done: make(chan struct{})} + bridge.inbound.resume = sync.NewCond(&bridge.inbound.access) + bridge.outbound.resume = sync.NewCond(&bridge.outbound.access) + bridge.memoryTun = NewMemoryTun(MemoryTunOptions{MTU: mtu, Outbound: bridge.writeOutbound}) + t.Cleanup(func() { bridge.memoryTun.Close() }) + go bridge.pumpInbound() + t.Cleanup(func() { + close(bridge.done) + bridge.inbound.pause(false) + bridge.outbound.pause(false) + }) + return bridge +} + +func (b *memoryBridge) pumpInbound() { + storage := make([]byte, 65536+PacketOffset) + for { + n, err := b.device.Read(storage) + if err != nil { + return + } + if n <= PacketOffset { + continue + } + b.inbound.pass() + select { + case <-b.done: + return + default: + } + b.observerAccess.Lock() + observer := b.inboundObserver + b.observerAccess.Unlock() + if observer != nil { + observer(storage[PacketOffset:n]) + } + _, err = b.memoryTun.WritePackets([][]byte{storage[PacketOffset:n]}) + if err != nil { + return + } + } +} + +func (b *memoryBridge) writeOutbound(packets []*buf.Buffer) { + defer buf.ReleaseMulti(packets) + for _, packet := range packets { + b.outbound.pass() + select { + case <-b.done: + return + default: + } + frame := make([]byte, PacketOffset+packet.Len()) + copy(frame[PacketOffset:], packet.Bytes()) + err := b.forwardOutbound(frame) + if err != nil { + return + } + } +} + +func (b *memoryBridge) forwardOutbound(frame []byte) error { + packet := frame[PacketOffset:] + if b.outboundDrop.Load() > 0 && b.outboundDrop.Add(-1) >= 0 { + return nil + } + b.observerAccess.Lock() + observer := b.outboundObserver + filter := b.outboundFilter + b.observerAccess.Unlock() + if filter != nil && !filter(packet) { + return nil + } + if observer != nil { + observer(packet) + } + for { + largest := b.outboundLargest.Load() + if int64(len(packet)) <= largest || b.outboundLargest.CompareAndSwap(largest, int64(len(packet))) { + break + } + } + PacketFillHeader(frame, header.IPVersion(packet)) + _, err := b.device.Write(frame) + return err +} + +func (b *memoryBridge) pauseOutbound(paused bool) { + b.outbound.pause(paused) +} + +func (b *memoryBridge) pauseInbound(paused bool) { + b.inbound.pause(paused) +} + +func (b *memoryBridge) observe(inbound func(packet []byte), outbound func(packet []byte)) { + b.observerAccess.Lock() + b.inboundObserver = inbound + b.outboundObserver = outbound + b.observerAccess.Unlock() +} + +func (b *memoryBridge) filterOutbound(filter func(packet []byte) bool) { + b.observerAccess.Lock() + b.outboundFilter = filter + b.observerAccess.Unlock() +} + +func (f *kernelStackFixture) kernelAddress(ipv6 bool) netip.Addr { + if ipv6 { + return f.options.Inet6Address[0].Addr() + } + return f.options.Inet4Address[0].Addr() +} + +func (f *kernelStackFixture) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionAccept} +} + +func (f *kernelStackFixture) NewDNSPacket([]byte, M.Socksaddr, M.Socksaddr, N.PacketWriter) {} + +func (f *kernelStackFixture) NewConnectionEx(_ context.Context, conn net.Conn, _, destination M.Socksaddr, _ N.CloseHandlerFunc) { + f.access.Lock() + accepted := f.tcp[destination.Port] + f.access.Unlock() + if accepted == nil { + conn.Close() + return + } + goConn := conn.(*GoConn) + goConn.SetDeadline(time.Now().Add(10 * time.Second)) + var err error + if f.handshake != nil { + err = f.handshake(goConn) + } else { + err = goConn.HandshakeSuccess() + } + accepted <- kernelAccept{conn: goConn, err: err} +} + +func (f *kernelStackFixture) NewPacketConnectionEx(_ context.Context, conn N.PacketConn, _, destination M.Socksaddr, _ N.CloseHandlerFunc) { + f.access.Lock() + accepted := f.udp[destination.Port] + f.access.Unlock() + if accepted == nil { + conn.Close() + return + } + accepted <- conn +} + +func (f *kernelStackFixture) address(ipv6 bool, port uint16) string { + return netip.AddrPortFrom(f.kernelAddress(ipv6).Next(), port).String() +} + +func (f *kernelStackFixture) pair(t *testing.T, ipv6 bool) (*net.TCPConn, *GoConn) { + t.Helper() + port := uint16(20000 + f.port.Add(1)) + accepted := make(chan kernelAccept, 1) + f.access.Lock() + f.tcp[port] = accepted + f.access.Unlock() + conn, err := f.dialer.Dial("tcp", f.address(ipv6, port)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + if f.socketBuffer > 0 { + err = conn.(*net.TCPConn).SetReadBuffer(f.socketBuffer) + if err != nil { + t.Fatal(err) + } + err = conn.(*net.TCPConn).SetWriteBuffer(f.socketBuffer) + if err != nil { + t.Fatal(err) + } + } + conn.SetDeadline(time.Now().Add(10 * time.Second)) + select { + case result := <-accepted: + if result.err != nil { + t.Fatal(result.err) + } + t.Cleanup(func() { result.conn.Close() }) + return conn.(*net.TCPConn), result.conn + case <-time.After(3 * time.Second): + t.Fatal("Go connection handshake did not complete") + return nil, nil + } +} + +type kernelSocket struct { + *net.TCPConn + access sync.Mutex + owner io.Closer + closed bool +} + +func (s *kernelSocket) Attach(owner io.Closer) (io.Closer, bool) { + s.access.Lock() + defer s.access.Unlock() + if s.closed || s.owner != nil { + return nil, false + } + s.owner = owner + return s.TCPConn, true +} + +func (s *kernelSocket) Detach() { + s.access.Lock() + s.owner = nil + closed := s.closed + s.access.Unlock() + if closed { + s.TCPConn.Close() + } +} + +func (s *kernelSocket) Close() error { + s.access.Lock() + s.closed = true + owner := s.owner + s.access.Unlock() + if owner != nil { + return owner.Close() + } + return s.TCPConn.Close() +} + +func (f *kernelStackFixture) splicePair(t *testing.T, ipv6 bool) (*net.TCPConn, *net.TCPConn, *atomic.Int64, *atomic.Int64) { + t.Helper() + client, goConn := f.pair(t, ipv6) + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + remote, err := net.DialTCP("tcp4", nil, listener.Addr().(*net.TCPAddr)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { remote.Close() }) + peer, err := listener.AcceptTCP() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { peer.Close() }) + if f.upstreamSocketBuffer > 0 { + for _, socket := range []*net.TCPConn{remote, peer} { + err = socket.SetReadBuffer(f.upstreamSocketBuffer) + if err != nil { + t.Fatal(err) + } + err = socket.SetWriteBuffer(f.upstreamSocketBuffer) + if err != nil { + t.Fatal(err) + } + } + } + peer.SetDeadline(time.Now().Add(10 * time.Second)) + upload := new(atomic.Int64) + download := new(atomic.Int64) + spliced := goConn.Splice(&kernelSocket{TCPConn: remote}, SpliceOptions{ + ReadCounters: []N.CountFunc{func(n int64) { upload.Add(n) }}, + WriteCounters: []N.CountFunc{func(n int64) { download.Add(n) }}, + OnClose: func(error) { remote.Close() }, + }) + if !spliced { + t.Fatal("splice refused") + } + return client, peer, upload, download +} + +func kernelPayload(size int, seed uint32) []byte { + data := make([]byte, size) + for index := range data { + seed ^= seed << 13 + seed ^= seed >> 17 + seed ^= seed << 5 + data[index] = byte(seed) + } + return data +} + +func kernelTransfer(sender net.Conn, receiver net.Conn, payload []byte, buffered bool) error { + written := make(chan error, 1) + go func() { + var err error + if buffered { + writer := sender.(N.ExtendedWriter) + for offset := 0; offset < len(payload); { + length := min(len(payload)-offset, 32749) + buffer := buf.NewSize(length + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload[offset : offset+length])) + err = writer.WriteBuffer(buffer) + if err != nil { + break + } + offset += length + } + } else { + _, err = sender.Write(payload) + } + if err == nil { + err = N.CloseWrite(sender) + } + written <- err + }() + data, err := io.ReadAll(io.LimitReader(receiver, int64(len(payload)+1))) + if err != nil { + return err + } + if !bytes.Equal(data, payload) { + return E.New("payload mismatch: received ", len(data), " bytes, wanted ", len(payload)) + } + return <-written +} + +func TestGoKernelStream(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 9000}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true}, kernelStackConfig{mtu: 1500, multiQueue: true}, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}) + } + if runtime.GOOS != "windows" { + configs = append(configs, kernelStackConfig{mtu: 1500, memoryLink: true}, kernelStackConfig{mtu: 9000, memoryLink: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v/memory=%v", config.mtu, config.gso, config.multiQueue, config.memoryLink), func(configurationTest *testing.T) { + fixture := newKernelStackFixture(configurationTest, config) + for _, ipv6 := range []bool{false, true} { + configurationTest.Run(fmt.Sprintf("ipv6=%v", ipv6), func(addressTest *testing.T) { + addressTest.Parallel() + for _, splice := range []bool{false, true} { + if splice && config.memoryLink { + continue + } + addressTest.Run(fmt.Sprintf("duplex/splice=%v", splice), func(scenarioTest *testing.T) { + scenarioTest.Parallel() + var client, server net.Conn + var upload, download *atomic.Int64 + if splice { + client, server, upload, download = fixture.splicePair(scenarioTest, ipv6) + } else { + client, server = fixture.pair(scenarioTest, ipv6) + } + payload := kernelPayload(4<<20, 71) + response := kernelPayload(4<<20, 97) + result := make(chan error, 1) + go func() { result <- kernelTransfer(client, server, payload, false) }() + err := kernelTransfer(server, client, response, !splice) + if err != nil { + scenarioTest.Error("download:", err) + } + err = <-result + if err != nil { + scenarioTest.Error("upload:", err) + } + if splice && (upload.Load() != int64(len(payload)) || download.Load() != int64(len(response))) { + scenarioTest.Errorf("counts upload=%d download=%d", upload.Load(), download.Load()) + } + }) + } + addressTest.Run("wait_read_buffer", func(scenarioTest *testing.T) { + scenarioTest.Parallel() + client, server := fixture.pair(scenarioTest, ipv6) + server.InitializeReadWaiter(N.ReadWaitOptions{FrontHeadroom: 91, RearHeadroom: 73}) + payload := kernelPayload(256<<10, 17) + written := make(chan error, 1) + go func() { + _, writeErr := client.Write(payload) + if writeErr == nil { + writeErr = client.CloseWrite() + } + written <- writeErr + }() + var data []byte + for { + buffer, readErr := server.WaitReadBuffer() + if readErr == io.EOF { + break + } + if readErr != nil { + scenarioTest.Fatal(readErr) + } + data = append(data, buffer.Bytes()...) + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + if !bytes.Equal(data, payload) { + scenarioTest.Fatalf("read waiter received %d/%d", len(data), len(payload)) + } + writeErr := <-written + if writeErr != nil { + scenarioTest.Fatal(writeErr) + } + }) + }) + } + }) + } +} + +func TestGoKernelFlowControl(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + defer runtime.GOMAXPROCS(previous) + configs := []kernelStackConfig{{mtu: 1500}, {mtu: 9000}, {mtu: 65535}} + if runtime.GOOS == "linux" { + configs = append(configs, kernelStackConfig{mtu: 1500, gso: true, multiQueue: true}, kernelStackConfig{mtu: 9000, gso: true, multiQueue: true}) + } + for _, config := range configs { + t.Run(fmt.Sprintf("mtu=%d/gso=%v/mq=%v", config.mtu, config.gso, config.multiQueue), func(configTest *testing.T) { + config.socketBuffer = 128 << 10 + config.upstreamSocketBuffer = 128 << 10 + fixture := newKernelStackFixture(configTest, config) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + configTest.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + size := 4 << 20 + if runtime.GOOS == "windows" && mode == "splice" { + size = 16 << 20 + } + kernelBackpressure(test, fixture, ipv6, mode, size) + }) + } + } + }) + } +} + +func kernelBackpressure(t *testing.T, fixture *kernelStackFixture, ipv6 bool, mode string, size int) { + t.Helper() + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(t, ipv6) + } else { + client, server = fixture.pair(t, ipv6) + } + err := client.(*net.TCPConn).SetReadBuffer(4096) + if err != nil { + t.Fatal(err) + } + t.Logf("flow %s -> %s", client.LocalAddr(), client.RemoteAddr()) + deadline := time.Now().Add(10 * time.Second) + client.SetDeadline(deadline) + server.SetDeadline(deadline) + upload := kernelPayload(size, 113) + download := kernelPayload(size, 127) + var uploadAccepted, downloadAccepted atomic.Int64 + written := make(chan error, 2) + go func() { written <- kernelFlowWrite(client, upload, false, &uploadAccepted) }() + go func() { written <- kernelFlowWrite(server, download, mode == "buffer", &downloadAccepted) }() + time.Sleep(100 * time.Millisecond) + if uploadAccepted.Load() == int64(len(upload)) || downloadAccepted.Load() == int64(len(download)) { + t.Fatalf("stalled readers did not backpressure both writers: upload=%d download=%d", uploadAccepted.Load(), downloadAccepted.Load()) + } + read := make(chan error, 2) + go func() { read <- kernelFlowRead(server, upload, mode == "buffer") }() + go func() { read <- kernelFlowRead(client, download, false) }() + for range 2 { + err = <-read + if err != nil { + t.Error(err) + } + } + for range 2 { + err = <-written + if err != nil { + t.Error(err) + } + } +} + +func kernelFlowWrite(conn net.Conn, payload []byte, buffered bool, accepted *atomic.Int64) error { + for offset := 0; offset < len(payload); { + length := min(32749, len(payload)-offset) + var n int + var err error + if buffered { + buffer := buf.NewSize(length + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(payload[offset : offset+length])) + err = conn.(N.ExtendedWriter).WriteBuffer(buffer) + if err == nil { + n = length + } + } else { + n, err = conn.Write(payload[offset : offset+length]) + } + offset += n + accepted.Add(int64(n)) + if err != nil { + return E.Cause(err, "write flow after ", offset, " bytes") + } + } + return N.CloseWrite(conn) +} + +func kernelFlowRead(conn net.Conn, payload []byte, buffered bool) error { + var waiter N.ReadWaiter + if buffered { + var created bool + waiter, created = bufio.CreateReadWaiter(conn) + if !created { + return E.New("flow connection has no read waiter") + } + waiter.InitializeReadWaiter(N.ReadWaitOptions{MTU: 4093, FrontHeadroom: 91, RearHeadroom: 73}) + } + storage := make([]byte, 16381) + sizes := [...]int{1, 127, 4093, len(storage)} + paused := 0 + for offset, index := 0, 0; ; index++ { + if offset/65536 > paused && paused < 4 && offset < len(payload) { + paused = offset / 65536 + time.Sleep(25 * time.Millisecond) + } + var data []byte + var buffer *buf.Buffer + var err error + if buffered { + buffer, err = waiter.WaitReadBuffer() + if buffer != nil { + data = buffer.Bytes() + } + } else { + var n int + n, err = conn.Read(storage[:sizes[index%len(sizes)]]) + data = storage[:n] + } + end := offset + len(data) + matches := end <= len(payload) && bytes.Equal(data, payload[offset:min(end, len(payload))]) + if buffer != nil { + buffer.ExtendHeader(91) + buffer.Extend(73) + buffer.Release() + } + if !matches { + return E.New("flow mismatch at byte ", offset) + } + offset = end + if err == io.EOF && offset == len(payload) { + return nil + } + if err != nil { + return E.Cause(err, "read flow after ", offset, "/", len(payload), " bytes") + } + } +} + +func TestGoKernelSequenceWrap(t *testing.T) { + previous := runtime.GOMAXPROCS(4) + t.Cleanup(func() { runtime.GOMAXPROCS(previous) }) + fixture := newKernelStackFixture(t, kernelStackConfig{mtu: 9000, gso: runtime.GOOS == "linux"}) + for _, ipv6 := range []bool{false, true} { + for _, mode := range []string{"write", "buffer", "splice"} { + t.Run(fmt.Sprintf("ipv6=%v/mode=%s", ipv6, mode), func(test *testing.T) { + test.Parallel() + var client, server net.Conn + if mode == "splice" { + client, server, _, _ = fixture.splicePair(test, ipv6) + } else { + client, server = fixture.pair(test, ipv6) + } + deadline := time.Now().Add(60 * time.Second) + client.SetDeadline(deadline) + server.SetDeadline(deadline) + completed := make(chan error, 1) + go func() { completed <- kernelWrapTransfer(client, server, false) }() + err := kernelWrapTransfer(server, client, mode == "buffer") + if err != nil { + test.Error("download:", err) + } + err = <-completed + if err != nil { + test.Error("upload:", err) + } + }) + } + } +} + +func kernelWrapTransfer(sender net.Conn, receiver net.Conn, buffered bool) error { + const total = uint64(1)<<32 | 123 + const blockSize = 32768 + pattern := kernelPayload(blockSize, 53) + completed := make(chan error, 1) + go func() { + block := bytes.Clone(pattern) + for offset := uint64(0); offset < total; { + length := min(uint64(blockSize), total-offset) + binary.BigEndian.PutUint64(block, offset) + var err error + if buffered { + buffer := buf.NewSize(int(length) + 128) + buffer.Resize(128, 0) + common.Must1(buffer.Write(block[:length])) + err = sender.(N.ExtendedWriter).WriteBuffer(buffer) + } else { + _, err = sender.Write(block[:length]) + } + if err != nil { + completed <- err + return + } + offset += length + } + completed <- N.CloseWrite(sender) + }() + block := make([]byte, blockSize) + for offset := uint64(0); offset < total; { + length := min(uint64(blockSize), total-offset) + _, err := io.ReadFull(receiver, block[:length]) + if err != nil { + return E.Cause(err, "read sequence offset ", offset) + } + if binary.BigEndian.Uint64(block) != offset || !bytes.Equal(block[8:length], pattern[8:length]) { + return E.New("sequence data mismatch at ", offset) + } + offset += length + } + _, err := receiver.Read(block[:1]) + if err != io.EOF { + return E.New("sequence stream end: ", err) + } + return <-completed +} diff --git a/stack_go_integration_windows_test.go b/stack_go_integration_windows_test.go new file mode 100644 index 00000000..000ad004 --- /dev/null +++ b/stack_go_integration_windows_test.go @@ -0,0 +1,48 @@ +package tun + +import ( + "testing" + + "github.com/sagernet/sing-tun/internal/winipcfg" + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/windows" +) + +const ( + kernelConnectionRefused = windows.WSAECONNREFUSED + kernelConnectionReset = windows.WSAECONNRESET +) + +func kernelSetSocketBuffers(descriptor uintptr, size int) error { + return E.Errors( + windows.SetsockoptInt(windows.Handle(descriptor), windows.SOL_SOCKET, windows.SO_RCVBUF, size), + windows.SetsockoptInt(windows.Handle(descriptor), windows.SOL_SOCKET, windows.SO_SNDBUF, size), + ) +} + +func configureKernelInterface(t *testing.T, device Tun, options Options) { + t.Helper() + luid := winipcfg.LUID(device.(*NativeTun).adapter.LUID()) + for _, family := range []winipcfg.AddressFamily{windows.AF_INET, windows.AF_INET6} { + addresses := options.Inet4Address + if family == windows.AF_INET6 { + addresses = options.Inet6Address + } + err := luid.SetIPAddressesForFamily(family, addresses) + if err != nil { + t.Fatal(err) + } + row, err := luid.IPInterface(family) + if err != nil { + t.Fatal(err) + } + row.NLMTU = options.MTU + row.DadTransmits = 0 + row.RouterDiscoveryBehavior = winipcfg.RouterDiscoveryDisabled + err = row.Set() + if err != nil { + t.Fatal(err) + } + } +} diff --git a/stack_go_io.go b/stack_go_io.go new file mode 100644 index 00000000..9e28f88f --- /dev/null +++ b/stack_go_io.go @@ -0,0 +1,107 @@ +package tun + +import ( + "sync/atomic" + "time" + + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" + N "github.com/sagernet/sing/common/network" +) + +const ( + goTransmitBackoffMin = 250 * time.Microsecond + goTransmitBackoffMax = time.Millisecond + goTransmitBackoffBudget = 3 * time.Millisecond + goDropReportInterval = time.Second +) + +var errGoFrameDropped = E.New("go: frame dropped") + +type goFrame struct { + buffer *buf.Buffer + meta ForwardFrameMeta +} + +type goSocketEvent struct { + token uint32 + readable bool + writable bool +} + +const ( + goInterestRead uint8 = 1 << iota + goInterestWrite +) + +var errGoQueueUnavailable = E.New("go: tun queue unavailable") + +type goPlatformIO interface { + start() error + wait(timeout time.Duration, events []goSocketEvent) (tunReadable bool, count int, err error) + registerSocket(socket *goSocket, token uint32, interest uint8) error + updateSocket(socket *goSocket, interest uint8) error + unregisterSocket(socket *goSocket) + readBurst(frames []goFrame, options N.ReadWaitOptions) (count int, drained bool, err error) + writeFrame(frame [][]byte, meta ForwardFrameMeta) error + writePacket(packet []byte, meta ForwardFrameMeta) error + writeDatagram(packet []byte, meta ForwardFrameMeta) error + writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error + writePacketBatch(frames []goUDPFrame) error + transmitBacklogBelowBatch(conn *GoConn) bool + releaseReadBuffers() + flush() + mtu() int + supportsSockets() bool + transmitPrefix() int + transmitChecksumOffload() bool + transmitSegmentOffload() bool + armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) + takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters + wake() + close() error +} + +type goWriteback struct { + platformIO goPlatformIO +} + +func (w *goWriteback) ReturnHeadroom() int { + return w.platformIO.transmitPrefix() +} + +func (w *goWriteback) WriteReturnPackets(packets [][]byte) error { + prefix := w.platformIO.transmitPrefix() + var writeErr error + for _, packet := range packets { + writeErr = E.Errors(writeErr, goIgnoreDropped(w.platformIO.writeDatagram(packet[prefix:], ForwardFrameMeta{}))) + } + return writeErr +} + +func goIgnoreDropped(err error) error { + if err == errGoFrameDropped { + return nil + } + return err +} + +type goDropCounter struct { + total atomic.Uint64 + reported atomic.Uint64 + reportedAt atomic.Int64 +} + +func (c *goDropCounter) record(stackLogger logger.Logger, kind string) { + total := c.total.Add(1) + now := time.Now().UnixNano() + previous := c.reportedAt.Load() + if now-previous < int64(goDropReportInterval) { + return + } + if !c.reportedAt.CompareAndSwap(previous, now) { + return + } + stackLogger.Warn("go: dropped ", total-c.reported.Swap(total), " ", kind, ", ", total, " since start") +} diff --git a/stack_go_io_darwin.go b/stack_go_io_darwin.go new file mode 100644 index 00000000..1d13e702 --- /dev/null +++ b/stack_go_io_darwin.go @@ -0,0 +1,884 @@ +package tun + +import ( + "net" + "os" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/header" + rawfile "github.com/sagernet/sing-tun/internal/rawfile_darwin" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" +) + +const ( + goWakeIdent = 0 + goEngineTransmits = true + + goDarwinTransmitBatch = 64 + goDarwinBatchHeader = 128 + goDarwinBatchIovecs = 8 + // sendit_x builds every packet of the batch before sending any of it, and mbuf_allocpacket + // takes a 16 KB cluster for a packet above MBIGCLBYTES (4096 bytes, prefix included) from a + // pool of a few hundred; a batch of large frames exhausts it and fails with ENOBUFS, so only + // frames that fit a 4 KB cluster are batched and larger ones keep the per-frame writev. + goDarwinBatchFrameLimit = 4096 - PacketOffset + + // utun_pkt_input in netif mode returns ENOSPC once utun_input_chain_count exceeds + // net.utun.max_pending_input (512) and does not free the mbuf, and neither does ctl_send: + // every ENOSPC leaks one mbuf cluster until reboot. The chain only drains when the host side + // syncs the rx ring, which kern_channel_notify triggers per successful input, so the gate keeps + // the estimated chain depth well under the limit and, when nothing moves, submits a small probe + // to trigger a sync rather than wait forever. + goDarwinGateLimit = 512 + goDarwinGateHighWater = 384 + goDarwinGateProbe = 8 + goDarwinGatePoll = 100 * time.Microsecond + goDarwinGateSample = time.Millisecond + goDarwinGateProbeAfter = 16 + goDarwinGateRIBSize = 4096 + goDarwinGateRIBSizeMax = 1 << 20 + + sysprotoControl = 2 + utunOptionEnableNetif = 20 +) + +type goDarwinIO struct { + stack *Go + tunFd int + kqueueFd int + pollAccess sync.RWMutex + transmitAccess *sync.Mutex + receiveSlots goReadSlots + receiveIovecs []unix.Iovec + messageHeaders []rawfile.MsgHdrX + events [goSocketEventBatch + 3]unix.Kevent_t + changes [2]unix.Kevent_t + wakeEvent [1]unix.Kevent_t + transmitWritableEvent [1]unix.Kevent_t + transmitWritable atomic.Bool + blocked goBlockedWriters + receiveBatch int + transmitIovecs []unix.Iovec + socketTokens []uint32 + socketInterests []uint8 + droppedEngineFrames goDropCounter + droppedBatchFrames goDropCounter + leakedClusters goDropCounter + closing atomic.Bool + + batchHeaders []byte + batchIovecs []unix.Iovec + batchMessages []rawfile.MsgHdrX + batchSpill []byte + batchSpilled []bool + batchStart int + batchCount int + packetIovecs [goPacketBatchSize * 3]unix.Iovec + packetMessages [goPacketBatchSize]rawfile.MsgHdrX + + netif bool + interfaceIndex int + epoch time.Time + gateRIB []byte + gateSubmitted int64 + gateSampledAt int64 + gateSampledTime atomic.Int64 + gateInflightSample int64 + gateConsumedBase int64 + gateBlocked atomic.Bool + gateIdlePolls int + gateLastConsumed int64 + gateProbeEnd int64 +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goDarwinIO{stack: stack, epoch: time.Now()}}, nil +} + +func (o *goDarwinIO) start() error { + nativeTun, isNative := o.stack.tun.(*NativeTun) + if !isNative { + return E.New("go: unsupported TUN implementation") + } + err := nativeTun.detachRuntimePoller() + if err != nil { + return E.Cause(err, "go: detach tun from runtime poller") + } + o.tunFd = nativeTun.rawFileDescriptor() + o.transmitAccess = nativeTun.transmitAccess() + err = nativeTun.enableMaxPendingPackets() + if err != nil { + return E.Cause(err, "go: set UTUN_OPT_MAX_PENDING_PACKETS") + } + kqueueFd, err := unix.Kqueue() + if err != nil { + return E.Cause(err, "go: create kqueue") + } + registrations := []unix.Kevent_t{ + {Ident: uint64(o.tunFd), Filter: unix.EVFILT_READ, Flags: unix.EV_ADD | unix.EV_ENABLE}, + {Ident: goWakeIdent, Filter: unix.EVFILT_USER, Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_CLEAR}, + } + _, err = unix.Kevent(kqueueFd, registrations, nil, nil) + if err != nil { + unix.Close(kqueueFd) + return E.Cause(err, "go: register kqueue events") + } + o.kqueueFd = kqueueFd + o.wakeEvent[0] = unix.Kevent_t{Ident: goWakeIdent, Filter: unix.EVFILT_USER, Fflags: unix.NOTE_TRIGGER} + o.transmitWritableEvent[0] = unix.Kevent_t{Ident: uint64(o.tunFd), Filter: unix.EVFILT_WRITE, Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_ONESHOT} + o.receiveIovecs = make([]unix.Iovec, goReadBatch) + o.messageHeaders = make([]rawfile.MsgHdrX, goReadBatch) + o.transmitIovecs = make([]unix.Iovec, 0, 8) + o.receiveBatch = goReceiveBatchMin + o.batchHeaders = make([]byte, goDarwinTransmitBatch*goDarwinBatchHeader) + o.batchIovecs = make([]unix.Iovec, goDarwinTransmitBatch*goDarwinBatchIovecs) + o.batchMessages = make([]rawfile.MsgHdrX, goDarwinTransmitBatch) + o.batchSpilled = make([]bool, goDarwinTransmitBatch) + netif, err := unix.GetsockoptInt(o.tunFd, sysprotoControl, utunOptionEnableNetif) + if err == nil && netif != 0 { + name, nameErr := nativeTun.Name() + if nameErr != nil { + unix.Close(kqueueFd) + return E.Cause(nameErr, "go: utun name") + } + netInterface, lookupErr := net.InterfaceByName(name) + if lookupErr != nil { + unix.Close(kqueueFd) + return E.Cause(lookupErr, "go: utun interface") + } + o.netif = true + o.interfaceIndex = netInterface.Index + consumed, sampleErr := o.readInterfacePackets() + if sampleErr != nil { + unix.Close(kqueueFd) + return E.Cause(sampleErr, "go: utun statistics") + } + o.gateConsumedBase = consumed + } + return nil +} + +// The NET_RT_IFLIST2 route sysctl takes the interface index as its last name component and +// sysctl_iflist2 skips every other interface, so one interface costs a single small transfer. +func (o *goDarwinIO) readInterfacePackets() (int64, error) { + mib := [6]int32{unix.CTL_NET, unix.AF_ROUTE, 0, unix.AF_UNSPEC, unix.NET_RT_IFLIST2, int32(o.interfaceIndex)} + if o.gateRIB == nil { + o.gateRIB = make([]byte, goDarwinGateRIBSize) + } + var rib []byte + for { + length := uintptr(len(o.gateRIB)) + _, _, errno := unix.Syscall6(unix.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), uintptr(len(mib)), uintptr(unsafe.Pointer(&o.gateRIB[0])), uintptr(unsafe.Pointer(&length)), 0, 0) + if errno == unix.ENOMEM && len(o.gateRIB) < goDarwinGateRIBSizeMax { + o.gateRIB = make([]byte, 2*len(o.gateRIB)) + continue + } + if errno != 0 { + return 0, errno + } + rib = o.gateRIB[:length] + break + } + for len(rib) >= int(unsafe.Sizeof(unix.IfMsghdr2{})) { + message := (*unix.IfMsghdr2)(unsafe.Pointer(&rib[0])) + if message.Type == unix.RTM_IFINFO2 && int(message.Index) == o.interfaceIndex { + return int64(message.Data.Ipackets), nil + } + if message.Msglen == 0 { + break + } + rib = rib[message.Msglen:] + } + return 0, unix.ENOENT +} + +func (o *goDarwinIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + if o.gateBlocked.Load() { + poll := goDarwinGatePoll + if o.netif { + poll = max(goDarwinGateSample-(time.Since(o.epoch)-time.Duration(o.gateSampledTime.Load())), 0) + } + if timeout < 0 || timeout > poll { + timeout = poll + } + } + var timeoutSpec *unix.Timespec + if timeout >= 0 { + spec := unix.NsecToTimespec(timeout.Nanoseconds()) + timeoutSpec = &spec + } + var eventCount int + var err error + if timeout == 0 { + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_KEVENT, uintptr(o.kqueueFd), 0, 0, uintptr(unsafe.Pointer(&o.events[0])), uintptr(len(o.events)), uintptr(unsafe.Pointer(timeoutSpec))) + eventCount = int(count) + if errno != 0 { + err = errno + } + } else { + eventCount, err = unix.Kevent(o.kqueueFd, nil, o.events[:], timeoutSpec) + } + if o.gateBlocked.Load() { + o.gatePoll() + } + if err != nil { + if err == unix.EINTR { + return false, 0, nil + } + return false, 0, E.Cause(err, "go: kevent wait") + } + tunReadable := false + socketCount := 0 + for index := range eventCount { + event := &o.events[index] + if event.Filter == unix.EVFILT_USER { + continue + } + if int(event.Ident) == o.tunFd { + if event.Filter == unix.EVFILT_WRITE { + o.transmitWritable.Store(true) + continue + } + if event.Flags&unix.EV_ERROR != 0 && event.Data != 0 { + return false, 0, unix.Errno(event.Data) + } + if event.Flags&unix.EV_EOF != 0 { + return false, 0, unix.ECONNRESET + } + tunReadable = true + continue + } + if int(event.Ident) >= len(o.socketTokens) || socketCount == len(events) { + continue + } + events[socketCount] = goSocketEvent{ + token: o.socketTokens[event.Ident], + readable: event.Filter == unix.EVFILT_READ, + writable: event.Filter == unix.EVFILT_WRITE, + } + socketCount++ + } + return tunReadable, socketCount, nil +} + +func (o *goDarwinIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + if socket.fd >= len(o.socketTokens) { + grown := max(socket.fd+1, 2*len(o.socketTokens), 64) + o.socketTokens = append(o.socketTokens, make([]uint32, grown-len(o.socketTokens))...) + o.socketInterests = append(o.socketInterests, make([]uint8, grown-len(o.socketInterests))...) + } + o.socketTokens[socket.fd] = token + o.socketInterests[socket.fd] = 0 + return o.updateSocket(socket, interest) +} + +func (o *goDarwinIO) updateSocket(socket *goSocket, interest uint8) error { + current := o.socketInterests[socket.fd] + if current == interest { + return nil + } + changes := o.changes[:0] + if current&goInterestRead != interest&goInterestRead { + changes = append(changes, unix.Kevent_t{Ident: uint64(socket.fd), Filter: unix.EVFILT_READ, Flags: goKqueueFlags(interest&goInterestRead != 0)}) + } + if current&goInterestWrite != interest&goInterestWrite { + changes = append(changes, unix.Kevent_t{Ident: uint64(socket.fd), Filter: unix.EVFILT_WRITE, Flags: goKqueueFlags(interest&goInterestWrite != 0)}) + } + o.socketInterests[socket.fd] = interest + _, err := unix.Kevent(o.kqueueFd, changes, nil, nil) + if err != nil { + return E.Cause(err, "go: update socket interest") + } + return nil +} + +func goKqueueFlags(enable bool) uint16 { + if enable { + return unix.EV_ADD | unix.EV_ENABLE + } + return unix.EV_DELETE +} + +func (o *goDarwinIO) unregisterSocket(socket *goSocket) { + o.updateSocket(socket, 0) + o.socketTokens[socket.fd] = 0 +} + +func (o *goDarwinIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + o.receiveSlots.configure(goReadBatch, o.stack.mtu+PacketOffset+options.FrontHeadroom+options.RearHeadroom) + // recvmsg_x walks every submitted msghdr slot, whether or not a packet is pending for it. + count := min(o.receiveBatch, len(frames)) + o.receiveSlots.wake(count) + for index := range count { + buffer := o.receiveSlots.slot(index) + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + o.receiveIovecs[index] = rawfile.IovecFromBytes(buffer.FreeBytes()) + // Cannot clear only the length field. Older versions of the darwin kernel will check whether other data is empty. + // https://github.com/Darm64/XNU/blob/xnu-2782.40.9/bsd/kern/uipc_syscalls.c#L2026-L2048 + o.messageHeaders[index] = rawfile.MsgHdrX{} + o.messageHeaders[index].Msg.Iov = &o.receiveIovecs[index] + o.messageHeaders[index].Msg.Iovlen = 1 + } + received, errno := receiveMessageBatch(o.tunFd, o.messageHeaders[:count]) + if errno != 0 { + if errno == unix.EWOULDBLOCK || errno == unix.EINTR { + return 0, true, nil + } + return 0, false, E.Cause(errno, "go: recvmsg_x") + } + if received >= count { + o.receiveBatch = min(count*2, len(o.messageHeaders)) + } else if received*2 < count { + o.receiveBatch = max(count/2, goReceiveBatchMin) + } + frameCount := 0 + for index := range received { + dataLen := int(o.messageHeaders[index].DataLen) + buffer := o.receiveSlots.slot(index) + if dataLen <= PacketOffset || dataLen > buffer.FreeLen() || o.messageHeaders[index].Msg.Flags&unix.MSG_TRUNC != 0 { + continue + } + buffer.Reset() + buffer.Resize(options.FrontHeadroom+PacketOffset, dataLen-PacketOffset) + frames[frameCount] = goFrame{buffer: buffer} + frameCount++ + } + return frameCount, received == 0, nil +} + +func (o *goDarwinIO) releaseReadBuffers() { + clear(o.receiveIovecs) + clear(o.messageHeaders) + o.receiveSlots.sleep() + o.transmitAccess.Lock() + clear(o.transmitIovecs) + if o.batchCount == 0 { + clear(o.batchIovecs) + clear(o.batchMessages) + o.batchSpill = nil + } + o.transmitAccess.Unlock() +} + +func receiveMessageBatch(fd int, headers []rawfile.MsgHdrX) (int, unix.Errno) { + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_RECVMSG_X, uintptr(fd), uintptr(unsafe.Pointer(&headers[0])), uintptr(len(headers)), unix.MSG_DONTWAIT, 0, 0) + return int(n), errno +} + +func (o *goDarwinIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + o.transmitAccess.Lock() + err := o.writePacketLocked(frame) + o.transmitAccess.Unlock() + if err == errGoTransmitBlocked { + o.droppedEngineFrames.record(o.stack.logger, "engine frames") + return errGoFrameDropped + } + return err +} + +func (o *goDarwinIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + +func (o *goDarwinIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + if o.closing.Load() { + return os.ErrClosed + } + frameLength := 0 + for _, segment := range frame { + frameLength += len(segment) + } + if frameLength > goDarwinBatchFrameLimit { + return o.writeUnbatchedLocked(frame) + } + if o.batchCount == goDarwinTransmitBatch { + o.flushLocked() + if o.batchCount == goDarwinTransmitBatch { + if o.batchStart == 0 { + return errGoTransmitBlocked + } + o.compactBatch() + } + } + slot := o.batchCount + iovecs := o.batchIovecs[slot*goDarwinBatchIovecs : slot*goDarwinBatchIovecs : (slot+1)*goDarwinBatchIovecs] + if header.IPVersion(frame[0]) == header.IPv4Version { + iovecs = append(iovecs, packetHeaderVec4) + } else { + iovecs = append(iovecs, packetHeaderVec6) + } + for index, segment := range frame { + if len(segment) == 0 { + continue + } + if index == 0 && len(frame) > 1 { + if len(segment) > goDarwinBatchHeader { + return o.writeUnbatchedLocked(frame) + } + headerCopy := o.batchHeaders[slot*goDarwinBatchHeader : slot*goDarwinBatchHeader+len(segment)] + copy(headerCopy, segment) + segment = headerCopy + } + if len(iovecs) == cap(iovecs) { + return o.writeUnbatchedLocked(frame) + } + iovecs = append(iovecs, rawfile.IovecFromBytes(segment)) + } + o.batchMessages[slot] = rawfile.MsgHdrX{} + o.batchMessages[slot].Msg.Iov = &iovecs[0] + o.batchMessages[slot].Msg.Iovlen = int32(len(iovecs)) + o.batchSpilled[slot] = false + o.batchCount++ + return nil +} + +func (o *goDarwinIO) writeUnbatchedLocked(frame [][]byte) error { + o.flushLocked() + if o.batchCount > o.batchStart { + return errGoTransmitBlocked + } + return o.writePacketLocked(frame) +} + +func (o *goDarwinIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goDarwinIO) transmitBacklogBelowBatch(conn *GoConn) bool { + return true +} + +func (o *goDarwinIO) flush() { + o.transmitAccess.Lock() + o.flushLocked() + o.transmitAccess.Unlock() +} + +func (o *goDarwinIO) gateEstimate() int64 { + return o.gateInflightSample + (o.gateSubmitted - o.gateSampledAt) +} + +func (o *goDarwinIO) gateSample() (int64, bool, bool) { + if !o.netif { + return o.gateEstimate(), false, false + } + now := int64(time.Since(o.epoch)) + if now-o.gateSampledTime.Load() < int64(goDarwinGateSample) { + return o.gateEstimate(), false, false + } + o.gateSampledTime.Store(now) + consumed, err := o.readInterfacePackets() + if err != nil { + return o.gateEstimate(), false, true + } + progressed := consumed != o.gateLastConsumed + if progressed { + o.gateProbeEnd = 0 + } + o.gateLastConsumed = consumed + o.gateInflightSample = max(o.gateSubmitted-(consumed-o.gateConsumedBase), 0) + o.gateSampledAt = o.gateSubmitted + return o.gateInflightSample, progressed, true +} + +func (o *goDarwinIO) gateRoom(pending int) int { + if !o.netif { + return pending + } + estimate := o.gateEstimate() + if estimate+int64(pending) > goDarwinGateHighWater { + estimate, _, _ = o.gateSample() + } + room := goDarwinGateHighWater - estimate + room = max(room, min(o.gateProbeEnd-o.gateSubmitted, goDarwinGateLimit-1-estimate)) + return int(max(min(room, int64(pending)), 0)) +} + +// flushLocked submits the queued frames. sendmsg_x reports no count on failure. ENOBUFS comes from +// sendit_x failing to build the packets before anything is sent, so the batch is kept and retried. +// For the other errors ctl_send_list frees whatever follows the failing packet, so the call +// consumed every frame handed to it: the ones before the failure were delivered, the failing one +// is charged to the kernel (an ENOSPC leaks it), the rest are dropped and left to retransmission. +func (o *goDarwinIO) flushLocked() { + if o.closing.Load() { + return + } + pending := o.batchCount - o.batchStart + if pending == 0 { + o.gateReleaseLocked() + return + } + room := o.gateRoom(pending) + sentFrom := o.batchStart + for room > 0 { + messages := o.batchMessages[o.batchStart : o.batchStart+room] + n, errno := rawfile.NonBlockingSendMMsg(o.tunFd, messages) + if errno == unix.EINTR { + if o.closing.Load() { + return + } + continue + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + break + } + if errno != 0 { + o.batchStart += room + o.gateSubmitted += int64(room) + o.recordWriteError(errno) + break + } + o.batchStart += n + o.gateSubmitted += int64(n) + room -= n + } + clear(o.batchIovecs[sentFrom*goDarwinBatchIovecs : o.batchStart*goDarwinBatchIovecs]) + if o.batchStart == o.batchCount { + o.batchStart = 0 + o.batchCount = 0 + o.gateReleaseLocked() + return + } + o.gateBlockLocked() + o.spillHeld() +} + +func (o *goDarwinIO) gateBlockLocked() { + if !o.gateBlocked.Swap(true) { + o.wake() + } +} + +func (o *goDarwinIO) gateReleaseLocked() { + if !o.gateBlocked.Load() { + return + } + if o.gateRoom(1) == 0 { + return + } + o.gateBlocked.Store(false) + o.gateIdlePolls = 0 + o.transmitWritable.Store(true) + o.wake() +} + +// recordWriteError realigns the estimate after ENOSPC: the chain held more than the limit when +// the kernel refused, and the frames the kernel freed behind the refused one never reach the +// interface counter, so the counter baseline is moved to make the estimate read the limit now. +func (o *goDarwinIO) recordWriteError(errno unix.Errno) { + if errno == unix.ENOSPC { + o.leakedClusters.record(o.stack.logger, "frames into a full utun input queue, each leaking one kernel mbuf") + if !o.netif { + return + } + consumed, err := o.readInterfacePackets() + if err == nil { + o.gateLastConsumed = consumed + o.gateConsumedBase = consumed - (o.gateSubmitted - (goDarwinGateLimit + 1)) + } + o.gateSampledTime.Store(int64(time.Since(o.epoch))) + o.gateInflightSample = goDarwinGateLimit + 1 + o.gateSampledAt = o.gateSubmitted + o.gateProbeEnd = 0 + return + } + o.droppedBatchFrames.record(o.stack.logger, "batched frames") +} + +func (o *goDarwinIO) gatePoll() { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + if !o.gateBlocked.Load() { + return + } + _, progressed, sampled := o.gateSample() + if !sampled { + o.flushLocked() + return + } + if progressed { + o.gateIdlePolls = 0 + } else { + o.gateIdlePolls++ + } + if o.gateIdlePolls >= goDarwinGateProbeAfter { + o.gateProbeEnd = o.gateSubmitted + goDarwinGateProbe + o.gateIdlePolls = 0 + } + o.flushLocked() +} + +func (o *goDarwinIO) spillHeld() { + slotSize := o.stack.mtu + PacketOffset + if o.batchSpill == nil { + o.batchSpill = make([]byte, goDarwinTransmitBatch*slotSize) + } + for slot := o.batchStart; slot < o.batchCount; slot++ { + if o.batchSpilled[slot] { + continue + } + message := &o.batchMessages[slot] + iovecs := unsafe.Slice(message.Msg.Iov, int(message.Msg.Iovlen)) + spill := o.batchSpill[slot*slotSize : (slot+1)*slotSize] + length := 0 + for _, iovec := range iovecs[1:] { + length += copy(spill[length:], bytesFromIovec(iovec)) + } + iovecs[1] = rawfile.IovecFromBytes(spill[:length]) + clear(iovecs[2:]) + message.Msg.Iovlen = 2 + o.batchSpilled[slot] = true + } +} + +func bytesFromIovec(iovec unix.Iovec) []byte { + return unsafe.Slice(iovec.Base, int(iovec.Len)) +} + +func (o *goDarwinIO) compactBatch() { + slotSize := o.stack.mtu + PacketOffset + for slot := o.batchStart; slot < o.batchCount; slot++ { + target := slot - o.batchStart + source := &o.batchMessages[slot] + sourceIovecs := unsafe.Slice(source.Msg.Iov, int(source.Msg.Iovlen)) + targetIovecs := o.batchIovecs[target*goDarwinBatchIovecs : target*goDarwinBatchIovecs+len(sourceIovecs)] + copy(targetIovecs, sourceIovecs) + for index := range targetIovecs { + base := uintptr(unsafe.Pointer(targetIovecs[index].Base)) + headerStart := uintptr(unsafe.Pointer(&o.batchHeaders[slot*goDarwinBatchHeader])) + if base >= headerStart && base < headerStart+goDarwinBatchHeader { + copy(o.batchHeaders[target*goDarwinBatchHeader:(target+1)*goDarwinBatchHeader], o.batchHeaders[slot*goDarwinBatchHeader:(slot+1)*goDarwinBatchHeader]) + targetIovecs[index].Base = &o.batchHeaders[target*goDarwinBatchHeader+int(base-headerStart)] + continue + } + if o.batchSpilled[slot] { + spillStart := uintptr(unsafe.Pointer(&o.batchSpill[slot*slotSize])) + if base >= spillStart && base < spillStart+uintptr(slotSize) { + copy(o.batchSpill[target*slotSize:(target+1)*slotSize], o.batchSpill[slot*slotSize:(slot+1)*slotSize]) + targetIovecs[index].Base = &o.batchSpill[target*slotSize+int(base-spillStart)] + } + } + } + o.batchMessages[target] = rawfile.MsgHdrX{} + o.batchMessages[target].Msg.Iov = &targetIovecs[0] + o.batchMessages[target].Msg.Iovlen = int32(len(targetIovecs)) + o.batchSpilled[target] = o.batchSpilled[slot] + } + clear(o.batchIovecs[(o.batchCount-o.batchStart)*goDarwinBatchIovecs : o.batchCount*goDarwinBatchIovecs]) + o.batchCount -= o.batchStart + o.batchStart = 0 +} + +func (o *goDarwinIO) writePacketLocked(frame [][]byte) error { + if o.closing.Load() { + return os.ErrClosed + } + defer func() { clear(o.transmitIovecs) }() + if o.gateRoom(1) == 0 { + o.gateBlockLocked() + return errGoTransmitBlocked + } + headerVec := packetHeaderVec6 + if header.IPVersion(frame[0]) == header.IPv4Version { + headerVec = packetHeaderVec4 + } + iovecs := append(o.transmitIovecs[:0], headerVec) + for _, segment := range frame { + if len(segment) == 0 { + continue + } + iovecs = append(iovecs, rawfile.IovecFromBytes(segment)) + } + o.transmitIovecs = iovecs + for { + errno := rawfile.NonBlockingWriteIovec(o.tunFd, iovecs) + if errno == 0 { + o.gateSubmitted++ + return nil + } + if errno == unix.EINTR { + if o.closing.Load() { + return os.ErrClosed + } + continue + } + if errno == unix.ENOSPC { + o.gateSubmitted++ + o.recordWriteError(errno) + o.gateBlockLocked() + return errGoTransmitBlocked + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + if o.closing.Load() { + return os.ErrClosed + } + o.gateBlockLocked() + return errGoTransmitBlocked + } + return errno + } +} + +func (o *goDarwinIO) transmitPrefix() int { + return PacketOffset +} + +func (o *goDarwinIO) transmitChecksumOffload() bool { + return false +} + +func (o *goDarwinIO) mtu() int { + return o.stack.mtu +} + +func (o *goDarwinIO) supportsSockets() bool { + return true +} + +func (o *goDarwinIO) transmitSegmentOffload() bool { + return false +} + +func (o *goDarwinIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + o.pollAccess.RLock() + defer o.pollAccess.RUnlock() + if o.closing.Load() { + return nil, os.ErrClosed + } + if o.gateBlocked.Load() { + return &o.blocked, nil + } + _, err := unix.Kevent(o.kqueueFd, o.transmitWritableEvent[:], nil, nil) + if err != nil { + return nil, E.Cause(err, "go: arm EVFILT_WRITE") + } + return &o.blocked, nil +} + +func (o *goDarwinIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + if !o.transmitWritable.Swap(false) { + return writable + } + return append(writable, &o.blocked) +} + +func (o *goDarwinIO) wake() { + o.pollAccess.RLock() + defer o.pollAccess.RUnlock() + if o.closing.Load() { + return + } + _, _ = unix.Kevent(o.kqueueFd, o.wakeEvent[:], nil, nil) +} + +func (o *goDarwinIO) close() error { + o.closing.Store(true) + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + o.pollAccess.Lock() + defer o.pollAccess.Unlock() + clear(o.batchIovecs) + clear(o.batchMessages) + o.batchSpill = nil + o.batchCount = 0 + o.batchStart = 0 + o.receiveSlots.release() + return unix.Close(o.kqueueFd) +} + +func (o *goDarwinIO) writePacketBatch(frames []goUDPFrame) error { + o.transmitAccess.Lock() + defer o.transmitAccess.Unlock() + if o.closing.Load() { + return os.ErrClosed + } + defer func() { + clear(o.packetMessages[:]) + clear(o.packetIovecs[:]) + }() + o.flushLocked() + limit := len(frames) + for index := 0; index < len(frames); { + room := o.gateRoom(min(len(frames)-index, limit)) + if room == 0 || o.batchCount > o.batchStart { + o.gateBlockLocked() + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } + frame := &frames[index] + if frame.length+len(frame.payload) > goDarwinBatchFrameLimit { + segments := [2][]byte{frame.header[:frame.length], frame.payload} + err := o.writePacketLocked(segments[:]) + if err == errGoTransmitBlocked { + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } else if err != nil { + return err + } + index++ + continue + } + count := 0 + for count < room { + next := &frames[index+count] + if next.length+len(next.payload) > goDarwinBatchFrameLimit { + break + } + iovecs := o.packetIovecs[count*3 : (count+1)*3] + iovecs[0] = packetHeaderVec6 + if next.length == header.IPv4MinimumSize+header.UDPMinimumSize { + iovecs[0] = packetHeaderVec4 + } + iovecs[1] = rawfile.IovecFromBytes(next.header[:next.length]) + iovecCount := 2 + if len(next.payload) > 0 { + iovecs[2] = rawfile.IovecFromBytes(next.payload) + iovecCount++ + } + o.packetMessages[count] = rawfile.MsgHdrX{} + o.packetMessages[count].Msg.Iov = &iovecs[0] + o.packetMessages[count].Msg.Iovlen = int32(iovecCount) + count++ + } + written, errno := rawfile.NonBlockingSendMMsg(o.tunFd, o.packetMessages[:count]) + if errno == unix.EINTR { + if o.closing.Load() { + return os.ErrClosed + } + continue + } + if (errno == unix.EMSGSIZE || errno == unix.ENOBUFS) && count > 1 { + limit = (count + 1) / 2 + continue + } + if errno == unix.ENOBUFS || errno == unix.EAGAIN { + o.gateBlockLocked() + o.droppedEngineFrames.record(o.stack.logger, "UDP frames") + return errGoFrameDropped + } + if errno != 0 { + o.gateSubmitted += int64(count) + o.recordWriteError(errno) + return errGoFrameDropped + } + if written == 0 { + return unix.EIO + } + o.gateSubmitted += int64(written) + index += written + } + return nil +} diff --git a/stack_go_io_linux.go b/stack_go_io_linux.go new file mode 100644 index 00000000..6cd412cf --- /dev/null +++ b/stack_go_io_linux.go @@ -0,0 +1,617 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "os" + "runtime" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/unix" +) + +const goEngineTransmits = false + +var goEmptyVirtioHeader [virtioNetHdrLen]byte + +type goLinuxDevice struct { + stack *Go + vnetHeader bool + offloadProbePending atomic.Bool + transmitOffload atomic.Bool + udpTransmitDisabled atomic.Bool + droppedEngineFrames goDropCounter + droppedDataFrames goDropCounter + closing atomic.Bool +} + +type goLinuxIO struct { + device *goLinuxDevice + tun *NativeTun + tunFd int + ownsFd bool + epollFd int + eventFd int + fdAccess sync.RWMutex + receiveSlots goReadSlots + events [goSocketEventBatch + 2]unix.EpollEvent +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + nativeTun, isNative := stack.tun.(*NativeTun) + if !isNative { + return nil, E.New("go: unsupported TUN implementation") + } + err := nativeTun.detachRuntimePoller() + if err != nil { + return nil, E.Cause(err, "go: detach tun from runtime poller") + } + device := &goLinuxDevice{stack: stack} + vnetHeader, err := nativeTun.vnetHeaderEnabled() + if err != nil { + stack.logger.Warn(E.Cause(err, "go: check IFF_VNET_HDR")) + vnetHeader = false + } + device.vnetHeader = vnetHeader + if vnetHeader { + offloadErr := setUDPOffload(nativeTun.rawFileDescriptor()) + if offloadErr != nil { + offloadErr = setTCPOffload(nativeTun.rawFileDescriptor()) + } + if offloadErr != nil { + stack.logger.Warn(E.Cause(offloadErr, "go: set TSO offload")) + } else { + device.offloadProbePending.Store(true) + } + } + queueCount := 1 + if nativeTun.multiQueue { + queueCount = runtime.GOMAXPROCS(0) + } else if nativeTun.options.MultiQueue { + stack.logger.Warn("go: multi-queue requested but the tun device is single-queue") + } + queues := make([]goPlatformIO, queueCount) + queues[0] = &goLinuxIO{device: device, tun: nativeTun, tunFd: nativeTun.rawFileDescriptor()} + for index := 1; index < len(queues); index++ { + queues[index] = &goLinuxIO{device: device, tun: nativeTun, tunFd: -1} + } + return queues, nil +} + +func (o *goLinuxIO) start() error { + if o.tunFd < 0 { + queueFd, err := o.tun.openQueue() + if err != nil { + o.device.stack.logger.Warn(E.Cause(err, "go: attach tun queue")) + return errGoQueueUnavailable + } + o.tunFd = queueFd + o.ownsFd = true + } + err := o.startPoller() + if err != nil && o.ownsFd { + err = E.Errors(err, unix.Close(o.tunFd)) + } + return err +} + +func (o *goLinuxIO) startPoller() error { + epollFd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC) + if err != nil { + return E.Cause(err, "go: create epoll") + } + eventFd, err := unix.Eventfd(0, unix.EFD_CLOEXEC|unix.EFD_NONBLOCK) + if err != nil { + unix.Close(epollFd) + return E.Cause(err, "go: create eventfd") + } + err = unix.EpollCtl(epollFd, unix.EPOLL_CTL_ADD, o.tunFd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: goEpollDataTun}) + if err == nil { + err = unix.EpollCtl(epollFd, unix.EPOLL_CTL_ADD, eventFd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: goEpollDataWake}) + } + if err != nil { + return E.Errors(E.Cause(err, "go: register epoll events"), unix.Close(epollFd), unix.Close(eventFd)) + } + o.epollFd = epollFd + o.eventFd = eventFd + return nil +} + +func (o *goLinuxIO) probeTransmitOffload() error { + probeAddr := netip.AddrFrom4([4]byte{127, 0, 0, 1}) + fingerprint := []byte("sing-tun-probe-tun-gro") + segmentSize := len(fingerprint) + const ipHeaderLength = 20 + const tcpHeaderLength = 20 + totalLength := ipHeaderLength + tcpHeaderLength + 2*segmentSize + packet := make([]byte, totalLength) + probeIPv4 := header.IPv4(packet) + probeIPv4.Encode(&header.IPv4Fields{ + SrcAddr: probeAddr, + DstAddr: probeAddr, + Protocol: unix.IPPROTO_TCP, + TTL: 0, + TotalLength: uint16(totalLength), + }) + probeTCP := header.TCP(packet[ipHeaderLength:]) + probeTCP.Encode(&header.TCPFields{ + SrcPort: 0, + DstPort: 0, + SeqNum: 1, + AckNum: 1, + DataOffset: tcpHeaderLength, + Flags: header.TCPFlagAck, + WindowSize: 3000, + }) + copy(packet[ipHeaderLength+tcpHeaderLength:], fingerprint) + copy(packet[ipHeaderLength+tcpHeaderLength+segmentSize:], fingerprint) + probeIPv4.SetChecksum(^probeIPv4.CalculateChecksum()) + pseudoSum := header.PseudoHeaderChecksum(unix.IPPROTO_TCP, probeIPv4.SourceAddressSlice(), probeIPv4.DestinationAddressSlice(), uint16(tcpHeaderLength+2*segmentSize)) + probeTCP.SetChecksum(pseudoSum) + errno := o.transmitFrame([][]byte{packet}, ForwardFrameMeta{ + needsChecksum: true, + checksumStart: ipHeaderLength, + checksumOffset: header.TCPChecksumOffset, + gsoType: unix.VIRTIO_NET_HDR_GSO_TCPV4, + gsoSize: uint16(segmentSize), + }) + if errno != 0 { + return E.Cause(errno, "go: tso probe write") + } + return nil +} + +const ( + goEpollDataTun int32 = -1 + goEpollDataWake int32 = -2 +) + +func (o *goLinuxIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + milliseconds := -1 + if timeout >= 0 { + milliseconds = int((timeout + time.Millisecond - 1) / time.Millisecond) + } + var eventCount int + var err error + if timeout == 0 { + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_EPOLL_PWAIT, uintptr(o.epollFd), uintptr(unsafe.Pointer(&o.events[0])), uintptr(len(o.events)), 0, 0, 0) + eventCount = int(count) + if errno != 0 { + err = errno + } + } else { + eventCount, err = unix.EpollWait(o.epollFd, o.events[:], milliseconds) + } + if err != nil { + if err == unix.EINTR { + return false, 0, nil + } + return false, 0, E.Cause(err, "go: epoll wait") + } + tunReadable := false + socketCount := 0 + for index := range eventCount { + event := &o.events[index] + switch event.Fd { + case goEpollDataWake: + var value [8]byte + _, _ = unix.Read(o.eventFd, value[:]) + case goEpollDataTun: + if event.Events&(unix.EPOLLERR|unix.EPOLLHUP) != 0 { + return false, 0, unix.ECONNRESET + } + tunReadable = true + default: + if socketCount == len(events) { + continue + } + events[socketCount] = goSocketEvent{ + token: uint32(event.Fd), + readable: event.Events&(unix.EPOLLIN|unix.EPOLLERR|unix.EPOLLHUP|unix.EPOLLRDHUP) != 0, + writable: event.Events&unix.EPOLLOUT != 0, + } + socketCount++ + } + } + return tunReadable, socketCount, nil +} + +func (o *goLinuxIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + socket.token = token + err := unix.EpollCtl(o.epollFd, unix.EPOLL_CTL_ADD, socket.fd, &unix.EpollEvent{Events: goEpollEvents(interest), Fd: int32(token)}) + if err != nil { + return E.Cause(err, "go: register socket") + } + socket.registered = true + return nil +} + +func (o *goLinuxIO) updateSocket(socket *goSocket, interest uint8) error { + operation := unix.EPOLL_CTL_MOD + if interest == 0 { + operation = unix.EPOLL_CTL_DEL + } else if !socket.registered { + operation = unix.EPOLL_CTL_ADD + } + err := unix.EpollCtl(o.epollFd, operation, socket.fd, &unix.EpollEvent{Events: goEpollEvents(interest), Fd: int32(socket.token)}) + if err != nil { + return E.Cause(err, "go: update socket interest") + } + socket.registered = interest != 0 + return nil +} + +func (o *goLinuxIO) unregisterSocket(socket *goSocket) { + if !socket.registered { + return + } + unix.EpollCtl(o.epollFd, unix.EPOLL_CTL_DEL, socket.fd, nil) + socket.registered = false +} + +func goEpollEvents(interest uint8) uint32 { + var events uint32 + if interest&goInterestRead != 0 { + events |= unix.EPOLLIN | unix.EPOLLRDHUP + } + if interest&goInterestWrite != 0 { + events |= unix.EPOLLOUT + } + return events +} + +func (o *goLinuxIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + // Linux hands pre-segmentation TSO aggregates to the TUN fd even with IFF_VNET_HDR off + // (observed on 6.x kernels). + o.receiveSlots.configure(goPacketBatchSize, virtioNetHdrLen+gsoMaxSize+options.FrontHeadroom+options.RearHeadroom) + limit := min(len(frames), goPacketBatchSize) + o.receiveSlots.wake(limit) + count := 0 + for count < limit { + buffer := o.receiveSlots.slot(count) + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + scratch := buffer.FreeBytes() + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_READ, uintptr(o.tunFd), uintptr(unsafe.Pointer(&scratch[0])), uintptr(len(scratch))) + if errno != 0 { + if errno == unix.EAGAIN { + return count, true, nil + } + if errno == unix.EINTR { + continue + } + return 0, false, E.Cause(errno, "go: read tun") + } + if n == 0 { + return 0, false, os.ErrClosed + } + // The kernel rejects writes with EIO while the device is not up (tun_get_user). + if o.device.offloadProbePending.CompareAndSwap(true, false) { + probeErr := o.probeTransmitOffload() + if probeErr != nil { + o.device.stack.logger.Warn(E.Cause(probeErr, "go: TSO write probe")) + } else { + o.device.transmitOffload.Store(true) + } + } + packet := scratch[:n] + if !o.device.vnetHeader { + buffer.Truncate(int(n)) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer} + count++ + continue + } + payload, virtioOptions, parseErr := parseVirtioRead(packet) + if parseErr != nil { + continue + } + var meta ForwardFrameMeta + switch virtioOptions.GSOType { + case GSONone: + case GSOTCPv4: + meta.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 + meta.gsoSize = virtioOptions.GSOSize + case GSOTCPv6: + meta.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6 + meta.gsoSize = virtioOptions.GSOSize + case GSOUDPL4: + meta.gsoType = goUDPGSOType + meta.gsoSize = virtioOptions.GSOSize + } + meta.needsChecksum = virtioOptions.NeedsCsum + meta.checksumStart = virtioOptions.CsumStart + meta.checksumOffset = virtioOptions.CsumOffset + buffer.Resize(options.FrontHeadroom+len(packet)-len(payload), len(payload)) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer, meta: meta} + count++ + } + return count, false, nil +} + +func (o *goLinuxIO) releaseReadBuffers() { + o.receiveSlots.sleep() +} + +func (o *goLinuxIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + for { + errno := o.transmitFrame(frame, meta) + switch errno { + case 0: + return nil + case unix.EINTR: + if o.device.closing.Load() { + return os.ErrClosed + } + case unix.EAGAIN: + o.device.droppedEngineFrames.record(o.device.stack.logger, "engine frames") + return errGoFrameDropped + default: + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } + return E.Cause(errno, "go: write tun") + } + } +} + +func (o *goLinuxIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + +func (o *goLinuxIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + delay := goTransmitBackoffMin + waited := time.Duration(0) + for { + errno := o.transmitFrame(frame, meta) + switch errno { + case 0: + return nil + case unix.EINTR: + if o.device.closing.Load() { + return os.ErrClosed + } + case unix.EAGAIN: + // tun chardev writes complete into netif_rx, which drops on backlog overflow + // instead of reporting EAGAIN, and tun_chr_poll reports the fd always writable. + if o.device.closing.Load() { + return os.ErrClosed + } + if waited >= goTransmitBackoffBudget { + o.device.droppedDataFrames.record(o.device.stack.logger, "data frames") + return errGoFrameDropped + } + time.Sleep(delay) + if o.device.closing.Load() { + return os.ErrClosed + } + waited += delay + delay = min(2*delay, goTransmitBackoffMax) + default: + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } + return E.Cause(errno, "go: write tun") + } + } +} + +func (o *goLinuxIO) transmitFrame(frame [][]byte, meta ForwardFrameMeta) unix.Errno { + o.fdAccess.RLock() + defer o.fdAccess.RUnlock() + if o.device.closing.Load() { + return unix.EBADF + } + var headerStorage [virtioNetHdrLen]byte + var iovecStorage [goPacketBatchSize + 2]unix.Iovec + iovecs := iovecStorage[:] + iovecCapacity := len(frame) + if o.device.vnetHeader { + iovecCapacity++ + } + if iovecCapacity > len(iovecs) { + iovecs = make([]unix.Iovec, iovecCapacity) + } + iovecCount := 0 + if o.device.vnetHeader { + prefix := goEmptyVirtioHeader[:] + if meta != (ForwardFrameMeta{}) { + virtioHeader := virtioNetHdr{gsoType: meta.gsoType, gsoSize: meta.gsoSize} + if meta.needsChecksum { + virtioHeader.flags = unix.VIRTIO_NET_HDR_F_NEEDS_CSUM + virtioHeader.csumStart = meta.checksumStart + virtioHeader.csumOffset = meta.checksumOffset + } + if meta.gsoType == goUDPGSOType { + virtioHeader.hdrLen = meta.checksumStart + header.UDPMinimumSize + } else if meta.gsoType != unix.VIRTIO_NET_HDR_GSO_NONE { + virtioHeader.hdrLen = meta.checksumStart + uint16(header.TCP(frame[0][meta.checksumStart:]).DataOffset()) + } + common.Must(virtioHeader.encode(headerStorage[:])) + prefix = headerStorage[:] + } + vector := unix.Iovec{Base: &prefix[0]} + vector.SetLen(len(prefix)) + iovecs[iovecCount] = vector + iovecCount++ + } + for _, segment := range frame { + if len(segment) == 0 { + continue + } + vector := unix.Iovec{Base: &segment[0]} + vector.SetLen(len(segment)) + iovecs[iovecCount] = vector + iovecCount++ + } + //nolint:staticcheck + _, _, errno := unix.RawSyscall(unix.SYS_WRITEV, uintptr(o.tunFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(iovecCount)) + return errno +} + +func (o *goLinuxIO) mtu() int { + return o.device.stack.mtu +} + +func (o *goLinuxIO) supportsSockets() bool { + return true +} + +func (o *goLinuxIO) transmitPrefix() int { + if o.device.vnetHeader { + return virtioNetHdrLen + } + return 0 +} + +func (o *goLinuxIO) transmitChecksumOffload() bool { + return o.device.vnetHeader +} + +func (o *goLinuxIO) transmitSegmentOffload() bool { + return o.device.transmitOffload.Load() +} + +func (o *goLinuxIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + return nil, nil +} + +func (o *goLinuxIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + return writable +} + +func (o *goLinuxIO) wake() { + o.fdAccess.RLock() + defer o.fdAccess.RUnlock() + if o.device.closing.Load() { + return + } + var value [8]byte + binary.NativeEndian.PutUint64(value[:], 1) + _, _ = unix.Write(o.eventFd, value[:]) +} + +func (o *goLinuxIO) close() error { + o.device.closing.Store(true) + o.fdAccess.Lock() + defer o.fdAccess.Unlock() + o.receiveSlots.release() + err := E.Errors(unix.Close(o.epollFd), unix.Close(o.eventFd)) + if o.ownsFd { + err = E.Errors(err, unix.Close(o.tunFd)) + } + return err +} + +func (o *goLinuxIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goLinuxIO) transmitBacklogBelowBatch(conn *GoConn) bool { + return true +} + +func (o *goLinuxIO) flush() { +} + +func (o *goLinuxIO) writePacketBatch(frames []goUDPFrame) error { + if o.device.closing.Load() { + return os.ErrClosed + } + var writeError error + var segments [goPacketBatchSize + 1][]byte + for index := 0; index < len(frames); { + frame := &frames[index] + end := index + 1 + payloadLength := len(frame.payload) + if o.device.vnetHeader && !o.device.udpTransmitDisabled.Load() && payloadLength > 0 { + for end < len(frames) { + next := &frames[end] + if next.length != frame.length || len(next.payload) == 0 || len(next.payload) > len(frame.payload) || frame.length+payloadLength+len(next.payload) > 65535 { + break + } + udpOffset := frame.length - header.UDPMinimumSize + if binary.BigEndian.Uint32(next.header[udpOffset:]) != binary.BigEndian.Uint32(frame.header[udpOffset:]) { + break + } + if frame.length == header.IPv4MinimumSize+header.UDPMinimumSize { + if [8]byte(next.header[12:20]) != [8]byte(frame.header[12:20]) { + break + } + } else if [32]byte(next.header[8:40]) != [32]byte(frame.header[8:40]) { + break + } + payloadLength += len(next.payload) + end++ + if len(next.payload) < len(frame.payload) { + break + } + } + } + packetHeader := frame.header + meta := frame.meta + if end-index > 1 { + udpLength := uint16(header.UDPMinimumSize + payloadLength) + var sourceAddress, destinationAddress []byte + if frame.length == header.IPv4MinimumSize+header.UDPMinimumSize { + ipHdr := header.IPv4(frame.header[:header.IPv4MinimumSize]) + ipHdr.SetTotalLength(uint16(frame.length + payloadLength)) + ipHdr.SetChecksum(0) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } else { + ipHdr := header.IPv6(frame.header[:header.IPv6MinimumSize]) + ipHdr.SetPayloadLength(udpLength) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } + udpHdr := header.UDP(frame.header[frame.length-header.UDPMinimumSize : frame.length]) + udpHdr.SetLength(udpLength) + udpHdr.SetChecksum(header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpLength)) + meta.gsoType = goUDPGSOType + meta.gsoSize = uint16(len(frame.payload)) + } + segments[0] = frame.header[:frame.length] + for packetIndex := index; packetIndex < end; packetIndex++ { + segments[packetIndex-index+1] = frames[packetIndex].payload + } + errno := o.transmitFrame(segments[:end-index+1], meta) + if errno == unix.EINTR { + frame.header = packetHeader + continue + } + if errno == unix.EBADF && o.device.closing.Load() { + return os.ErrClosed + } + if errno != 0 && end-index > 1 { + switch errno { + case unix.EINVAL, unix.EIO, unix.EOPNOTSUPP: + o.device.udpTransmitDisabled.Store(true) + frame.header = packetHeader + continue + } + } + if errno == unix.EAGAIN { + o.device.droppedEngineFrames.record(o.device.stack.logger, "UDP frames") + writeError = E.Errors(writeError, errGoFrameDropped) + } else if errno != 0 { + writeError = E.Errors(writeError, E.Cause(errno, "go: write UDP batch")) + } + index = end + } + return writeError +} diff --git a/stack_go_io_memory.go b/stack_go_io_memory.go new file mode 100644 index 00000000..c319aee7 --- /dev/null +++ b/stack_go_io_memory.go @@ -0,0 +1,262 @@ +package tun + +import ( + "os" + "time" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" +) + +type goMemoryIO struct { + stack *Go + tun *MemoryTun + timer *time.Timer + handed []*buf.Buffer + blocked map[*OutboundQueue]*goBlockedWriters + writableSpare []*OutboundQueue +} + +func (o *goMemoryIO) start() error { + o.tun.attached.Store(true) + return nil +} + +func (o *goMemoryIO) inboundReady() (bool, error) { + if o.tun.closed.Load() { + return false, os.ErrClosed + } + o.tun.inboundAccess.Lock() + defer o.tun.inboundAccess.Unlock() + return !o.tun.inbound.empty(), nil +} + +func (o *goMemoryIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + ready, err := o.inboundReady() + if ready || err != nil || timeout == 0 { + return ready, 0, err + } + if timeout < 0 { + select { + case <-o.tun.engineWake: + case <-o.tun.closeSignal: + } + } else { + if o.timer == nil { + o.timer = time.NewTimer(timeout) + } else { + o.timer.Reset(timeout) + } + select { + case <-o.tun.engineWake: + o.timer.Stop() + case <-o.tun.closeSignal: + o.timer.Stop() + case <-o.timer.C: + } + } + ready, err = o.inboundReady() + return ready, 0, err +} + +func (o *goMemoryIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + return nil +} + +func (o *goMemoryIO) updateSocket(socket *goSocket, interest uint8) error { + return nil +} + +func (o *goMemoryIO) unregisterSocket(socket *goSocket) { +} + +func (o *goMemoryIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + o.releaseReadBuffers() + if o.tun.closed.Load() { + return 0, false, os.ErrClosed + } + o.tun.inboundHeadroom.Store(int32(options.FrontHeadroom)) + o.tun.inboundRearSpace.Store(int32(options.RearHeadroom)) + o.tun.inboundAccess.Lock() + defer o.tun.inboundAccess.Unlock() + count := 0 + for count < len(frames) && !o.tun.inbound.empty() { + buffer, _, _ := o.tun.inbound.pop() + if options.NeedHeadroom() { + buffer = options.Copy(buffer) + options.PostReturn(buffer) + } + o.handed = append(o.handed, buffer) + frames[count] = goFrame{buffer: buffer} + count++ + } + return count, o.tun.inbound.empty(), nil +} + +func (o *goMemoryIO) releaseReadBuffers() { + buf.ReleaseMulti(o.handed) + o.handed = o.handed[:0] +} + +type goOutboundClass uint8 + +const ( + goOutboundControl goOutboundClass = iota + goOutboundDatagram + goOutboundData +) + +func (o *goMemoryIO) enqueue(segments [][]byte, class goOutboundClass, owner *GoConn, segmentEnd uint64) error { + if o.tun.closed.Load() || !o.tun.attached.Load() { + return os.ErrClosed + } + length := 0 + for _, segment := range segments { + length += len(segment) + } + packet := o.tun.newOutboundPacket(length) + for _, segment := range segments { + common.Must1(packet.Write(segment)) + } + queue := o.tun.outbound + if o.tun.route != nil { + routed := o.tun.route(packet.Bytes()) + if routed != nil { + queue = routed + } + } + if owner != nil { + owner.outboundQueue.Store(queue) + } + var fragments [][]byte + mtu := o.tun.MTU() + if length > mtu { + var fragmentsBuilt bool + fragments, fragmentsBuilt = goFragmentPacket(&o.stack.fragmentIdentification, packet.Bytes(), mtu) + if !fragmentsBuilt { + packet.Release() + return errGoFrameDropped + } + } + return queue.enqueue(packet, fragments, class, owner, segmentEnd) +} + +func (o *goMemoryIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + return o.enqueue(frame, goOutboundControl, nil, 0) +} + +func (o *goMemoryIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + segments := [1][]byte{packet} + return o.enqueue(segments[:], goOutboundControl, nil, 0) +} + +func (o *goMemoryIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + segments := [1][]byte{packet} + return o.enqueue(segments[:], goOutboundDatagram, nil, 0) +} + +func (o *goMemoryIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + return o.enqueue(frame, goOutboundData, owner, segmentEnd) +} + +func (o *goMemoryIO) writePacketBatch(frames []goUDPFrame) error { + var writeErr error + for index := range frames { + frame := &frames[index] + segments := [2][]byte{frame.header[:frame.length], frame.payload} + writeErr = E.Errors(writeErr, o.enqueue(segments[:], goOutboundDatagram, nil, 0)) + } + return writeErr +} + +func (o *goMemoryIO) transmitBacklogBelowBatch(conn *GoConn) bool { + queue := conn.outboundQueue.Load() + if queue == nil { + queue = o.tun.outbound + } + return int(queue.dataQueued.Load()) < o.tun.batchSize +} + +func (o *goMemoryIO) flush() { +} + +func (o *goMemoryIO) mtu() int { + return o.tun.MTU() +} + +func (o *goMemoryIO) supportsSockets() bool { + return false +} + +func (o *goMemoryIO) transmitPrefix() int { + return 0 +} + +func (o *goMemoryIO) transmitChecksumOffload() bool { + return false +} + +func (o *goMemoryIO) transmitSegmentOffload() bool { + return false +} + +func (o *goMemoryIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + queue := o.tun.outbound + if conn != nil { + routed := conn.outboundQueue.Load() + if routed != nil { + queue = routed + } + } + queue.access.Lock() + armed := !queue.closed.Load() && queue.data.full() + if armed { + queue.transmitInterest = true + } + queue.access.Unlock() + if !armed { + return nil, nil + } + waiters := o.blocked[queue] + if waiters == nil { + waiters = new(goBlockedWriters) + o.blocked[queue] = waiters + } + return waiters, nil +} + +func (o *goMemoryIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + queues := o.tun.takeWritable(o.writableSpare) + for _, queue := range queues { + waiters := o.blocked[queue] + if waiters == nil { + continue + } + if queue.closed.Load() { + delete(o.blocked, queue) + } + writable = append(writable, waiters) + } + clear(queues) + o.writableSpare = queues[:0] + return writable +} + +func (o *goMemoryIO) wake() { + o.tun.engineWake.notify() +} + +func (o *goMemoryIO) close() error { + o.releaseReadBuffers() + o.tun.attached.Store(false) + o.tun.inboundAccess.Lock() + o.tun.inbound.releaseAll() + o.tun.inboundAccess.Unlock() + if o.timer != nil { + o.timer.Stop() + } + clear(o.blocked) + return nil +} diff --git a/stack_go_io_other.go b/stack_go_io_other.go new file mode 100644 index 00000000..5dc00117 --- /dev/null +++ b/stack_go_io_other.go @@ -0,0 +1,72 @@ +//go:build !(linux || windows || darwin) + +package tun + +import ( + "net/netip" + "syscall" + + E "github.com/sagernet/sing/common/exceptions" +) + +const goEngineTransmits = false + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return nil, E.New("go: unsupported platform") +} + +const goSpliceDuplicatesSocket = false + +type goIOVector struct{} + +type goSocket struct{} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + return goSocket{}, E.New("go: splice not supported") +} + +func (s *goSocket) close() { +} + +func (s *goSocket) readVector(iovecs []goIOVector) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) writeVector(iovecs []goIOVector) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + return syscall.ENOSYS +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + return 0, netip.AddrPort{}, syscall.ENOSYS +} + +func (s *goSocket) shutdownWrite() { +} + +func (s *goSocket) family() (uint8, error) { + return 0, syscall.ENOSYS +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + return netip.AddrPort{}, false +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + return iovecs[:0] +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK || errno == syscall.EINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == syscall.EMSGSIZE || errno == syscall.EAFNOSUPPORT || errno == syscall.ENOBUFS +} diff --git a/stack_go_io_windows.go b/stack_go_io_windows.go new file mode 100644 index 00000000..7627542c --- /dev/null +++ b/stack_go_io_windows.go @@ -0,0 +1,506 @@ +package tun + +import ( + "os" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/internal/afd" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "golang.org/x/sys/windows" +) + +const goEngineTransmits = false + +const ( + goCompletionKeyTun uintptr = iota + 1 + goCompletionKeyWake + goCompletionKeySocket +) + +const goAFDReadEvents = afd.POLL_RECEIVE | afd.POLL_DISCONNECT | afd.POLL_ABORT | afd.POLL_LOCAL_CLOSE | afd.POLL_CONNECT_FAIL + +type goWindowsIO struct { + stack *Go + tun *NativeTun + iocp windows.Handle + afd *afd.Device + waitPacket *afd.WaitCompletionPacket + waitArmed bool + bridgeArm windows.Handle + bridgeClose windows.Handle + bridgeDone chan struct{} + entries map[*goAFDEntry]struct{} + completions [goSocketEventBatch + 2]afd.OverlappedEntry + receiveSlots goReadSlots + // wintun's read-wait event is auto-reset and only set by the driver when it appends to + // the ring; WintunReceivePacket neither re-signals nor resets it (wintun api/session.c: + // CreateEventW(&SecurityAttributes, FALSE, FALSE, NULL)). + ringDrained bool + droppedEngineFrames goDropCounter + wakeAccess sync.RWMutex + closing atomic.Bool + droppedDataFrames goDropCounter +} + +func newGoPlatformQueues(stack *Go) ([]goPlatformIO, error) { + return []goPlatformIO{&goWindowsIO{stack: stack}}, nil +} + +func (o *goWindowsIO) start() error { + nativeTun, isNative := o.stack.tun.(*NativeTun) + if !isNative { + return E.New("go: unsupported TUN implementation") + } + o.tun = nativeTun + iocp, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 1) + if err != nil { + return E.Cause(err, "go: create completion port") + } + o.iocp = iocp + device, err := afd.Open(iocp, "sing-tun") + if err != nil { + windows.CloseHandle(iocp) + return E.Cause(err, "go: open afd device") + } + o.afd = device + if afd.WaitCompletionPacketSupported() { + waitPacket, packetErr := afd.NewWaitCompletionPacket() + if packetErr != nil { + device.Close() + windows.CloseHandle(iocp) + return E.Cause(packetErr, "go: create wait completion packet") + } + o.waitPacket = waitPacket + } else { + armEvent, eventErr := windows.CreateEvent(nil, 0, 0, nil) + if eventErr != nil { + device.Close() + windows.CloseHandle(iocp) + return E.Cause(eventErr, "go: create read wait event") + } + closeEvent, eventErr := windows.CreateEvent(nil, 1, 0, nil) + if eventErr != nil { + windows.CloseHandle(armEvent) + device.Close() + windows.CloseHandle(iocp) + return E.Cause(eventErr, "go: create read wait close event") + } + o.bridgeArm = armEvent + o.bridgeClose = closeEvent + o.bridgeDone = make(chan struct{}) + go o.bridgeReadWait() + } + o.entries = make(map[*goAFDEntry]struct{}) + o.ringDrained = true + return nil +} + +func (o *goWindowsIO) bridgeReadWait() { + defer close(o.bridgeDone) + armHandles := []windows.Handle{o.bridgeArm, o.bridgeClose} + readHandles := []windows.Handle{o.tun.readWaitHandle(), o.bridgeClose} + for { + signaled, err := windows.WaitForMultipleObjects(armHandles, false, windows.INFINITE) + if err != nil || signaled != windows.WAIT_OBJECT_0 { + return + } + signaled, err = windows.WaitForMultipleObjects(readHandles, false, windows.INFINITE) + if err != nil || signaled != windows.WAIT_OBJECT_0 { + return + } + windows.PostQueuedCompletionStatus(o.iocp, 0, goCompletionKeyTun, nil) + } +} + +func (o *goWindowsIO) armReadWait() (bool, error) { + if o.waitArmed { + return false, nil + } + o.waitArmed = true + if o.waitPacket == nil { + windows.SetEvent(o.bridgeArm) + return false, nil + } + alreadySignaled, err := o.waitPacket.Associate(o.iocp, o.tun.readWaitHandle(), goCompletionKeyTun) + if err != nil { + o.waitArmed = false + return false, E.Cause(err, "go: associate wintun read event") + } + if alreadySignaled { + o.waitArmed = false + return true, nil + } + return false, nil +} + +func (o *goWindowsIO) wait(timeout time.Duration, events []goSocketEvent) (bool, int, error) { + tunReadable := !o.ringDrained + if o.ringDrained { + signaled, err := o.armReadWait() + if err != nil { + return false, 0, err + } + if signaled { + tunReadable = true + o.waitPacket.Cancel() + } + } + var waitMillis uint32 + if tunReadable || timeout == 0 { + waitMillis = 0 + } else if timeout < 0 { + waitMillis = windows.INFINITE + } else { + waitMillis = uint32((timeout + time.Millisecond - 1) / time.Millisecond) + } + var removed uint32 + errno := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) + if errno != 0 { + if errno == windows.WAIT_TIMEOUT { + return tunReadable, 0, nil + } + return false, 0, E.Cause(errno, "go: wait for completion") + } + socketCount := 0 + for index := range removed { + completion := &o.completions[index] + switch completion.CompletionKey { + case goCompletionKeyTun: + o.waitArmed = false + tunReadable = true + case goCompletionKeyWake: + default: + entry := (*goAFDEntry)(unsafe.Pointer(completion.Overlapped)) + entry.armed = false + if entry.cancelled { + o.releaseEntry(entry) + continue + } + var pollEvents uint32 + if entry.pollInfo.NumberOfHandles > 0 { + pollEvents = entry.pollInfo.Handles[0].Events + } + if uint32(entry.ioStatusBlock.Status) != afd.STATUS_CANCELLED && socketCount < len(events) { + readable := pollEvents&goAFDReadEvents != 0 || entry.ioStatusBlock.Status != 0 + writable := pollEvents&afd.POLL_SEND != 0 + if readable || writable { + events[socketCount] = goSocketEvent{token: entry.token, readable: readable, writable: writable} + socketCount++ + } + } + if entry.interest != 0 { + armErr := o.armEntry(entry) + if armErr != nil && socketCount < len(events) { + events[socketCount] = goSocketEvent{token: entry.token, readable: true} + socketCount++ + } + } + } + } + return tunReadable, socketCount, nil +} + +func (o *goWindowsIO) armEntry(entry *goAFDEntry) error { + var pollEvents uint32 + if entry.interest&goInterestRead != 0 { + pollEvents |= goAFDReadEvents + } + if entry.interest&goInterestWrite != 0 { + pollEvents |= afd.POLL_SEND + } + err := o.afd.Poll(entry.baseHandle, pollEvents, &entry.ioStatusBlock, &entry.pollInfo) + if err != nil { + return err + } + entry.armed = true + return nil +} + +func (o *goWindowsIO) registerSocket(socket *goSocket, token uint32, interest uint8) error { + baseHandle, err := afd.BaseSocket(socket.handle) + if err != nil { + return E.Cause(err, "go: query base socket") + } + entry := &goAFDEntry{baseHandle: baseHandle, token: token, interest: interest} + entry.pinner.Pin(entry) + o.entries[entry] = struct{}{} + socket.entry = entry + if interest == 0 { + return nil + } + err = o.armEntry(entry) + if err != nil { + o.releaseEntry(entry) + socket.entry = nil + return E.Cause(err, "go: poll socket") + } + return nil +} + +func (o *goWindowsIO) releaseEntry(entry *goAFDEntry) { + delete(o.entries, entry) + entry.pinner.Unpin() +} + +func (o *goWindowsIO) updateSocket(socket *goSocket, interest uint8) error { + entry := socket.entry + if entry == nil || entry.interest == interest { + return nil + } + entry.interest = interest + if entry.armed { + return o.afd.Cancel(&entry.ioStatusBlock) + } + if interest == 0 { + return nil + } + err := o.armEntry(entry) + if err != nil { + return E.Cause(err, "go: poll socket") + } + return nil +} + +func (o *goWindowsIO) unregisterSocket(socket *goSocket) { + entry := socket.entry + if entry == nil { + return + } + socket.entry = nil + entry.interest = 0 + entry.cancelled = true + if !entry.armed { + o.releaseEntry(entry) + return + } + o.afd.Cancel(&entry.ioStatusBlock) +} + +func (o *goWindowsIO) readBurst(frames []goFrame, options N.ReadWaitOptions) (int, bool, error) { + o.receiveSlots.configure(goReadBatch, o.stack.mtu+options.FrontHeadroom+options.RearHeadroom) + limit := min(len(frames), goReadBatch) + o.receiveSlots.wake(limit) + count := 0 + for count < limit { + buffer := o.receiveSlots.slot(count) + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + n, err := o.tun.receiveInto(buffer.FreeBytes()) + if err != nil { + return 0, false, err + } + if n == 0 { + o.ringDrained = true + return count, true, nil + } + buffer.Truncate(n) + options.PostReturn(buffer) + frames[count] = goFrame{buffer: buffer} + count++ + } + o.ringDrained = false + return count, false, nil +} + +func (o *goWindowsIO) releaseReadBuffers() { + o.receiveSlots.sleep() +} + +func (o *goWindowsIO) writePacketBatch(frames []goUDPFrame) error { + var writeError error + var segments [2][]byte + for index := range frames { + frame := &frames[index] + segments[0] = frame.header[:frame.length] + segments[1] = frame.payload + err := o.writeFrame(segments[:], frame.meta) + if err != nil { + writeError = E.Errors(writeError, err) + } + } + return writeError +} + +func (o *goWindowsIO) writeFrame(frame [][]byte, meta ForwardFrameMeta) error { + err := o.transmitFrame(frame) + if err == windows.ERROR_BUFFER_OVERFLOW { + o.droppedEngineFrames.record(o.stack.logger, "engine frames") + return errGoFrameDropped + } + return err +} + +func (o *goWindowsIO) writePacket(packet []byte, meta ForwardFrameMeta) error { + frame := [1][]byte{packet} + return o.writeFrame(frame[:], meta) +} + +func (o *goWindowsIO) writeData(frame [][]byte, meta ForwardFrameMeta, owner *GoConn, segmentEnd uint64) error { + backoff := goTransmitBackoffMin + waited := time.Duration(0) + for { + err := o.transmitFrame(frame) + if err != windows.ERROR_BUFFER_OVERFLOW { + return err + } + if o.closing.Load() { + return os.ErrClosed + } + if waited >= goTransmitBackoffBudget { + o.droppedDataFrames.record(o.stack.logger, "data frames") + return errGoFrameDropped + } + time.Sleep(backoff) + waited += backoff + backoff = min(backoff*2, goTransmitBackoffMax) + } +} + +func (o *goWindowsIO) transmitFrame(frame [][]byte) error { + if o.closing.Load() { + return os.ErrClosed + } + err := o.tun.transmitGather(frame) + if err == nil || err == windows.ERROR_BUFFER_OVERFLOW || err == os.ErrClosed { + return err + } + return E.Cause(err, "go: wintun send") +} + +func (o *goWindowsIO) transmitPrefix() int { + return 0 +} + +func (o *goWindowsIO) transmitChecksumOffload() bool { + return false +} + +func (o *goWindowsIO) mtu() int { + return o.stack.mtu +} + +func (o *goWindowsIO) supportsSockets() bool { + return true +} + +func (o *goWindowsIO) transmitSegmentOffload() bool { + return false +} + +func (o *goWindowsIO) armTransmitWritable(conn *GoConn) (*goBlockedWriters, error) { + return nil, nil +} + +func (o *goWindowsIO) takeTransmitWritable(writable []*goBlockedWriters) []*goBlockedWriters { + return writable +} + +func (o *goWindowsIO) wake() { + o.wakeAccess.RLock() + defer o.wakeAccess.RUnlock() + if o.closing.Load() { + return + } + _ = windows.PostQueuedCompletionStatus(o.iocp, 0, goCompletionKeyWake, nil) +} + +func (o *goWindowsIO) close() error { + o.wakeAccess.Lock() + closing := o.closing.Swap(true) + o.wakeAccess.Unlock() + if closing { + return nil + } + o.receiveSlots.release() + var err error + if o.waitPacket != nil { + err = E.Errors(o.waitPacket.Cancel(), o.waitPacket.Close()) + } else { + windows.SetEvent(o.bridgeClose) + <-o.bridgeDone + err = E.Errors(windows.CloseHandle(o.bridgeArm), windows.CloseHandle(o.bridgeClose)) + } + for entry := range o.entries { + entry.interest = 0 + entry.cancelled = true + if entry.armed { + err = E.Errors(err, o.afd.Cancel(&entry.ioStatusBlock)) + } else { + o.releaseEntry(entry) + } + } + err = E.Errors(err, o.afd.Close()) + err = E.Errors(err, o.drainClosingEntries(goShutdownBudget)) + if len(o.entries) == 0 { + return E.Errors(err, windows.CloseHandle(o.iocp)) + } + go o.finishClose() + return err +} + +func (o *goWindowsIO) drainClosingEntries(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for len(o.entries) > 0 { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil + } + waitMillis := uint32((remaining + time.Millisecond - 1) / time.Millisecond) + var removed uint32 + errno := afd.GetQueuedCompletionStatusEx(o.iocp, &o.completions[0], uint32(len(o.completions)), &removed, waitMillis, false) + if errno == windows.WAIT_TIMEOUT { + return nil + } + if errno != 0 { + return E.Cause(errno, "go: drain cancelled socket polls") + } + for index := range removed { + completion := &o.completions[index] + if completion.CompletionKey != 0 { + continue + } + entry := (*goAFDEntry)(unsafe.Pointer(completion.Overlapped)) + if _, pending := o.entries[entry]; pending { + entry.armed = false + o.releaseEntry(entry) + } + } + clear(o.completions[:removed]) + } + return nil +} + +func (o *goWindowsIO) finishClose() { + logged := false + for len(o.entries) > 0 { + err := o.drainClosingEntries(time.Second) + if err != nil { + if !logged { + o.stack.logger.Error(err) + logged = true + } + time.Sleep(time.Second) + } + } + err := windows.CloseHandle(o.iocp) + if err != nil { + o.stack.logger.Error(E.Cause(err, "go: close completion port")) + } +} + +func (o *goWindowsIO) writeDatagram(packet []byte, meta ForwardFrameMeta) error { + return o.writePacket(packet, meta) +} + +func (o *goWindowsIO) transmitBacklogBelowBatch(conn *GoConn) bool { + return true +} + +func (o *goWindowsIO) flush() { +} diff --git a/stack_go_listen.go b/stack_go_listen.go new file mode 100644 index 00000000..b9ef6c58 --- /dev/null +++ b/stack_go_listen.go @@ -0,0 +1,240 @@ +package tun + +import ( + "net" + "net/netip" + "os" + "sync" + "syscall" + "time" + + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/pipe" +) + +const goListenBacklog = 4096 + +type GoListener struct { + engine *goEngine + local netip.AddrPort + access sync.Mutex + pending []*GoConn + pendingHead int + backlog int + closed bool + openErr error + openSignal chan struct{} + closeSignal chan struct{} + closeDone chan struct{} + acceptSignal goSignal + deadline pipe.Deadline + openMessage goMessage + closeMessage goMessage +} + +func (s *Go) ListenTCP(local netip.AddrPort) (*GoListener, error) { + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + if !local.Addr().IsValid() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(local) + if err != nil { + return nil, err + } + listener := &GoListener{ + engine: engine, + local: local, + openSignal: make(chan struct{}), + closeSignal: make(chan struct{}), + closeDone: make(chan struct{}), + acceptSignal: make(goSignal, 1), + deadline: pipe.MakeDeadline(), + } + listener.openMessage = goMessage{kind: goMessageListenOpen, listener: listener} + listener.closeMessage = goMessage{kind: goMessageListenClose, listener: listener} + engine.postMessage(&listener.openMessage) + select { + case <-listener.openSignal: + case <-s.ctx.Done(): + listener.Close() + return nil, E.Cause(s.ctx.Err(), "go: listen") + } + listener.access.Lock() + err = listener.openErr + listener.access.Unlock() + if err != nil { + return nil, err + } + return listener, nil +} + +func (e *goEngine) lookupListener(local netip.AddrPort) *GoListener { + listener := e.tcpListeners[local] + if listener == nil && len(e.stack.engines) > 1 { + listener = e.stack.directory.lookupListener(local) + } + return listener +} + +func (e *goEngine) handleListenOpen(listener *GoListener) { + listener.access.Lock() + closed := listener.closed + listener.access.Unlock() + if closed { + close(listener.openSignal) + return + } + available := func(port uint16) bool { + return e.lookupListener(netip.AddrPortFrom(listener.local.Addr(), port)) == nil + } + port := listener.local.Port() + if port == 0 { + allocated := false + port, allocated = goAllocatePort(available) + if !allocated { + listener.access.Lock() + listener.openErr = E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted") + listener.access.Unlock() + close(listener.openSignal) + return + } + } else if !available(port) { + listener.access.Lock() + listener.openErr = E.Cause(syscall.EADDRINUSE, "go: listen ", listener.local) + listener.access.Unlock() + close(listener.openSignal) + return + } + listener.local = netip.AddrPortFrom(listener.local.Addr(), port) + e.stack.directory.insertListener(listener) + close(listener.openSignal) +} + +func (e *goEngine) handleListenClose(listener *GoListener) { + e.stack.directory.removeListener(listener) + listener.drain() + close(listener.closeDone) +} + +func (e *goEngine) closeAllListeners() { + for _, listener := range e.tcpListeners { + e.stack.directory.removeListener(listener) + listener.markClosed() + listener.drain() + } +} + +func (l *GoListener) reserve() bool { + l.access.Lock() + defer l.access.Unlock() + if l.closed || l.backlog >= goListenBacklog { + return false + } + l.backlog++ + return true +} + +func (l *GoListener) abandon() { + l.access.Lock() + l.backlog-- + l.access.Unlock() +} + +func (l *GoListener) deliver(conn *GoConn) { + l.access.Lock() + if l.closed { + l.backlog-- + l.access.Unlock() + conn.fail(net.ErrClosed) + return + } + l.pending = append(l.pending, conn) + l.access.Unlock() + l.acceptSignal.notify() +} + +func (l *GoListener) AcceptTCP() (*GoConn, error) { + for { + l.access.Lock() + if l.pendingHead < len(l.pending) { + conn := l.pending[l.pendingHead] + l.pending[l.pendingHead] = nil + l.pendingHead++ + if l.pendingHead == len(l.pending) { + l.pending = l.pending[:0] + l.pendingHead = 0 + } else if l.pendingHead >= len(l.pending)/2 { + remaining := copy(l.pending, l.pending[l.pendingHead:]) + clear(l.pending[remaining:]) + l.pending = l.pending[:remaining] + l.pendingHead = 0 + } + l.backlog-- + l.access.Unlock() + return conn, nil + } + closed := l.closed + l.access.Unlock() + if closed { + return nil, net.ErrClosed + } + select { + case <-l.acceptSignal: + case <-l.closeSignal: + case <-l.deadline.Wait(): + return nil, os.ErrDeadlineExceeded + } + } +} + +func (l *GoListener) Accept() (net.Conn, error) { + conn, err := l.AcceptTCP() + if err != nil { + return nil, err + } + return conn, nil +} + +func (l *GoListener) markClosed() bool { + l.access.Lock() + if l.closed { + l.access.Unlock() + return false + } + l.closed = true + l.access.Unlock() + close(l.closeSignal) + return true +} + +func (l *GoListener) drain() { + l.access.Lock() + pending := l.pending[l.pendingHead:] + l.backlog -= len(pending) + l.pending = nil + l.pendingHead = 0 + l.access.Unlock() + for _, conn := range pending { + conn.fail(net.ErrClosed) + } +} + +func (l *GoListener) Close() error { + if l.markClosed() { + l.engine.postMessage(&l.closeMessage) + } + select { + case <-l.closeDone: + case <-l.engine.exitSignal: + } + return nil +} + +func (l *GoListener) Addr() net.Addr { + return net.TCPAddrFromAddrPort(l.local) +} + +func (l *GoListener) SetDeadline(t time.Time) error { + l.deadline.Set(t) + return nil +} diff --git a/stack_go_memory.go b/stack_go_memory.go new file mode 100644 index 00000000..5f361d93 --- /dev/null +++ b/stack_go_memory.go @@ -0,0 +1,113 @@ +package tun + +import ( + "os" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" +) + +var goPageSize = os.Getpagesize() + +type goSlabStore interface { + acquire() *goSlab + release(slab *goSlab, discard bool) + purge() + close() +} + +type goHeapSlabStore struct{} + +func (goHeapSlabStore) acquire() *goSlab { + return (*goSlab)(buf.Get(goSlabSize)) +} + +func (goHeapSlabStore) release(slab *goSlab, discard bool) { + if discard { + return + } + common.Must(buf.Put(slab[:])) +} + +func (goHeapSlabStore) purge() { +} + +func (goHeapSlabStore) close() { +} + +type goReadSlots struct { + buffers []*buf.Buffer + region []byte + stride int + woken int +} + +func (s *goReadSlots) configure(count int, stride int) { + if s.buffers != nil && len(s.buffers) == count && s.stride == stride { + return + } + s.release() + s.buffers = make([]*buf.Buffer, count) + s.stride = stride +} + +func (s *goReadSlots) wake(count int) { + if !goArenaEnabled() { + return + } + if s.region == nil { + s.region = goRegionMap(len(s.buffers) * s.stride) + } + if count <= s.woken { + return + } + start := s.woken * s.stride &^ (goPageSize - 1) + end := min((count*s.stride+goPageSize-1)&^(goPageSize-1), len(s.region)) + goRegionReuse(s.region[start:end]) + s.woken = count +} + +func (s *goReadSlots) slot(index int) *buf.Buffer { + if goArenaEnabled() && index >= s.woken { + s.wake(index + 1) + } + buffer := s.buffers[index] + if buffer != nil { + return buffer + } + if goArenaEnabled() { + buffer = buf.With(s.region[index*s.stride : (index+1)*s.stride]) + } else { + buffer = buf.NewSize(s.stride) + } + s.buffers[index] = buffer + return buffer +} + +func (s *goReadSlots) sleep() { + if !goArenaEnabled() { + buf.ReleaseMulti(s.buffers) + clear(s.buffers) + return + } + if s.woken == 0 { + return + } + end := min((s.woken*s.stride+goPageSize-1)&^(goPageSize-1), len(s.region)) + goRegionPark(s.region[:end]) + s.woken = 0 +} + +func (s *goReadSlots) release() { + if goArenaEnabled() { + if s.region != nil { + goRegionUnmap(s.region) + s.region = nil + } + } else { + buf.ReleaseMulti(s.buffers) + } + s.buffers = nil + s.stride = 0 + s.woken = 0 +} diff --git a/stack_go_memory_arena.go b/stack_go_memory_arena.go new file mode 100644 index 00000000..50d0d101 --- /dev/null +++ b/stack_go_memory_arena.go @@ -0,0 +1,218 @@ +//go:build darwin || linux + +package tun + +import ( + "cmp" + "slices" + "sync" + "unsafe" + + E "github.com/sagernet/sing/common/exceptions" + + "golang.org/x/sys/unix" +) + +const ( + goSlabChunkSlabs = 32 + goSlabChunkSize = goSlabChunkSlabs * goSlabSize +) + +const ( + goSlabFresh uint8 = iota + goSlabLive + goSlabCached + goSlabReusable +) + +func newGoSlabStore(lowWater int) goSlabStore { + if !goArenaEnabled() { + return goHeapSlabStore{} + } + return &goArenaSlabStore{cached: make([]*goSlab, 0, lowWater), lowWater: lowWater} +} + +func goRegionMap(size int) []byte { + size = (size + goPageSize - 1) &^ (goPageSize - 1) + memory, err := unix.Mmap(-1, 0, size, unix.PROT_READ|unix.PROT_WRITE, goRegionMapFlags) + if err != nil { + panic(E.Cause(err, "go: map memory region")) + } + return memory +} + +func goRegionUnmap(memory []byte) { + unix.Munmap(memory) +} + +type goSlabChunk struct { + base uintptr + memory []byte + states [goSlabChunkSlabs]uint8 + live int + cached int + reusable int + fresh int +} + +func (c *goSlabChunk) slab(index int) *goSlab { + return (*goSlab)(unsafe.Pointer(&c.memory[index*goSlabSize])) +} + +func (c *goSlabChunk) span(index int) []byte { + return c.memory[index*goSlabSize : (index+1)*goSlabSize] +} + +func (c *goSlabChunk) idle() bool { + return c.live == 0 && c.cached == 0 +} + +func (c *goSlabChunk) take(state uint8) (*goSlab, bool) { + for index := range c.states { + if c.states[index] != state { + continue + } + switch state { + case goSlabReusable: + c.reusable-- + goRegionReuse(c.span(index)) + case goSlabFresh: + c.fresh-- + } + c.states[index] = goSlabLive + c.live++ + return c.slab(index), true + } + return nil, false +} + +type goArenaSlabStore struct { + access sync.Mutex + chunks []*goSlabChunk + cached []*goSlab + lowWater int + closed bool +} + +func (s *goArenaSlabStore) acquire() *goSlab { + s.access.Lock() + defer s.access.Unlock() + if count := len(s.cached); count > 0 { + slab := s.cached[count-1] + s.cached[count-1] = nil + s.cached = s.cached[:count-1] + chunk, index := s.locate(slab) + chunk.states[index] = goSlabLive + chunk.cached-- + chunk.live++ + return slab + } + for _, chunk := range s.chunks { + if chunk.reusable == 0 { + continue + } + slab, _ := chunk.take(goSlabReusable) + return slab + } + for _, chunk := range s.chunks { + if chunk.fresh == 0 { + continue + } + slab, _ := chunk.take(goSlabFresh) + return slab + } + memory := goRegionMap(goSlabChunkSize) + chunk := &goSlabChunk{base: uintptr(unsafe.Pointer(&memory[0])), memory: memory, fresh: goSlabChunkSlabs} + position, _ := slices.BinarySearchFunc(s.chunks, chunk.base, goCompareChunk) + s.chunks = slices.Insert(s.chunks, position, chunk) + slab, _ := chunk.take(goSlabFresh) + return slab +} + +func goCompareChunk(chunk *goSlabChunk, base uintptr) int { + return cmp.Compare(chunk.base, base) +} + +func (s *goArenaSlabStore) locate(slab *goSlab) (*goSlabChunk, int) { + address := uintptr(unsafe.Pointer(slab)) + position, found := slices.BinarySearchFunc(s.chunks, address, goCompareChunk) + if !found { + position-- + } + chunk := s.chunks[position] + return chunk, int((address - chunk.base) / goSlabSize) +} + +func (s *goArenaSlabStore) release(slab *goSlab, discard bool) { + s.access.Lock() + defer s.access.Unlock() + chunk, index := s.locate(slab) + chunk.live-- + if discard || s.closed { + s.park(chunk, index) + s.trim(chunk) + return + } + chunk.states[index] = goSlabCached + chunk.cached++ + s.cached = append(s.cached, slab) + if len(s.cached) <= s.lowWater { + return + } + oldest := s.cached[0] + remaining := copy(s.cached, s.cached[1:]) + s.cached[remaining] = nil + s.cached = s.cached[:remaining] + oldestChunk, oldestIndex := s.locate(oldest) + oldestChunk.cached-- + s.park(oldestChunk, oldestIndex) + s.trim(oldestChunk) +} + +func (s *goArenaSlabStore) park(chunk *goSlabChunk, index int) { + goRegionPark(chunk.span(index)) + chunk.states[index] = goSlabReusable + chunk.reusable++ +} + +func (s *goArenaSlabStore) trim(chunk *goSlabChunk) { + if !chunk.idle() { + return + } + if !s.closed { + idle := 0 + for _, candidate := range s.chunks { + if candidate.idle() { + idle++ + } + } + if idle <= 1 { + return + } + } + position, _ := slices.BinarySearchFunc(s.chunks, chunk.base, goCompareChunk) + s.chunks = slices.Delete(s.chunks, position, position+1) + goRegionUnmap(chunk.memory) +} + +func (s *goArenaSlabStore) purge() { + s.access.Lock() + defer s.access.Unlock() + for _, slab := range s.cached { + chunk, index := s.locate(slab) + chunk.cached-- + s.park(chunk, index) + } + clear(s.cached) + s.cached = s.cached[:0] + for index := len(s.chunks) - 1; index >= 0; index-- { + s.trim(s.chunks[index]) + } +} + +func (s *goArenaSlabStore) close() { + s.access.Lock() + s.closed = true + s.access.Unlock() + s.purge() +} diff --git a/stack_go_memory_darwin.go b/stack_go_memory_darwin.go new file mode 100644 index 00000000..95cf38e1 --- /dev/null +++ b/stack_go_memory_darwin.go @@ -0,0 +1,17 @@ +package tun + +import "golang.org/x/sys/unix" + +const goRegionMapFlags = unix.MAP_ANON | unix.MAP_PRIVATE + +func goArenaEnabled() bool { + return true +} + +func goRegionPark(memory []byte) { + unix.Madvise(memory, unix.MADV_FREE_REUSABLE) +} + +func goRegionReuse(memory []byte) { + unix.Madvise(memory, unix.MADV_FREE_REUSE) +} diff --git a/stack_go_memory_linux.go b/stack_go_memory_linux.go new file mode 100644 index 00000000..8118386b --- /dev/null +++ b/stack_go_memory_linux.go @@ -0,0 +1,26 @@ +package tun + +import ( + "os" + "sync" + + "github.com/sagernet/sing/common/memory" + + "golang.org/x/sys/unix" +) + +const ( + goLowMemoryLimit = 512 << 20 + goRegionMapFlags = unix.MAP_ANON | unix.MAP_PRIVATE | unix.MAP_NORESERVE +) + +var goArenaEnabled = sync.OnceValue(func() bool { + return goSlabSize%os.Getpagesize() == 0 && memory.LimitAvailable() && memory.Limit() < goLowMemoryLimit +}) + +func goRegionPark(memory []byte) { + unix.Madvise(memory, unix.MADV_DONTNEED) +} + +func goRegionReuse(memory []byte) { +} diff --git a/stack_go_memory_other.go b/stack_go_memory_other.go new file mode 100644 index 00000000..793446dc --- /dev/null +++ b/stack_go_memory_other.go @@ -0,0 +1,24 @@ +//go:build !darwin && !linux + +package tun + +func goArenaEnabled() bool { + return false +} + +func newGoSlabStore(lowWater int) goSlabStore { + return goHeapSlabStore{} +} + +func goRegionMap(size int) []byte { + return nil +} + +func goRegionUnmap(memory []byte) { +} + +func goRegionPark(memory []byte) { +} + +func goRegionReuse(memory []byte) { +} diff --git a/stack_go_profile_default.go b/stack_go_profile_default.go new file mode 100644 index 00000000..3e14643a --- /dev/null +++ b/stack_go_profile_default.go @@ -0,0 +1,19 @@ +//go:build !ios + +package tun + +import "time" + +const ( + goReceiveCapacityBase = 64 << 10 + goReceiveCapacityMax = 4 << 20 + goTransmitCapacityMax = 2 << 20 + goEngineBurstBytes = 1 << 20 + goFlowCapacity = 16384 + goReceiveBatchMin = 8 + goReadBatch = 64 + goReadBufferIdle = time.Second + goDescriptorRingCapacity = 2048 + goReassemblyEntries = 8 + goSlabPoolLowWater = 128 +) diff --git a/stack_go_profile_ios.go b/stack_go_profile_ios.go new file mode 100644 index 00000000..f083353c --- /dev/null +++ b/stack_go_profile_ios.go @@ -0,0 +1,17 @@ +package tun + +import "time" + +const ( + goReceiveCapacityBase = 32 << 10 + goReceiveCapacityMax = 256 << 10 + goTransmitCapacityMax = 512 << 10 + goEngineBurstBytes = 256 << 10 + goFlowCapacity = 1024 + goReceiveBatchMin = 4 + goReadBatch = 32 + goReadBufferIdle = 250 * time.Millisecond + goDescriptorRingCapacity = 512 + goReassemblyEntries = 4 + goSlabPoolLowWater = 8 +) diff --git a/stack_go_ring.go b/stack_go_ring.go new file mode 100644 index 00000000..5d4ac539 --- /dev/null +++ b/stack_go_ring.go @@ -0,0 +1,717 @@ +package tun + +import ( + "slices" + "sort" + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common/buf" +) + +const ( + goSlabSize = 32 << 10 + goSlabShareMin = 4 * goSlabSize +) + +type goSlab [goSlabSize]byte + +type goSlabPool struct { + store goSlabStore + inUse atomic.Int32 + holders atomic.Int32 + pressure func() MemoryPressure +} + +func newGoSlabPool(pressure func() MemoryPressure, lowWater int) *goSlabPool { + return &goSlabPool{store: newGoSlabStore(lowWater), pressure: pressure} +} + +func (p *goSlabPool) acquire() *goSlab { + slab := p.store.acquire() + p.inUse.Add(1) + return slab +} + +func (p *goSlabPool) release(slab *goSlab) { + p.store.release(slab, p.pressureLevel() != MemoryPressureNone) + p.inUse.Add(-1) +} + +func (p *goSlabPool) close() { + p.store.close() +} + +func (p *goSlabPool) purge() { + p.store.purge() +} + +func (p *goSlabPool) pressureLevel() MemoryPressure { + if p.pressure == nil { + return MemoryPressureNone + } + return p.pressure() +} + +func (p *goSlabPool) shareBytes() uint64 { + var share int + switch p.pressureLevel() { + case MemoryPressureNone: + return 0 + case MemoryPressureWarning: + share = int(p.inUse.Load()) / max(int(p.holders.Load()), 1) + } + return uint64(max(share, goSlabShareMin/goSlabSize)) * goSlabSize +} + +type goSlabHolder struct { + held atomic.Int32 +} + +func (h *goSlabHolder) heldBytes() uint64 { + return uint64(h.held.Load()) * goSlabSize +} + +const goSlabChainMinSlots = 4 + +type goSlabChain struct { + slots atomic.Pointer[[]*goSlab] + pool *goSlabPool + holder *goSlabHolder + held atomic.Int32 + maxSlots uint32 + releasedIndex uint64 +} + +func (c *goSlabChain) acquire() *goSlab { + slab := c.pool.acquire() + c.held.Add(1) + if c.holder.held.Add(1) == 1 { + c.pool.holders.Add(1) + } + return slab +} + +func (c *goSlabChain) release(slab *goSlab) { + c.held.Add(-1) + if c.holder.held.Add(-1) == 0 { + c.pool.holders.Add(-1) + } + c.pool.release(slab) +} + +func (c *goSlabChain) heldBytes() uint64 { + return uint64(c.held.Load()) * goSlabSize +} + +func (c *goSlabChain) loadSlots() []*goSlab { + slots := c.slots.Load() + if slots == nil { + return nil + } + return *slots +} + +func (c *goSlabChain) ensure(last uint64) []*goSlab { + slots := c.loadSlots() + required := last - c.releasedIndex + 1 + if required <= uint64(len(slots)) { + return slots + } + grown := make([]*goSlab, max(required, min(max(2*uint64(len(slots)), goSlabChainMinSlots), uint64(c.maxSlots)))) + for index := c.releasedIndex; index < c.releasedIndex+uint64(len(slots)); index++ { + grown[index%uint64(len(grown))] = slots[index%uint64(len(slots))] + } + c.slots.Store(&grown) + return grown +} + +func goSlotOf(slots []*goSlab, offset uint64) int { + return int((offset / goSlabSize) % uint64(len(slots))) +} + +func (c *goSlabChain) reserve(offset uint64, length int) { + first := offset / goSlabSize + last := (offset + uint64(length) - 1) / goSlabSize + slots := c.ensure(last) + for index := first; index <= last; index++ { + slot := int(index % uint64(len(slots))) + if slots[slot] != nil { + continue + } + slots[slot] = c.acquire() + } +} + +func (c *goSlabChain) writeAt(offset uint64, data []byte) { + slots := c.loadSlots() + for len(data) > 0 { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := copy(slab[start:], data) + data = data[n:] + offset += uint64(n) + } +} + +func (c *goSlabChain) readAt(offset uint64, target []byte) { + slots := c.loadSlots() + copied := 0 + for copied < len(target) { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := copy(target[copied:], slab[start:]) + copied += n + offset += uint64(n) + } +} + +func (c *goSlabChain) appendRuns(segments [][]byte, offset uint64, length int) [][]byte { + slots := c.loadSlots() + for length > 0 { + slab := slots[goSlotOf(slots, offset)] + start := int(offset % goSlabSize) + n := min(goSlabSize-start, length) + segments = append(segments, slab[start:start+n]) + offset += uint64(n) + length -= n + } + return segments +} + +func (c *goSlabChain) releaseRange(first uint64, last uint64) { + slots := c.loadSlots() + if len(slots) == 0 { + return + } + for index := first; index <= last; index++ { + slot := int(index % uint64(len(slots))) + slab := slots[slot] + if slab == nil { + continue + } + slots[slot] = nil + c.release(slab) + } +} + +func (c *goSlabChain) releaseBelow(offset uint64) { + target := offset / goSlabSize + if target <= c.releasedIndex { + return + } + slots := c.loadSlots() + end := min(target, c.releasedIndex+uint64(len(slots))) + if end > c.releasedIndex { + c.releaseRange(c.releasedIndex, end-1) + } + c.releasedIndex = target +} + +func (c *goSlabChain) releaseDrained(offset uint64) { + c.releaseBelow(offset) + c.releaseRange(offset/goSlabSize, offset/goSlabSize) +} + +func (c *goSlabChain) releaseAll() { + slots := c.loadSlots() + for slot, slab := range slots { + if slab == nil { + continue + } + slots[slot] = nil + c.release(slab) + } +} + +const goTransmitRunCapacity = 32 + +type goTransmitRun struct { + start uint64 + end uint64 + buffer *buf.Buffer + data []byte +} + +type goTransmitStore struct { + access sync.Mutex + chain goSlabChain + runs []goTransmitRun + reserveTail uint64 +} + +func (s *goTransmitStore) reserve(offset uint64, length int) { + s.access.Lock() + defer s.access.Unlock() + s.chain.reserve(offset, length) + s.reserveTail = offset + uint64(length) +} + +func (s *goTransmitStore) shrinkReserve(tail uint64) { + s.access.Lock() + defer s.access.Unlock() + if tail >= s.reserveTail { + return + } + first := (tail + goSlabSize - 1) / goSlabSize + last := (s.reserveTail - 1) / goSlabSize + if first <= last { + s.chain.releaseRange(first, last) + } + s.reserveTail = tail +} + +func (s *goTransmitStore) writeAt(offset uint64, data []byte) { + for len(data) > 0 { + s.access.Lock() + slots := s.chain.loadSlots() + slab := slots[goSlotOf(slots, offset)] + s.access.Unlock() + start := int(offset % goSlabSize) + n := copy(slab[start:], data) + data = data[n:] + offset += uint64(n) + } +} + +func (s *goTransmitStore) adopt(offset uint64, buffer *buf.Buffer) bool { + s.access.Lock() + defer s.access.Unlock() + if len(s.runs) == goTransmitRunCapacity { + return false + } + data := buffer.Bytes() + s.runs = append(s.runs, goTransmitRun{ + start: offset, + end: offset + uint64(len(data)), + buffer: buffer, + data: data, + }) + return true +} + +func (s *goTransmitStore) appendRuns(segments [][]byte, offset uint64, length int) [][]byte { + s.access.Lock() + defer s.access.Unlock() + end := offset + uint64(length) + for index := range s.runs { + run := &s.runs[index] + if run.end <= offset { + continue + } + if run.start >= end { + break + } + if run.start > offset { + segments = s.chain.appendRuns(segments, offset, int(run.start-offset)) + offset = run.start + } + stop := min(run.end, end) + segments = append(segments, run.data[offset-run.start:stop-run.start]) + offset = stop + if offset == end { + return segments + } + } + if offset < end { + segments = s.chain.appendRuns(segments, offset, int(end-offset)) + } + return segments +} + +func (s *goTransmitStore) leadingRun(offset uint64, length int) (*buf.Buffer, []byte, bool) { + s.access.Lock() + defer s.access.Unlock() + for index := range s.runs { + run := &s.runs[index] + if run.end <= offset { + continue + } + if run.start != offset || run.end < offset+uint64(length) { + return nil, nil, false + } + return run.buffer, run.data[:length], true + } + return nil, nil, false +} + +func (s *goTransmitStore) releaseBelow(offset uint64) { + s.access.Lock() + defer s.access.Unlock() + s.chain.releaseBelow(offset) + released := 0 + for released < len(s.runs) && s.runs[released].end <= offset { + s.runs[released].buffer.Release() + released++ + } + if released > 0 { + remaining := copy(s.runs, s.runs[released:]) + clear(s.runs[remaining:]) + s.runs = s.runs[:remaining] + } +} + +func (s *goTransmitStore) releaseDrained(offset uint64) { + s.access.Lock() + defer s.access.Unlock() + if len(s.runs) > 0 || s.reserveTail > offset { + return + } + s.chain.releaseDrained(offset) +} + +func (s *goTransmitStore) releaseAll() { + s.access.Lock() + defer s.access.Unlock() + s.chain.releaseAll() + for index := range s.runs { + s.runs[index].buffer.Release() + } + clear(s.runs) + s.runs = s.runs[:0] +} + +const ( + goDescriptorRetransmitted uint16 = 1 << iota + goDescriptorNoSample + goDescriptorSacked + goDescriptorDropped + goDescriptorAmend + goDescriptorRetransmitOut + goDescriptorRated + goDescriptorAppLimited + goDescriptorTxStart + goDescriptorLost +) + +type goSentDescriptor struct { + endOffset uint64 + sentAt int32 + deliveredStamp int32 + firstSentStamp int32 + delivered uint32 + flags uint16 + _ [2]byte +} + +const ( + goDescriptorBlockCapacity = 64 + goScoreboardInlineCapacity = 16 +) + +type goDescriptorBlock [goDescriptorBlockCapacity]goSentDescriptor + +// The producer publishes entries through tail; the engine consumes them through +// head. Blocks are allocated on demand and returned after their final entry is +// consumed. The directory stays fixed so neither side has to resize shared state. +type goDescriptorRing struct { + blocks [goDescriptorRingCapacity / goDescriptorBlockCapacity]atomic.Pointer[goDescriptorBlock] + pool *goDescriptorPool + head atomic.Uint32 + tail atomic.Uint32 + overflowed atomic.Bool + overflowAccess sync.Mutex + overflow []goSentDescriptor +} + +type goDescriptorPool struct { + blocks sync.Pool +} + +func (p *goDescriptorPool) acquire() *goDescriptorBlock { + block, _ := p.blocks.Get().(*goDescriptorBlock) + if block == nil { + block = new(goDescriptorBlock) + } + return block +} + +func (p *goDescriptorPool) release(block *goDescriptorBlock) { + p.blocks.Put(block) +} + +// release requires exclusive access to both sides of the ring. For a live +// connection the engine holds transmitAccess and checks that all sent data is ACKed. +func (r *goDescriptorRing) release() { + for index := range r.blocks { + if block := r.blocks[index].Swap(nil); block != nil { + r.pool.release(block) + } + } + r.head.Store(0) + r.tail.Store(0) + r.overflow = nil + r.overflowed.Store(false) +} + +func (r *goDescriptorRing) push(entry goSentDescriptor) { + if !r.overflowed.Load() && r.store(entry) { + return + } + r.overflowAccess.Lock() + defer r.overflowAccess.Unlock() + if !r.overflowed.Load() && r.store(entry) { + return + } + if entry.flags&goDescriptorAmend != 0 && len(r.overflow) > 0 { + last := &r.overflow[len(r.overflow)-1] + if last.flags&goDescriptorAmend != 0 && last.endOffset == entry.endOffset && last.sentAt == entry.sentAt { + last.flags |= entry.flags + return + } + } + r.overflow = append(r.overflow, entry) + r.overflowed.Store(true) +} + +func (r *goDescriptorRing) store(entry goSentDescriptor) bool { + tail := r.tail.Load() + if tail-r.head.Load() >= goDescriptorRingCapacity { + return false + } + slot := &r.blocks[tail/goDescriptorBlockCapacity%uint32(len(r.blocks))] + block := slot.Load() + // A wrapped producer must not reuse the consumed prefix of a block that + // the consumer still owns; the consumer releases whole blocks. + if tail%goDescriptorBlockCapacity == 0 && block != nil { + return false + } + if block == nil { + block = r.pool.acquire() + slot.Store(block) + } + block[tail%goDescriptorBlockCapacity] = entry + r.tail.Store(tail + 1) + return true +} + +func (r *goDescriptorRing) take(index uint32) goSentDescriptor { + slot := &r.blocks[index/goDescriptorBlockCapacity%uint32(len(r.blocks))] + block := slot.Load() + entry := block[index%goDescriptorBlockCapacity] + if (index+1)%goDescriptorBlockCapacity == 0 { + slot.Store(nil) + r.pool.release(block) + } + return entry +} + +type goScoreboard struct { + entries []goSentDescriptor + inline [goScoreboardInlineCapacity]goSentDescriptor +} + +func (s *goScoreboard) append(entry goSentDescriptor) { + if s.entries == nil { + s.entries = s.inline[:0] + } + s.entries = append(s.entries, entry) +} + +func (s *goScoreboard) drain(ring *goDescriptorRing, unacked uint64) { + s.drainRing(ring, unacked) + if !ring.overflowed.Load() { + return + } + ring.overflowAccess.Lock() + defer ring.overflowAccess.Unlock() + s.drainRing(ring, unacked) + for _, entry := range ring.overflow { + s.drainEntry(entry, unacked) + } + ring.overflow = ring.overflow[:0] + ring.overflowed.Store(false) +} + +func (s *goScoreboard) drainRing(ring *goDescriptorRing, unacked uint64) { + head := ring.head.Load() + tail := ring.tail.Load() + for head != tail { + entry := ring.take(head) + head++ + s.drainEntry(entry, unacked) + } + ring.head.Store(head) +} + +func (s *goScoreboard) drainEntry(entry goSentDescriptor, unacked uint64) { + if entry.flags&goDescriptorAmend != 0 { + s.amend(entry.endOffset-uint64(entry.sentAt), entry.endOffset, entry.flags&^goDescriptorAmend) + } else if entry.endOffset > unacked { + s.append(entry) + } +} + +func (s *goScoreboard) amend(start uint64, end uint64, flags uint16) { + first := sort.Search(len(s.entries), func(index int) bool { return s.entries[index].endOffset > start }) + for index := first; index < len(s.entries) && s.entries[index].endOffset <= end; index++ { + entry := &s.entries[index] + entry.flags |= flags + } +} + +func (s *goScoreboard) advance(acked uint64) { + dropped := 0 + for dropped < len(s.entries) && s.entries[dropped].endOffset <= acked { + dropped++ + } + if dropped == 0 { + return + } + remaining := copy(s.entries, s.entries[dropped:]) + s.entries = s.entries[:remaining] +} + +func (s *goScoreboard) startOf(index int, unacked uint64) uint64 { + if index == 0 { + return unacked + } + return s.entries[index-1].endOffset +} + +func (s *goScoreboard) split(index int, offset uint64) { + entry := s.entries[index] + if entry.endOffset == offset { + return + } + s.append(goSentDescriptor{}) + copy(s.entries[index+1:], s.entries[index:len(s.entries)-1]) + s.entries[index].endOffset = offset +} + +func (s *goScoreboard) trim() { + if len(s.entries) == 0 && cap(s.entries) > goScoreboardInlineCapacity { + s.entries = nil + } +} + +const goMaxSackBlocks = 4 + +const goMaxSackBlocksSent = 3 + +type goSackBlock struct { + start uint64 + end uint64 +} + +const goMaxOOORanges = 32 + +type goRange struct { + start uint64 + end uint64 + fin bool +} + +type goRangeSet struct { + ranges [goMaxOOORanges]goRange + count int + recent [3]uint64 +} + +func (s *goRangeSet) insert(start uint64, end uint64, fin bool) bool { + index := 0 + for index < s.count && s.ranges[index].end < start { + index++ + } + if index == s.count || s.ranges[index].start > end { + if s.count == goMaxOOORanges { + return false + } + copy(s.ranges[index+1:s.count+1], s.ranges[index:s.count]) + s.ranges[index] = goRange{start: start, end: end, fin: fin} + s.count++ + s.touch(start) + return true + } + mergedStart := min(start, s.ranges[index].start) + mergedEnd := end + mergedFin := fin + mergeEnd := index + for mergeEnd < s.count && s.ranges[mergeEnd].start <= end { + mergedEnd = max(mergedEnd, s.ranges[mergeEnd].end) + mergedFin = mergedFin || s.ranges[mergeEnd].fin + mergeEnd++ + } + s.ranges[index] = goRange{start: mergedStart, end: mergedEnd, fin: mergedFin} + copy(s.ranges[index+1:], s.ranges[mergeEnd:s.count]) + s.count -= mergeEnd - index - 1 + s.touch(mergedStart) + return true +} + +func (s *goRangeSet) touch(start uint64) { + if s.recent[0] == start { + return + } + s.recent[2] = s.recent[1] + s.recent[1] = s.recent[0] + s.recent[0] = start +} + +func (s *goRangeSet) first() (goRange, bool) { + if s.count == 0 { + return goRange{}, false + } + return s.ranges[0], true +} + +func (s *goRangeSet) removeBelow(offset uint64) { + kept := 0 + for index := range s.count { + if s.ranges[index].end < offset || s.ranges[index].end == offset && !s.ranges[index].fin { + continue + } + s.ranges[kept] = s.ranges[index] + if s.ranges[kept].start < offset { + s.ranges[kept].start = offset + } + kept++ + } + s.count = kept +} + +func (s *goRangeSet) overlap(start uint64, end uint64) (uint64, uint64, bool) { + for index := range s.count { + candidate := s.ranges[index] + if candidate.end <= start || candidate.start >= end { + continue + } + return max(start, candidate.start), min(end, candidate.end), true + } + return 0, 0, false +} + +func (s *goRangeSet) bytes() uint64 { + var total uint64 + for index := range s.count { + total += s.ranges[index].end - s.ranges[index].start + } + return total +} + +func (s *goRangeSet) blocks(out *[goMaxSackBlocks]goSackBlock, base int) int { + emitted := base + for _, start := range s.recent { + if emitted == goMaxSackBlocksSent { + break + } + if start == 0 || slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { + continue + } + found := slices.IndexFunc(s.ranges[:s.count], func(candidate goRange) bool { return candidate.start == start && candidate.end > start }) + if found < 0 { + continue + } + out[emitted] = goSackBlock{start: s.ranges[found].start, end: s.ranges[found].end} + emitted++ + } + for index := 0; index < s.count && emitted < goMaxSackBlocksSent; index++ { + start := s.ranges[index].start + if start == s.ranges[index].end || slices.ContainsFunc(out[base:emitted], func(block goSackBlock) bool { return block.start == start }) { + continue + } + out[emitted] = goSackBlock{start: s.ranges[index].start, end: s.ranges[index].end} + emitted++ + } + return emitted +} diff --git a/stack_go_socket_darwin.go b/stack_go_socket_darwin.go new file mode 100644 index 00000000..2cd6ee14 --- /dev/null +++ b/stack_go_socket_darwin.go @@ -0,0 +1,170 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "syscall" + "unsafe" + + rawfile "github.com/sagernet/sing-tun/internal/rawfile_darwin" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +func (s *goSocket) writeVector(iovecs []unix.Iovec) (int, syscall.Errno) { + var message unix.Msghdr + message.Iov = &iovecs[0] + message.SetIovlen(len(iovecs)) + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_SENDMSG, uintptr(s.fd), uintptr(unsafe.Pointer(&message)), goSocketSendFlags) + return int(n), errno +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + var base *byte + if len(data) > 0 { + base = &data[0] + } + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_SENDTO, uintptr(s.fd), uintptr(unsafe.Pointer(base)), uintptr(len(data)), goSocketSendFlags, 0, 0) + return int(n), errno +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + sockaddr unsafe.Pointer + sockaddrLength uintptr + inet4 unix.RawSockaddrInet4 + inet6 unix.RawSockaddrInet6 + ) + address := destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return unix.EAFNOSUPPORT + } + inet4.Len = unix.SizeofSockaddrInet4 + inet4.Family = unix.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], destination.Port()) + sockaddr = unsafe.Pointer(&inet4) + sockaddrLength = unix.SizeofSockaddrInet4 + } else { + inet6.Len = unix.SizeofSockaddrInet6 + inet6.Family = unix.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], destination.Port()) + sockaddr = unsafe.Pointer(&inet6) + sockaddrLength = unix.SizeofSockaddrInet6 + } + var base *byte + if len(data) > 0 { + base = &data[0] + } + //nolint:staticcheck + _, _, errno := unix.RawSyscall6(unix.SYS_SENDTO, uintptr(s.fd), uintptr(unsafe.Pointer(base)), uintptr(len(data)), goSocketSendFlags, uintptr(sockaddr), sockaddrLength) + return errno +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + var storage unix.RawSockaddrInet6 + length := uint32(unsafe.Sizeof(storage)) + //nolint:staticcheck + n, _, errno := unix.RawSyscall6(unix.SYS_RECVFROM, uintptr(s.fd), uintptr(unsafe.Pointer(&buffer[0])), uintptr(len(buffer)), unix.MSG_DONTWAIT, uintptr(unsafe.Pointer(&storage)), uintptr(unsafe.Pointer(&length))) + if errno != 0 { + return 0, netip.AddrPort{}, errno + } + return int(n), M.SocksaddrFromNetIP(M.AddrPortFromRawSockaddr((*unix.RawSockaddr)(unsafe.Pointer(&storage)))).Unwrap().AddrPort(), 0 +} + +type goPacketBatchIO struct { + headers [goPacketBatchSize]rawfile.MsgHdrX + vectors [goPacketBatchSize]unix.Iovec + names [goPacketBatchSize]unix.RawSockaddrAny +} + +func (b *goPacketBatchIO) reset() { + clear(b.headers[:]) + clear(b.vectors[:]) +} + +func (s *goSocket) enablePacketOffload() {} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + b.vectors[index] = rawfile.IovecFromBytes(packets[index].data) + message := &b.headers[index] + *message = rawfile.MsgHdrX{} + message.Msg.Iov = &b.vectors[index] + message.Msg.Iovlen = 1 + if !connected { + b.names[index] = unix.RawSockaddrAny{} + message.Msg.Name = (*byte)(unsafe.Pointer(&b.names[index])) + message.Msg.Namelen = unix.SizeofSockaddrAny + } + } + count, errno := receiveMessageBatch(socket.fd, b.headers[:len(packets)]) + if errno != 0 { + return 0, errno + } + for index := range count { + message := &b.headers[index] + packet := &packets[index] + packet.truncated = message.Msg.Flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 || int(message.DataLen) > len(packet.data) + packet.data = packet.data[:min(int(message.DataLen), len(packet.data))] + if !connected { + packet.destination = M.SocksaddrFromRawSockaddrAny(&b.names[index]).AddrPort() + } + } + return count, 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + sent := 0 + limit := len(packets) + for sent < len(packets) { + if !connected || len(packets[sent].data) == 0 { + var errno syscall.Errno + if connected { + _, errno = socket.write(packets[sent].data) + } else { + errno = socket.sendTo(packets[sent].data, packets[sent].destination, family) + } + if errno == unix.EINTR { + continue + } + if errno != 0 { + return sent, errno + } + sent++ + continue + } + count := 0 + for index := sent; index < len(packets) && count < limit; index++ { + if len(packets[index].data) == 0 { + break + } + b.vectors[count] = rawfile.IovecFromBytes(packets[index].data) + b.headers[count] = rawfile.MsgHdrX{} + b.headers[count].Msg.Iov = &b.vectors[count] + b.headers[count].Msg.Iovlen = 1 + count++ + } + written, errno := rawfile.NonBlockingSendMMsg(socket.fd, b.headers[:count]) + if errno == unix.EINTR { + continue + } + if errno == unix.EMSGSIZE && count > 1 { + limit = (count + 1) / 2 + continue + } + if errno != 0 { + return sent, errno + } + if written == 0 { + return sent, unix.EIO + } + sent += written + } + return sent, 0 +} diff --git a/stack_go_socket_linux.go b/stack_go_socket_linux.go new file mode 100644 index 00000000..0deaca11 --- /dev/null +++ b/stack_go_socket_linux.go @@ -0,0 +1,256 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "syscall" + "unsafe" + + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +func (s *goSocket) writeVector(iovecs []unix.Iovec) (int, syscall.Errno) { + var message unix.Msghdr + message.Iov = &iovecs[0] + message.SetIovlen(len(iovecs)) + return goSendmsg(s.fd, &message, goSocketSendFlags) +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + var vector unix.Iovec + if len(data) > 0 { + vector.Base = &data[0] + } + vector.SetLen(len(data)) + var message unix.Msghdr + message.Iov = &vector + message.SetIovlen(1) + return goSendmsg(s.fd, &message, goSocketSendFlags) +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + inet4 unix.SockaddrInet4 + inet6 unix.SockaddrInet6 + to unix.Sockaddr + ) + address := destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return unix.EAFNOSUPPORT + } + inet4.Addr = address.As4() + inet4.Port = int(destination.Port()) + to = &inet4 + } else { + inet6.Addr = address.As16() + inet6.Port = int(destination.Port()) + to = &inet6 + } + err := unix.Sendto(s.fd, data, goSocketSendFlags, to) + if err != nil { + return err.(syscall.Errno) + } + return 0 +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + n, from, err := unix.Recvfrom(s.fd, buffer, unix.MSG_DONTWAIT) + if err != nil { + return 0, netip.AddrPort{}, err.(syscall.Errno) + } + return n, M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(from)).Unwrap().AddrPort(), 0 +} + +func goSendmsg(fd int, message *unix.Msghdr, flags int) (int, syscall.Errno) { + n, _, errno := unix.Syscall(unix.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(message)), uintptr(flags)) + if errno != 0 { + return 0, errno + } + return int(n), 0 +} + +type goPacketMessageHeader struct { + header unix.Msghdr + length uint32 +} + +type goPacketBatchIO struct { + headers [goPacketBatchSize]goPacketMessageHeader + vectors [goPacketBatchSize]unix.Iovec + names [goPacketBatchSize]unix.RawSockaddrAny + controls [goPacketBatchSize][64]byte + counts [goPacketBatchSize]int +} + +func (b *goPacketBatchIO) reset() { + clear(b.headers[:]) + clear(b.vectors[:]) +} + +func (s *goSocket) enablePacketOffload() { + err := unix.SetsockoptInt(s.fd, unix.IPPROTO_UDP, unix.UDP_GRO, 1) + s.packetGRO = err == nil +} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + vector := &b.vectors[index] + *vector = unix.Iovec{Base: &packets[index].data[0]} + vector.SetLen(len(packets[index].data)) + message := &b.headers[index] + *message = goPacketMessageHeader{} + message.header.Iov = vector + message.header.SetIovlen(1) + if !connected { + b.names[index] = unix.RawSockaddrAny{} + message.header.Name = (*byte)(unsafe.Pointer(&b.names[index])) + message.header.Namelen = unix.SizeofSockaddrAny + } + if socket.packetGRO { + message.header.Control = &b.controls[index][0] + message.header.SetControllen(len(b.controls[index])) + } + } + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_RECVMMSG, uintptr(socket.fd), uintptr(unsafe.Pointer(&b.headers[0])), uintptr(len(packets)), unix.MSG_DONTWAIT, 0, 0) + if errno != 0 { + return 0, errno + } + for index := range int(count) { + message := &b.headers[index] + packet := &packets[index] + packet.truncated = message.header.Flags&(unix.MSG_TRUNC|unix.MSG_CTRUNC) != 0 || int(message.length) > len(packet.data) + packet.data = packet.data[:min(int(message.length), len(packet.data))] + if !connected { + packet.destination = M.SocksaddrFromRawSockaddrAny(&b.names[index]).AddrPort() + } + if socket.packetGRO { + control := b.controls[index][:message.header.Controllen] + for len(control) >= unix.CmsgLen(0) { + controlHeader := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + length := int(controlHeader.Len) + if length < unix.CmsgLen(0) || length > len(control) { + packet.truncated = true + break + } + if controlHeader.Level == unix.IPPROTO_UDP && controlHeader.Type == unix.UDP_GRO { + if length < unix.CmsgLen(4) { + packet.truncated = true + } else { + packet.segmentSize = int(binary.NativeEndian.Uint32(control[unix.CmsgLen(0):])) + packet.truncated = packet.truncated || packet.segmentSize <= 0 || packet.segmentSize > 65535 + } + } + control = control[min(unix.CmsgSpace(length-unix.CmsgLen(0)), len(control)):] + } + } + } + return int(count), 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + coalesce := !socket.packetGSODisabled + sent := 0 + for sent < len(packets) { + messageCount := 0 + hasSegments := false + for index := sent; index < len(packets); { + packet := &packets[index] + segmentSize := len(packet.data) + end := index + 1 + length := segmentSize + if coalesce && segmentSize > 0 { + for end < len(packets) && end-index < 64 { + next := &packets[end] + if !connected && next.destination != packet.destination || len(next.data) == 0 || len(next.data) > segmentSize || length+len(next.data) > 65507 { + break + } + length += len(next.data) + end++ + if len(next.data) < segmentSize { + break + } + } + } + message := &b.headers[messageCount] + *message = goPacketMessageHeader{} + for vectorIndex := index; vectorIndex < end; vectorIndex++ { + data := packets[vectorIndex].data + vector := &b.vectors[vectorIndex-sent] + *vector = unix.Iovec{} + if len(data) > 0 { + vector.Base = &data[0] + } + vector.SetLen(len(data)) + } + message.header.Iov = &b.vectors[index-sent] + message.header.SetIovlen(end - index) + if !connected { + name := &b.names[messageCount] + *name = unix.RawSockaddrAny{} + address := packet.destination.Addr().Unmap() + if family == unix.AF_INET { + if !address.Is4() { + return sent, unix.EAFNOSUPPORT + } + inet4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(name)) + inet4.Family = unix.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], packet.destination.Port()) + message.header.Namelen = unix.SizeofSockaddrInet4 + } else { + inet6 := (*unix.RawSockaddrInet6)(unsafe.Pointer(name)) + inet6.Family = unix.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], packet.destination.Port()) + message.header.Namelen = unix.SizeofSockaddrInet6 + } + message.header.Name = (*byte)(unsafe.Pointer(name)) + } + if end-index > 1 { + hasSegments = true + control := b.controls[messageCount][:unix.CmsgSpace(2)] + clear(control) + controlHeader := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) + controlHeader.Level = unix.IPPROTO_UDP + controlHeader.Type = unix.UDP_SEGMENT + controlHeader.SetLen(unix.CmsgLen(2)) + binary.NativeEndian.PutUint16(control[unix.CmsgLen(0):], uint16(segmentSize)) + message.header.Control = &control[0] + message.header.SetControllen(len(control)) + } + b.counts[messageCount] = end - index + messageCount++ + index = end + } + //nolint:staticcheck + count, _, errno := unix.RawSyscall6(unix.SYS_SENDMMSG, uintptr(socket.fd), uintptr(unsafe.Pointer(&b.headers[0])), uintptr(messageCount), goSocketSendFlags, 0, 0) + if errno == unix.EINTR { + continue + } + if errno != 0 { + if hasSegments { + switch errno { + case unix.EINVAL, unix.EIO, unix.ENOPROTOOPT, unix.EOPNOTSUPP: + socket.packetGSODisabled = true + coalesce = false + continue + case unix.EMSGSIZE: + coalesce = false + continue + } + } + return sent, errno + } + if count == 0 { + return sent, unix.EIO + } + for index := range int(count) { + sent += b.counts[index] + } + } + return sent, 0 +} diff --git a/stack_go_socket_nonunix.go b/stack_go_socket_nonunix.go new file mode 100644 index 00000000..cb057213 --- /dev/null +++ b/stack_go_socket_nonunix.go @@ -0,0 +1,38 @@ +//go:build !linux && !darwin + +package tun + +import "syscall" + +type goPacketBatchIO struct{} + +func (b *goPacketBatchIO) reset() {} + +func (s *goSocket) enablePacketOffload() {} + +func (b *goPacketBatchIO) receive(socket *goSocket, packets []goPacketMessage, connected bool) (int, syscall.Errno) { + for index := range packets { + length, source, errno := socket.receiveFrom(packets[index].data) + if errno != 0 { + return index, errno + } + packets[index].data = packets[index].data[:length] + packets[index].destination = source + } + return len(packets), 0 +} + +func (b *goPacketBatchIO) send(socket *goSocket, packets []goPacketMessage, connected bool, family uint8) (int, syscall.Errno) { + for index, packet := range packets { + var errno syscall.Errno + if connected { + _, errno = socket.write(packet.data) + } else { + errno = socket.sendTo(packet.data, packet.destination, family) + } + if errno != 0 { + return index, errno + } + } + return len(packets), 0 +} diff --git a/stack_go_socket_unix.go b/stack_go_socket_unix.go new file mode 100644 index 00000000..5ee3ff37 --- /dev/null +++ b/stack_go_socket_unix.go @@ -0,0 +1,111 @@ +//go:build darwin || linux + +package tun + +import ( + "net/netip" + "os" + "syscall" + "unsafe" + + "github.com/sagernet/sing/common/control" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/unix" +) + +const ( + goSocketSendFlags = unix.MSG_DONTWAIT | unix.MSG_NOSIGNAL + goSpliceDuplicatesSocket = true +) + +type goIOVector = unix.Iovec + +type goSocket struct { + fd int + token uint32 + registered bool + packetGRO bool + packetGSODisabled bool +} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + fd, err := control.Conn0[int](conn, func(fd uintptr) (int, error) { + return unix.FcntlInt(fd, unix.F_DUPFD_CLOEXEC, 0) + }) + if err != nil { + return goSocket{}, err + } + return goSocket{fd: fd}, nil +} + +func (s *goSocket) close() { + if s.fd < 0 { + return + } + descriptor := s.fd + s.fd = -1 + unix.Close(descriptor) +} + +func newUnpolledFile(fd int, name string) *os.File { + unix.SetNonblock(fd, false) + file := os.NewFile(uintptr(fd), name) + unix.SetNonblock(fd, true) + return file +} + +func (s *goSocket) shutdownWrite() { + unix.Shutdown(s.fd, unix.SHUT_WR) +} + +func (s *goSocket) family() (uint8, error) { + name, err := unix.Getsockname(s.fd) + if err != nil { + return 0, err + } + switch name.(type) { + case *unix.SockaddrInet4: + return unix.AF_INET, nil + case *unix.SockaddrInet6: + return unix.AF_INET6, nil + default: + return 0, E.New("unsupported socket family") + } +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + name, err := unix.Getpeername(s.fd) + if err != nil { + return netip.AddrPort{}, false + } + return M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(name)).Unwrap().AddrPort(), true +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + iovecs = iovecs[:0] + for _, segment := range segments { + if len(segment) == 0 { + continue + } + vector := unix.Iovec{Base: &segment[0]} + vector.SetLen(len(segment)) + iovecs = append(iovecs, vector) + } + return iovecs +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == unix.EAGAIN || errno == unix.EWOULDBLOCK || errno == unix.EINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == unix.EMSGSIZE || errno == unix.EAFNOSUPPORT || errno == unix.ENOBUFS +} + +func (s *goSocket) readVector(iovecs []unix.Iovec) (int, syscall.Errno) { + //nolint:staticcheck + n, _, errno := unix.RawSyscall(unix.SYS_READV, uintptr(s.fd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs))) + return int(n), errno +} diff --git a/stack_go_socket_windows.go b/stack_go_socket_windows.go new file mode 100644 index 00000000..83521795 --- /dev/null +++ b/stack_go_socket_windows.go @@ -0,0 +1,222 @@ +package tun + +import ( + "encoding/binary" + "net/netip" + "runtime" + "syscall" + "unsafe" + + "github.com/sagernet/sing-tun/internal/afd" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + + "golang.org/x/sys/windows" +) + +const ( + goSpliceDuplicatesSocket = false + goSocketFIONBIO = 0x8004667e + goSocketReceiveBuffer = 4 << 20 +) + +type goIOVector = windows.WSABuf + +var ( + modws2_32 = windows.NewLazySystemDLL("ws2_32.dll") + procWSARecv = modws2_32.NewProc("WSARecv") + procWSARecvFrom = modws2_32.NewProc("WSARecvFrom") + procWSASend = modws2_32.NewProc("WSASend") + procWSASendTo = modws2_32.NewProc("WSASendTo") +) + +const goSocketError = uintptr(^uint32(0)) + +func goSocketErrno(r1 uintptr, errno syscall.Errno) syscall.Errno { + if r1 != goSocketError { + return 0 + } + if errno == 0 { + return syscall.EINVAL + } + return errno +} + +type goSocket struct { + handle windows.Handle + entry *goAFDEntry +} + +type goAFDEntry struct { + ioStatusBlock windows.IO_STATUS_BLOCK + pollInfo afd.PollInfo + baseHandle windows.Handle + token uint32 + interest uint8 + armed bool + cancelled bool + pinner runtime.Pinner +} + +func goSpliceSocket(conn syscall.Conn) (goSocket, error) { + rawConn, err := conn.SyscallConn() + if err != nil { + return goSocket{}, err + } + var handle windows.Handle + err = rawConn.Control(func(fd uintptr) { + handle = windows.Handle(fd) + }) + if err != nil { + return goSocket{}, err + } + var ( + mode uint32 = 1 + bytesReturned uint32 + ) + err = windows.WSAIoctl(handle, goSocketFIONBIO, (*byte)(unsafe.Pointer(&mode)), uint32(unsafe.Sizeof(mode)), nil, 0, &bytesReturned, nil, 0) + if err != nil { + return goSocket{}, E.Cause(err, "set socket non-blocking") + } + // Without an explicit receive buffer the socket advertises 64 KiB, and readiness-driven reads + // then cost a thread wake-up per window; the runtime poller hid this behind posted overlapped reads. + err = windows.SetsockoptInt(handle, windows.SOL_SOCKET, windows.SO_RCVBUF, goSocketReceiveBuffer) + if err != nil { + return goSocket{}, E.Cause(err, "set socket receive buffer") + } + return goSocket{handle: handle}, nil +} + +func (s *goSocket) close() { +} + +func (s *goSocket) shutdownWrite() { + windows.Shutdown(s.handle, windows.SHUT_WR) +} + +func (s *goSocket) family() (uint8, error) { + name, err := windows.Getsockname(s.handle) + if err != nil { + return 0, err + } + switch name.(type) { + case *windows.SockaddrInet4: + return windows.AF_INET, nil + case *windows.SockaddrInet6: + return windows.AF_INET6, nil + default: + return 0, E.New("unsupported socket family") + } +} + +func (s *goSocket) peerAddress() (netip.AddrPort, bool) { + name, err := windows.Getpeername(s.handle) + if err != nil { + return netip.AddrPort{}, false + } + address := M.SocksaddrFromNetIP(M.AddrPortFromSockaddr(name)).Unwrap() + return address.AddrPort(), address.IsValid() +} + +func goIovecsFromSegments(iovecs []goIOVector, segments [][]byte) []goIOVector { + iovecs = iovecs[:0] + for _, segment := range segments { + if len(segment) == 0 { + continue + } + iovecs = append(iovecs, windows.WSABuf{Len: uint32(len(segment)), Buf: &segment[0]}) + } + return iovecs +} + +func goSocketWouldBlock(errno syscall.Errno) bool { + return errno == windows.WSAEWOULDBLOCK || errno == windows.WSAEINTR +} + +func goSocketDropped(errno syscall.Errno) bool { + return errno == windows.WSAEMSGSIZE || errno == windows.WSAEAFNOSUPPORT || errno == windows.WSAENOBUFS +} + +func (s *goSocket) readVector(iovecs []goIOVector) (int, syscall.Errno) { + var ( + received uint32 + flags uint32 + ) + r1, _, e1 := syscall.SyscallN(procWSARecv.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs)), uintptr(unsafe.Pointer(&received)), uintptr(unsafe.Pointer(&flags)), 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno + } + return int(received), 0 +} + +func (s *goSocket) writeVector(iovecs []goIOVector) (int, syscall.Errno) { + var sent uint32 + r1, _, e1 := syscall.SyscallN(procWSASend.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(len(iovecs)), uintptr(unsafe.Pointer(&sent)), 0, 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno + } + return int(sent), 0 +} + +func (s *goSocket) write(data []byte) (int, syscall.Errno) { + vector := windows.WSABuf{Len: uint32(len(data))} + if len(data) > 0 { + vector.Buf = &data[0] + } + var sent uint32 + r1, _, e1 := syscall.SyscallN(procWSASend.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&sent)), 0, 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, errno + } + return int(sent), 0 +} + +func (s *goSocket) sendTo(data []byte, destination netip.AddrPort, family uint8) syscall.Errno { + var ( + storage windows.RawSockaddrAny + length int32 + ) + address := destination.Addr().Unmap() + if family == windows.AF_INET { + if !address.Is4() { + return windows.WSAEAFNOSUPPORT + } + inet4 := (*windows.RawSockaddrInet4)(unsafe.Pointer(&storage)) + inet4.Family = windows.AF_INET + inet4.Addr = address.As4() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet4.Port))[:], destination.Port()) + length = int32(unsafe.Sizeof(*inet4)) + } else { + inet6 := (*windows.RawSockaddrInet6)(unsafe.Pointer(&storage)) + inet6.Family = windows.AF_INET6 + inet6.Addr = address.As16() + binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&inet6.Port))[:], destination.Port()) + length = int32(unsafe.Sizeof(*inet6)) + } + vector := windows.WSABuf{Len: uint32(len(data))} + if len(data) > 0 { + vector.Buf = &data[0] + } + var sent uint32 + r1, _, e1 := syscall.SyscallN(procWSASendTo.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&sent)), 0, uintptr(unsafe.Pointer(&storage)), uintptr(length), 0, 0) + return goSocketErrno(r1, e1) +} + +func (s *goSocket) receiveFrom(buffer []byte) (int, netip.AddrPort, syscall.Errno) { + var ( + storage windows.RawSockaddrAny + length = int32(unsafe.Sizeof(storage)) + received uint32 + flags uint32 + ) + vector := windows.WSABuf{Len: uint32(len(buffer)), Buf: &buffer[0]} + r1, _, e1 := syscall.SyscallN(procWSARecvFrom.Addr(), uintptr(s.handle), uintptr(unsafe.Pointer(&vector)), 1, uintptr(unsafe.Pointer(&received)), uintptr(unsafe.Pointer(&flags)), uintptr(unsafe.Pointer(&storage)), uintptr(unsafe.Pointer(&length)), 0, 0) + errno := goSocketErrno(r1, e1) + if errno != 0 { + return 0, netip.AddrPort{}, errno + } + return int(received), M.SocksaddrFromNetIP(M.AddrPortFromRawSockaddr(&storage.Addr)).Unwrap().AddrPort(), 0 +} diff --git a/stack_go_splice.go b/stack_go_splice.go new file mode 100644 index 00000000..31d1de1c --- /dev/null +++ b/stack_go_splice.go @@ -0,0 +1,1004 @@ +package tun + +import ( + "io" + "net" + "net/netip" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +const ( + goSocketEventBatch = 64 + goSpliceTokenShift = 8 + goSpliceReadChunk = 2 * goSlabSize +) + +type SpliceSocket interface { + syscall.Conn + io.Closer + Attach(closer io.Closer) (io.Closer, bool) + Detach() +} + +type SpliceOptions struct { + ReadCounters []N.CountFunc + WriteCounters []N.CountFunc + OnClose N.CloseHandlerFunc +} + +type PacketNAT struct { + Origin M.Socksaddr + Destination M.Socksaddr + FakeIP bool + Unidirectional bool +} + +type SplicePacketOptions struct { + SpliceOptions + Timeout time.Duration + NAT PacketNAT + Cached []*N.PacketBuffer + Offload N.PacketOffload + FrontHeadroom int + RearHeadroom int +} + +type goSpliceSlot struct { + stream *GoConn + packet *GoPacketConn + generation uint32 +} + +func (e *goEngine) allocateSpliceToken(stream *GoConn, packet *GoPacketConn) uint32 { + var index uint32 + if free := len(e.spliceFree); free > 0 { + index = e.spliceFree[free-1] + e.spliceFree = e.spliceFree[:free-1] + } else { + index = uint32(len(e.spliceSlots)) + e.spliceSlots = append(e.spliceSlots, goSpliceSlot{}) + } + slot := &e.spliceSlots[index] + slot.generation = (slot.generation + 1) & (1<> goSpliceTokenShift + slot := &e.spliceSlots[index] + slot.stream = nil + slot.packet = nil + e.spliceFree = append(e.spliceFree, index) +} + +func (e *goEngine) spliceSlot(token uint32) *goSpliceSlot { + slot := &e.spliceSlots[token>>goSpliceTokenShift] + if slot.generation != token&(1<= available { + break + } + e.spliceSegments = conn.receiveChain.appendRuns(e.spliceSegments[:0], consumed, int(available-consumed)) + e.spliceIovecs = goIovecsFromSegments(e.spliceIovecs, e.spliceSegments) + n, errno := splice.socket.writeVector(e.spliceIovecs) + switch { + case errno == 0: + consumed = conn.consumedTail.Add(uint64(n)) + goCount(splice.options.ReadCounters, n) + if consumed < available { + splice.blocked = true + } + if consumed >= conn.windowUpdateAt.Load() { + e.handleWindowUpdate(conn) + } + case goSocketWouldBlock(errno): + splice.blocked = true + default: + e.abortConn(conn, E.Cause(errno, "go: write peer")) + return + } + } + consumed := conn.consumedTail.Load() + conn.receiveChain.releaseBelow(consumed) + if consumed == conn.receiveAvailable.Load() && (conn.oooRanges == nil || conn.oooRanges.count == 0) { + conn.receiveChain.releaseDrained(consumed) + } + interest := splice.interest &^ goInterestWrite + if splice.blocked { + interest |= goInterestWrite + } + e.spliceSetInterest(conn, interest) + if conn.splice != nil { + e.spliceCheckUpload(conn) + } +} + +func (e *goEngine) spliceCheckUpload(conn *GoConn) { + splice := conn.splice + if splice.uploadClosed || !conn.finReceived || conn.receiveAvailable.Load() > conn.consumedTail.Load() { + return + } + splice.socket.shutdownWrite() + splice.uploadClosed = true + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceAfterInput(conn *GoConn) { + splice := conn.splice + if splice == nil { + return + } + if conn.receiveAvailable.Load() > conn.consumedTail.Load() || (conn.finReceived && !splice.uploadClosed) { + e.spliceMarkDirty(conn) + return + } + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceReadPeer(conn *GoConn) { + // Empty speculative reads must not keep the partial tail slab while the + // socket is waiting. Unacknowledged and out-of-order data remain retained. + defer conn.releaseDrainedSlabs() + splice := conn.splice + capacity := conn.loadTransmitCapacity() + for !splice.downloadClosed { + budget := conn.writeBudget(0, capacity) + if budget <= 0 { + splice.readStalled = true + e.spliceSetInterest(conn, splice.interest&^goInterestRead) + return + } + budget = min(budget, goSpliceReadChunk) + buffered := conn.bufferedTail.Load() + conn.transmitStore.reserve(buffered, budget) + e.spliceSegments = conn.transmitStore.chain.appendRuns(e.spliceSegments[:0], buffered, budget) + e.spliceIovecs = goIovecsFromSegments(e.spliceIovecs, e.spliceSegments) + n, errno := splice.socket.readVector(e.spliceIovecs) + if errno != 0 || n < 0 { + n = 0 + } + if n < budget { + conn.transmitStore.shrinkReserve(buffered + uint64(n)) + } + switch { + case errno == 0 && n == 0: + splice.downloadClosed = true + e.spliceSetInterest(conn, splice.interest&^goInterestRead) + if conn.splice == nil { + return + } + conn.finPending = true + case errno == 0: + conn.bufferedTail.Store(buffered + uint64(n)) + goCount(splice.options.WriteCounters, n) + if !e.spliceTransmit(conn) { + return + } + case goSocketWouldBlock(errno): + e.spliceMaybeFinish(conn) + return + default: + e.abortConn(conn, E.Cause(errno, "go: read peer")) + return + } + } + e.maybeSendFin(conn) + e.spliceMaybeFinish(conn) +} + +func (e *goEngine) spliceTransmit(conn *GoConn) bool { + if !conn.transmitAccess.TryLock() { + return true + } + if conn.blockedValid { + conn.transmitAccess.Unlock() + return true + } + _, result, armed := conn.transmitLoop(false, 0) + conn.transmitAccess.Unlock() + if conn.retransmitDeadline == 0 && (armed || conn.hasOutstanding()) { + e.rearmRetransmit(conn) + } + if conn.dead { + return false + } + if result == goTransmitBlocked { + e.handleTransmitBlocked(conn) + } + return true +} + +func (e *goEngine) spliceResume(conn *GoConn) { + if !e.spliceTransmit(conn) { + return + } + splice := conn.splice + if splice == nil || !splice.readStalled || splice.downloadClosed || conn.writeBudget(0, conn.loadTransmitCapacity()) <= 0 { + return + } + splice.readStalled = false + e.spliceSetInterest(conn, splice.interest|goInterestRead) +} + +func (e *goEngine) spliceMaybeFinish(conn *GoConn) { + splice := conn.splice + if splice == nil || !splice.uploadClosed || !splice.downloadClosed || !conn.finSent { + return + } + e.spliceDetach(conn, nil) +} + +type goPacketMessage struct { + data []byte + destination netip.AddrPort + segmentSize int + truncated bool + payloadLength int +} + +type goPacketUpload struct { + message goPacketMessage + next int +} + +type goSplicePacket struct { + writer *GoPacketConn + owner SpliceSocket + original io.Closer + socket goSocket + token uint32 + family uint8 + connected bool + rewritePort bool + finished bool + peer netip.AddrPort + origin M.Socksaddr + destination M.Socksaddr + direct map[netip.Addr]struct{} + options SplicePacketOptions + uploadHead int + uploadTail int + nextDirty *goSplicePacket +} + +func (c *UDPNatConn) Splice(owner SpliceSocket, options SplicePacketOptions) bool { + writer, isGoPacketConn := c.writer.(*GoPacketConn) + if !isGoPacketConn || writer.conn.Load() != c { + return false + } + return writer.spliceTo(owner, options) +} + +func (w *GoPacketConn) spliceTo(owner SpliceSocket, options SplicePacketOptions) bool { + if !w.platformIO.supportsSockets() { + return false + } + if options.NAT.Origin.IsValid() && (!options.NAT.Origin.IsIP() || !options.NAT.Destination.IsIP() && options.Offload == nil) { + return false + } + splice := &goSplicePacket{writer: w, owner: owner, options: options, uploadHead: -1, uploadTail: -1} + original, attached := owner.Attach(splice) + if !attached { + return false + } + splice.original = original + socket, err := goSpliceSocket(owner) + if err != nil { + owner.Detach() + w.engine.stack.logger.Debug(E.Cause(err, "go: splice packet connection")) + return false + } + family, err := socket.family() + if err != nil { + socket.close() + owner.Detach() + return false + } + splice.socket = socket + splice.family = family + splice.peer, splice.connected = socket.peerAddress() + if options.Offload != nil && !splice.connected { + socket.close() + owner.Detach() + return false + } + if options.NAT.Origin.IsValid() { + splice.origin = options.NAT.Origin + splice.destination = options.NAT.Destination + splice.rewritePort = splice.origin.Port != splice.destination.Port + } + if !w.splicePending.CompareAndSwap(nil, splice) { + socket.close() + owner.Detach() + return false + } + w.engine.postMessage(&w.spliceMessage) + return true +} + +func (s *goSplicePacket) Close() error { + s.writer.closeSplice(net.ErrClosed) + return nil +} + +func (e *goEngine) handlePacketSpliceEngage(w *GoPacketConn) { + splice := w.splicePending.Load() + if splice == nil { + return + } + if w.splice != nil { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, net.ErrClosed) + return + } + natConn := w.conn.Load() + if natConn == nil || natConn.isClosed() { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, io.ErrClosedPipe) + return + } + closeError := w.spliceCloseError.Load() + if closeError != nil { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, closeError.err) + natConn.Close() + return + } + if splice.options.Timeout > 0 { + current := natConn.Timeout() + if !(current > 0 && splice.options.Timeout >= current) && !natConn.SetTimeout(splice.options.Timeout) { + w.splicePending.Store(nil) + e.packetSpliceRelease(splice, io.ErrClosedPipe) + return + } + } + splice.token = e.allocateSpliceToken(nil, w) + var err error + if goSpliceDuplicatesSocket { + err = splice.original.Close() + } + if err == nil { + err = e.platformIO.registerSocket(&splice.socket, splice.token, goInterestRead) + } + if err != nil { + w.splicePending.Store(nil) + e.releaseSpliceToken(splice.token) + e.packetSpliceRelease(splice, err) + natConn.Close() + return + } + w.splice = splice + splice.socket.enablePacketOffload() + w.spliceActive.Store(true) + w.splicePending.Store(nil) + readOptions := N.ReadWaitOptions{ + FrontHeadroom: splice.options.FrontHeadroom, + RearHeadroom: splice.options.RearHeadroom, + } + e.updatePacketHeadroom(readOptions) + cached := splice.options.Cached + splice.options.Cached = nil + for _, packet := range cached { + if w.splice != nil { + packet.Buffer = readOptions.Copy(packet.Buffer) + e.packetSpliceUpload(w, packet.Buffer, packet.Destination) + e.flushPacketUploads() + } + packet.Buffer.Release() + N.PutPacketBuffer(packet) + } + for w.splice != nil { + select { + case packet := <-natConn.packetChan: + packet.Buffer = readOptions.Copy(packet.Buffer) + e.packetSpliceUpload(w, packet.Buffer, packet.Destination) + e.flushPacketUploads() + packet.Buffer.Release() + N.PutPacketBuffer(packet) + default: + e.packetSpliceRead(w) + return + } + } +} + +func (e *goEngine) packetSpliceRelease(splice *goSplicePacket, err error) { + splice.finished = true + N.ReleaseMultiPacketBuffer(splice.options.Cached) + splice.options.Cached = nil + splice.socket.close() + splice.owner.Detach() + splice.owner.Close() + if splice.options.OnClose != nil { + splice.options.OnClose(err) + } +} + +func (e *goEngine) handlePacketSpliceClose(w *GoPacketConn) { + var err error + closeError := w.spliceCloseError.Load() + if closeError != nil { + err = closeError.err + } + pending := w.splicePending.Swap(nil) + if pending != nil { + e.packetSpliceRelease(pending, err) + } + if w.splice == nil { + return + } + e.packetSpliceClose(w, err) +} + +func (e *goEngine) packetSpliceClose(w *GoPacketConn, err error) { + splice := w.splice + w.splice = nil + w.spliceActive.Store(false) + e.platformIO.unregisterSocket(&splice.socket) + e.releaseSpliceToken(splice.token) + natConn := w.conn.Load() + if natConn != nil { + natConn.Close() + } + e.packetSpliceRelease(splice, err) +} + +func (e *goEngine) releasePending(message *goMessage) { + switch message.kind { + case goMessageConnSplice: + conn := message.conn + conn.access.Lock() + pending := conn.splicePending + conn.splicePending = nil + conn.access.Unlock() + if pending != nil { + e.spliceRelease(pending, net.ErrClosed) + } + case goMessagePacketSplice: + pending := message.packet.splicePending.Swap(nil) + if pending != nil { + e.packetSpliceRelease(pending, net.ErrClosed) + } + case goMessageConnDial: + conn := message.conn + conn.access.Lock() + if conn.err == nil { + conn.err = net.ErrClosed + } + dead := conn.dead + conn.dead = true + conn.access.Unlock() + if !dead { + close(conn.establishedSignal) + close(conn.closeSignal) + } + case goMessageUDPOpen: + socket := message.socket + socket.access.Lock() + if socket.openErr == nil { + socket.openErr = net.ErrClosed + } + socket.access.Unlock() + close(socket.openSignal) + case goMessageUDPClose: + message.socket.drain() + case goMessageListenOpen: + listener := message.listener + listener.access.Lock() + if listener.openErr == nil { + listener.openErr = net.ErrClosed + } + listener.access.Unlock() + close(listener.openSignal) + case goMessageListenClose: + message.listener.drain() + case goMessageInject: + e.releaseInjected() + } +} + +func (e *goEngine) closeAllPacketSplices() { + for index := range e.spliceSlots { + packet := e.spliceSlots[index].packet + if packet != nil && packet.splice != nil { + e.packetSpliceClose(packet, net.ErrClosed) + } + } +} + +func (s *goSplicePacket) mapUpload(destination M.Socksaddr) M.Socksaddr { + if s.origin.IsValid() && destination.Addr == s.origin.Addr { + if !s.rewritePort { + return M.Socksaddr{Addr: s.destination.Addr, Fqdn: s.destination.Fqdn, Port: destination.Port} + } + if destination.Port == s.origin.Port { + return s.destination + } + } + if s.options.NAT.FakeIP && destination.IsIP() { + if s.direct == nil { + s.direct = make(map[netip.Addr]struct{}) + } + s.direct[destination.Addr] = struct{}{} + } + return destination +} + +func (s *goSplicePacket) mapDownload(source M.Socksaddr) M.Socksaddr { + if s.options.NAT.Unidirectional { + return source + } + if s.origin.IsValid() && source.Addr == s.destination.Addr && (s.destination.IsIP() || source.Fqdn == s.destination.Fqdn) { + if !s.rewritePort { + return M.Socksaddr{Addr: s.origin.Addr, Port: source.Port} + } + if source.Port == s.destination.Port { + return s.origin + } + } + if s.options.NAT.FakeIP { + _, direct := s.direct[source.Addr] + if !direct { + return M.Socksaddr{Addr: s.origin.Addr, Port: source.Port} + } + } + return source +} + +func (e *goEngine) packetSpliceUpload(w *GoPacketConn, buffer *buf.Buffer, destination M.Socksaddr) { + splice := w.splice + target := splice.mapUpload(destination) + payloadLength := buffer.Len() + if splice.options.Offload != nil { + err := splice.options.Offload.EncodePacket(buffer, target) + if err != nil { + e.packetSpliceClose(w, E.Cause(err, "go: encode packet")) + return + } + } + e.queuePacketUpload(splice, buffer.Bytes(), target.AddrPort(), payloadLength) +} + +func (e *goEngine) updatePacketHeadroom(options N.ReadWaitOptions) { + previous := *e.packetReadOptions.Load() + options.FrontHeadroom = max(previous.FrontHeadroom, options.FrontHeadroom-header.IPv4MinimumSize-header.UDPMinimumSize) + options.RearHeadroom = max(previous.RearHeadroom, options.RearHeadroom) + if options == previous { + return + } + e.flushPacketUploads() + for index := range e.reassemblyEntries { + entry := &e.reassemblyEntries[index] + if !entry.active { + continue + } + buffer := buf.NewSize(options.FrontHeadroom + goReassemblyCapacity + options.RearHeadroom) + copy(buffer.FreeBytes()[options.FrontHeadroom:], entry.buffer.Range(entry.headroom-goReassemblyHeadroom, entry.headroom+65535)) + entry.buffer.Release() + entry.buffer = buffer + entry.headroom = options.FrontHeadroom + goReassemblyHeadroom + } + clear(e.frames) + e.udpUserData.packet = nil + e.packetReadOptions.Store(&options) +} + +func (e *goEngine) packetSpliceRead(w *GoPacketConn) { + splice := w.splice + natConn := w.conn.Load() + batchSize := e.packetReceiveBatch + e.packetSlots.configure(goPacketBatchSize, 65535) + e.packetSlots.wake(batchSize) + for index := range batchSize { + buffer := e.packetSlots.slot(index) + buffer.Reset() + e.packetMessages[index] = goPacketMessage{data: buffer.FreeBytes()} + } + count, errno := e.packetIO.receive(&splice.socket, e.packetMessages[:batchSize], splice.connected) + if count == batchSize { + e.packetReceiveBatch = min(batchSize*2, goPacketBatchSize) + } else if count*2 < batchSize { + e.packetReceiveBatch = max(batchSize/2, goReceiveBatchMin) + } + defer func() { + e.flushPacketFrames() + clear(e.packetMessages[:]) + if w.splice == nil { + return + } + if count == 0 && errno == 0 { + e.packetSpliceClose(w, io.EOF) + } else if errno != 0 && !goSocketWouldBlock(errno) { + e.packetSpliceClose(w, E.Cause(errno, "go: receive peer batch")) + } + }() + for index := range count { + message := e.packetMessages[index] + if message.truncated { + continue + } + buffer := e.packetSlots.slot(index) + offset := 0 + for { + length := len(message.data) - offset + if message.segmentSize > 0 { + length = min(length, message.segmentSize) + } + buffer.Resize(offset, length) + var destination M.Socksaddr + if splice.options.Offload != nil { + packetSource, err := splice.options.Offload.DecodePacket(buffer) + if err != nil { + e.packetSpliceClose(w, E.Cause(err, "go: decode packet")) + return + } + destination = splice.mapDownload(packetSource.Unwrap()) + } else { + source := message.destination + if !source.IsValid() { + source = splice.peer + } + destination = splice.mapDownload(M.SocksaddrFromNetIP(source)) + } + if destination.IsIP() && natConn.allowPeer(destination) { + goCount(splice.options.WriteCounters, buffer.Len()) + e.packetSpliceDownload(w, buffer.Bytes(), destination) + } + offset += length + if offset == len(message.data) { + break + } + } + } +} + +func (e *goEngine) flushPacketUploads() { + for e.packetDirtyList != nil { + splice := e.packetDirtyList + e.packetDirtyList = splice.nextDirty + splice.nextDirty = nil + count := 0 + for index := splice.uploadHead; index >= 0; index = e.packetUploads[index].next { + upload := &e.packetUploads[index] + e.packetMessages[count] = upload.message + count++ + } + splice.uploadHead = -1 + splice.uploadTail = -1 + if !splice.finished { + sent, errno := e.packetIO.send(&splice.socket, e.packetMessages[:count], splice.connected, splice.family) + length := 0 + for index := range sent { + length += e.packetMessages[index].payloadLength + } + goCount(splice.options.ReadCounters, length) + if errno != 0 && !goSocketWouldBlock(errno) && !goSocketDropped(errno) { + e.packetSpliceClose(splice.writer, E.Cause(errno, "go: send peer batch")) + } + } + clear(e.packetMessages[:count]) + } + clear(e.packetUploads[:e.packetUploadCount]) + e.packetUploadCount = 0 +} + +func (e *goEngine) queuePacketUpload(splice *goSplicePacket, data []byte, destination netip.AddrPort, length int) { + if e.packetUploadCount == len(e.packetUploads) { + e.flushPacketUploads() + if splice.finished { + return + } + } + index := e.packetUploadCount + e.packetUploadCount++ + e.packetUploads[index] = goPacketUpload{ + message: goPacketMessage{data: data, destination: destination, payloadLength: length}, + next: -1, + } + if splice.uploadHead < 0 { + splice.uploadHead = index + splice.nextDirty = e.packetDirtyList + e.packetDirtyList = splice + } else { + e.packetUploads[splice.uploadTail].next = index + } + splice.uploadTail = index +} + +func (e *goEngine) flushPacketFrames() { + if e.packetFrameCount == 0 { + return + } + err := e.platformIO.writePacketBatch(e.packetFrames[:e.packetFrameCount]) + if err != nil && err != errGoFrameDropped { + e.stack.logger.Trace(E.Cause(err, "go: write packet batch")) + } + clear(e.packetFrames[:e.packetFrameCount]) + e.packetFrameCount = 0 +} + +func (e *goEngine) packetSpliceDownload(w *GoPacketConn, data []byte, destination M.Socksaddr) { + if len(data)+w.templateLength > w.platformIO.mtu() { + e.flushPacketFrames() + buffer := buf.NewSize(w.FrontHeadroom() + len(data)) + buffer.Resize(w.FrontHeadroom(), 0) + copy(buffer.Extend(len(data)), data) + err := w.transmit(buffer, destination) + buffer.Release() + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write fragmented packet")) + } + return + } + frame := &e.packetFrames[e.packetFrameCount] + meta, err := w.preparePacketHeader(frame.header[:], data, destination, w.checksumOffload) + if err != nil { + if err != errGoFrameDropped { + e.stack.logger.Trace(err) + } + return + } + frame.length = w.templateLength + frame.payload = data + frame.meta = meta + e.packetFrameCount++ + if e.packetFrameCount == len(e.packetFrames) { + e.flushPacketFrames() + } +} diff --git a/stack_go_tcp_ack.go b/stack_go_tcp_ack.go new file mode 100644 index 00000000..cde9c6b4 --- /dev/null +++ b/stack_go_tcp_ack.go @@ -0,0 +1,1198 @@ +package tun + +import ( + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" +) + +// Mirrors the ACK processing and loss recovery state machine of net/ipv4/tcp_input.c. + +type goSackState struct { + firstStamp int32 + lastStamp int32 + sampled bool +} + +func (e *goEngine) drainDescriptors(conn *GoConn) { + before := len(conn.scoreboard.entries) + conn.scoreboard.drain(&conn.descriptors, conn.sendUnacked.Load()) + entries := conn.scoreboard.entries + for index := before; index < len(entries); index++ { + descriptor := &entries[index] + sentAt := conn.descriptorSentTime(descriptor) + if descriptor.flags&goDescriptorTxStart != 0 { + descriptor.flags &^= goDescriptorTxStart + conn.rackEndOffset = 0 + conn.rackAdvanced = false + if conn.congestion.windowEventTxStart != nil { + var idle int64 + if conn.lastSendStamp >= 0 { + idle = max(sentAt-conn.lastSendStamp, 0) + } + conn.congestion.windowEventTxStart(conn, idle) + } + } + conn.lastSendStamp = sentAt + } + e.refreshPacketsOut(conn) +} + +func (e *goEngine) summarizeFlight(conn *GoConn) { + var summary goFlightSummary + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.flags&goDescriptorRetransmitted != 0 { + summary.retransmitted++ + } + if descriptor.flags&goDescriptorSacked != 0 { + summary.sackedOut++ + continue + } + if descriptor.flags&goDescriptorLost != 0 { + summary.lostOut++ + } + if descriptor.flags&goDescriptorRetransmitOut != 0 { + summary.retransmitOut++ + } + } + if !conn.sackPermitted { + summary.sackedOut = conn.renoSacked + } + conn.flight = summary + e.refreshPacketsOut(conn) +} + +func (e *goEngine) refreshPacketsOut(conn *GoConn) { + entries := conn.scoreboard.entries + start := conn.sendUnacked.Load() + if len(entries) > 0 { + start = entries[len(entries)-1].endOffset + } + sent := conn.sentTail.Load() + if sent > start { + start = sent + } + conn.flight.packetsOut = conn.dataSegmentsOut.Load() - conn.packetCreditBase.Load() + conn.flight.packetsEnd = start +} + +func (e *goEngine) accountRemoved(conn *GoConn, flags uint16) { + conn.packetCreditBase.Add(1) + if flags&goDescriptorRetransmitted != 0 { + conn.flight.retransmitted-- + } + if flags&goDescriptorSacked != 0 { + conn.flight.sackedOut-- + return + } + if flags&goDescriptorLost != 0 { + conn.flight.lostOut-- + } + if flags&goDescriptorRetransmitOut != 0 { + conn.flight.retransmitOut-- + } +} + +func (e *goEngine) processAck(conn *GoConn, tcpHdr header.TCP, segOffset int64, payloadLength int, flags header.TCPFlags, timestampEcho uint32, hasTimestamp bool) bool { + e.refreshCoarseTime() + e.drainDescriptors(conn) + ackOffset := conn.sendOffset(tcpHdr.AckNumber()) + limit := int64(conn.sentTail.Load()) + if conn.finSent { + limit++ + } + if ackOffset > limit { + e.markAck(conn, true) + return false + } + priorUnacked := conn.sendUnacked.Load() + segmentAck := ackOffset + if ackOffset < int64(priorUnacked) { + ackOffset = int64(priorUnacked) + } + if conn.state == goTCPSynReceived { + if conn.phase != goPhaseEngaged { + return false + } + if segmentAck != int64(conn.sentTail.Load()) { + e.sendReset(conn) + e.detachConn(conn, E.New("go: handshake ack out of range"), goDeathImmediate) + return false + } + e.establishConn(conn) + } + if hasTimestamp { + conn.lastTimestampEcho = timestampEcho + } + var ackFlags goAckFlags + if payloadLength > 0 || flags&(header.TCPFlagSyn|header.TCPFlagFin) != 0 { + ackFlags |= goAckData + } + rawCount := 0 + if conn.sackPermitted { + rawCount = goParseSackBlocks(tcpHdr, int(tcpHdr.DataOffset()), conn, &e.sackRaw) + } + if segmentAck < int64(priorUnacked) { + if rawCount > 0 { + e.processOldAckSack(conn, priorUnacked, segmentAck, rawCount) + } + return true + } + windowRaw := tcpHdr.WindowSize() + newWindow := uint64(windowRaw) << conn.peerWindowShift + if segmentAck > int64(priorUnacked) || segOffset > conn.windowLeft1 || (segOffset == conn.windowLeft1 && (newWindow > conn.peerWindow || newWindow == 0)) { + ackFlags |= goAckWindowUpdate + conn.windowLeft1 = segOffset + conn.windowLeft2 = segmentAck + conn.peerWindow = newWindow + conn.maxPeerWindow = max(conn.maxPeerWindow, newWindow) + conn.lastPeerWindow = windowRaw + } + if ackOffset > int64(priorUnacked) { + ackFlags |= goAckUnackedAdvanced + conn.retransmitAttempts = 0 + } + priorFack := conn.highestSacked + hadOutstanding := conn.hasOutstanding() + priorDelivered := conn.delivered + priorLost := conn.lost + sample := &e.rateSample + *sample = goRateSample{roundTripMicros: -1, priorInFlight: conn.flight.inFlight()} + var sackState goSackState + if rawCount > 0 { + raw := e.sackRaw[:rawCount] + if goIsDuplicateSack(raw, ackOffset) { + e.applyDuplicateSack(conn, raw[0], priorUnacked, &ackFlags) + raw = raw[1:] + } + sackCount := goSelectSackBlocks(raw, uint64(ackOffset), conn.sentTail.Load(), &e.sackScratch) + if sackCount > 0 { + e.applySackBlocks(conn, e.sackScratch[:sackCount], &ackFlags, sample, &sackState) + } + } + ackFlags |= e.cleanRetransmitQueue(conn, uint64(ackOffset), priorFack, priorUnacked, sample, &sackState, timestampEcho, hasTimestamp) + e.rackUpdateReorderWindow(conn, sample) + if conn.probeHighSeq != 0 { + e.processProbeAck(conn, uint64(ackOffset), ackFlags) + } + rexmit := goRexmitNone + if !hadOutstanding { + if ackFlags&goAckDSACK != 0 { + rexmit = e.fastretransAlert(conn, priorUnacked, 0, &ackFlags) + } + } else { + if ackFlags&goAckNotDuplicate == 0 || ackFlags&goAckAlert != 0 || conn.congestionState != goCongestionOpen { + numDupack := uint32(0) + if ackFlags&(goAckUnackedAdvanced|goAckNotDuplicate|goAckDSACK) == 0 { + numDupack = 1 + } + rexmit = e.fastretransAlert(conn, priorUnacked, numDupack, &ackFlags) + } + sample.ackDelayed = ackFlags&goAckMaybeDelayed != 0 + delivered := conn.delivered - priorDelivered + e.rateGenerate(conn, sample, delivered, conn.lost-priorLost, e.coarseTime.Load()) + e.validateWindow(conn) + e.congestionControl(conn, delivered, ackFlags, sample) + e.updateFrameLimit(conn) + } + if ackFlags&goAckUnackedAdvanced != 0 { + e.advanceFinState(conn) + if conn.dead { + return true + } + if len(conn.scoreboard.entries) == 0 && conn.sendUnacked.Load() == conn.sentTail.Load() { + e.armIdleRestart(conn) + } + if ackFlags&goAckKeepTimer == 0 { + e.rearmRetransmit(conn) + } + } + e.xmitRecovery(conn, rexmit) + e.updatePersist(conn) + conn.releaseIdleDescriptors() + conn.publishPermit(uint64(ackOffset)) + return true +} + +func (e *goEngine) processOldAckSack(conn *GoConn, priorUnacked uint64, ackOffset int64, rawCount int) { + var ackFlags goAckFlags + sample := goRateSample{roundTripMicros: -1, priorInFlight: conn.flight.inFlight()} + var sackState goSackState + raw := e.sackRaw[:rawCount] + if goIsDuplicateSack(raw, ackOffset) { + e.applyDuplicateSack(conn, raw[0], priorUnacked, &ackFlags) + raw = raw[1:] + } + sackCount := goSelectSackBlocks(raw, priorUnacked, conn.sentTail.Load(), &e.sackScratch) + if sackCount > 0 { + e.applySackBlocks(conn, e.sackScratch[:sackCount], &ackFlags, &sample, &sackState) + } + e.refreshPacketsOut(conn) + rexmit := e.fastretransAlert(conn, priorUnacked, 0, &ackFlags) + e.xmitRecovery(conn, rexmit) + conn.publishPermit(priorUnacked) +} + +func (e *goEngine) applyDuplicateSack(conn *GoConn, block goRawSackBlock, priorUnacked uint64, flags *goAckFlags) { + if block.start < 1 || block.end <= block.start || uint64(block.end) > conn.sentTail.Load() { + return + } + start := uint64(block.start) + end := uint64(block.end) + if end-start > conn.maxPeerWindow { + return + } + packets := (end - start + uint64(conn.effectiveMSS.Load()) - 1) / uint64(conn.effectiveMSS.Load()) + conn.duplicateSegments += packets + if conn.duplicateSegments > conn.totalRetransmits { + return + } + *flags |= goAckDSACK + if end-start <= uint64(conn.effectiveMSS.Load()) && conn.probeHighSeq != 0 && conn.probeRetransmitted && end == conn.probeHighSeq { + *flags |= goAckDSACKProbe + } + if conn.reorderingSeen != 0 && *flags&goAckDSACKProbe == 0 { + conn.rackDSACKSeen = true + } + if end <= priorUnacked { + if conn.undoMarker != 0 && conn.undoRetransmits > 0 && end > conn.undoMarker { + conn.undoRetransmits -= int32(min(packets, uint64(conn.undoRetransmits))) + } + return + } + descriptorStart := priorUnacked + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + descriptorEnd := descriptor.endOffset + if descriptorStart >= end { + break + } + if descriptorStart >= start && descriptorEnd <= end && descriptor.flags&goDescriptorRetransmitted != 0 { + if conn.undoMarker != 0 && conn.undoRetransmits > 0 && descriptorEnd > conn.undoMarker { + conn.undoRetransmits-- + } + if descriptor.flags&goDescriptorRetransmitOut != 0 && descriptor.flags&goDescriptorSacked == 0 { + conn.flight.retransmitOut-- + } + descriptor.flags &^= goDescriptorRetransmitOut + } + descriptorStart = descriptorEnd + } +} + +func (e *goEngine) applySackBlocks(conn *GoConn, blocks []goSackBlock, flags *goAckFlags, sample *goRateSample, sackState *goSackState) { + unacked := conn.sendUnacked.Load() + priorHighest := conn.highestSacked + reorder := conn.sentTail.Load() + start := unacked + scoreboard := &conn.scoreboard + for index := 0; index < len(scoreboard.entries); index++ { + descriptor := &scoreboard.entries[index] + end := descriptor.endOffset + if descriptor.flags&goDescriptorSacked == 0 { + for _, block := range blocks { + if block.start > start || block.end < end { + continue + } + e.rackAdvance(conn, descriptor, end) + if descriptor.flags&goDescriptorRetransmitted == 0 { + if start < priorHighest && start < reorder { + reorder = start + } + if end <= conn.highSeq { + *flags |= goAckOriginalSacked + } + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) == 0 { + if !sackState.sampled { + sackState.firstStamp = descriptor.sentAt + sackState.sampled = true + } + sackState.lastStamp = descriptor.sentAt + } + } + if descriptor.flags&goDescriptorRetransmitOut != 0 { + if descriptor.flags&goDescriptorLost != 0 { + descriptor.flags &^= goDescriptorLost | goDescriptorRetransmitOut + conn.flight.lostOut-- + conn.flight.retransmitOut-- + } else { + conn.flight.retransmitOut-- + } + } else if descriptor.flags&goDescriptorLost != 0 { + descriptor.flags &^= goDescriptorLost + conn.flight.lostOut-- + } + descriptor.flags |= goDescriptorSacked + conn.flight.sackedOut++ + conn.highestSacked = max(conn.highestSacked, end) + *flags |= goAckDataSacked + conn.countDelivered(1) + e.rateDescriptorDelivered(conn, descriptor, sample) + break + } + } + start = end + } + if conn.congestionState != goCongestionLoss || conn.undoMarker != 0 { + e.checkSackReordering(conn, reorder) + } +} + +func (e *goEngine) cleanRetransmitQueue(conn *GoConn, ackOffset uint64, priorFack uint64, priorUnacked uint64, sample *goRateSample, sackState *goSackState, timestampEcho uint32, hasTimestamp bool) goAckFlags { + var flags goAckFlags + sent := conn.sentTail.Load() + newUnacked := min(ackOffset, sent) + priorSacked := conn.flight.sackedOut + var ( + firstStamp int32 + lastStamp int32 + sampled bool + unsampled bool + packetsAcked uint32 + ) + reorder := sent + start := priorUnacked + entries := conn.scoreboard.entries + for index := range entries { + descriptor := &entries[index] + if descriptor.endOffset > newUnacked { + break + } + if descriptor.flags&goDescriptorRetransmitted != 0 { + flags |= goAckRetransmittedAcked + } else if descriptor.flags&goDescriptorSacked == 0 { + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + unsampled = true + } else { + lastStamp = descriptor.sentAt + if !sampled { + firstStamp = descriptor.sentAt + sampled = true + } + } + if start < reorder { + reorder = start + } + if descriptor.endOffset <= conn.highSeq { + flags |= goAckOriginalSacked + } + } + if descriptor.flags&goDescriptorSacked == 0 && conn.sackPermitted { + conn.countDelivered(1) + if !conn.spuriousRetransmit(descriptor) { + e.rackAdvance(conn, descriptor, descriptor.endOffset) + } + } + packetsAcked++ + e.rateDescriptorDelivered(conn, descriptor, sample) + e.accountRemoved(conn, descriptor.flags) + flags |= goAckDataAcked + start = descriptor.endOffset + } + if conn.finSent && !conn.finAcked && ackOffset > conn.finOffset { + conn.finAcked = true + close(conn.finAckedSignal) + } + if newUnacked > priorUnacked { + conn.scoreboard.advance(newUnacked) + conn.sendUnacked.Store(newUnacked) + conn.releaseTransmitted(newUnacked) + if len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorSacked != 0 { + flags |= goAckReneging + } + } + e.refreshPacketsOut(conn) + now := e.coarseTime.Load() + nowStamp := conn.stamp(now) + sequenceRoundTrip := int64(-1) + sackRoundTrip := int64(-1) + controlRoundTrip := int64(-1) + if sampled && !unsampled && flags&goAckRetransmittedAcked == 0 { + sequenceRoundTrip = goStampMicros(nowStamp - firstStamp) + controlRoundTrip = goStampMicros(nowStamp - lastStamp) + if packetsAcked == 1 && priorSacked == 0 && newUnacked-priorUnacked < uint64(conn.effectiveMSS.Load()) && + sample.priorDelivered+1 == conn.delivered && flags&goAckAlert == 0 { + flags |= goAckMaybeDelayed + } + } + if sackState.sampled { + sackRoundTrip = goStampMicros(nowStamp - sackState.firstStamp) + controlRoundTrip = goStampMicros(nowStamp - sackState.lastStamp) + } + e.ackUpdateRoundTrip(conn, flags, sequenceRoundTrip, sackRoundTrip, controlRoundTrip, sample, timestampEcho, hasTimestamp, now) + if flags&goAckDataAcked != 0 { + if !conn.sackPermitted { + e.removeRenoSacks(conn, packetsAcked) + if flags&goAckRetransmittedAcked != 0 { + flags &^= goAckOriginalSacked + } + } else if reorder < priorFack { + e.checkSackReordering(conn, reorder) + } + conn.probeAttempts = 0 + } + if conn.congestion.packetsAcked != nil { + ackSample := &e.ackSample + *ackSample = goAckSample{ + packetsAcked: packetsAcked, + roundTripMicros: sample.roundTripMicros, + inFlightBytes: uint64(conn.effectiveMSS.Load()) * uint64(conn.delivered-sample.priorDelivered), + } + conn.congestion.packetsAcked(conn, ackSample) + } + return flags +} + +func (e *goEngine) ackUpdateRoundTrip(conn *GoConn, flags goAckFlags, sequenceRoundTrip int64, sackRoundTrip int64, controlRoundTrip int64, sample *goRateSample, timestampEcho uint32, hasTimestamp bool, now int64) { + if sequenceRoundTrip < 0 { + sequenceRoundTrip = sackRoundTrip + } + if sequenceRoundTrip < 0 && hasTimestamp && timestampEcho != 0 && flags&goAckDataAcked != 0 { + elapsed := goTimestampAt(now) - timestampEcho + if int32(elapsed) >= 0 { + sequenceRoundTrip = max(int64(elapsed)< conn.roundTripMin.get() { + return + } + conn.roundTripMin.runningMin(uint32(goMinRoundTripWindow/time.Millisecond), uint32(now/int64(time.Millisecond)), max(roundTripMicros, 1)) +} + +func (e *goEngine) limitRenoSacked(conn *GoConn) bool { + holes := min(max(conn.flight.lostOut, 1), conn.flight.packetsOut) + if conn.renoSacked+holes > conn.flight.packetsOut { + conn.renoSacked = conn.flight.packetsOut - holes + conn.flight.sackedOut = conn.renoSacked + return true + } + return false +} + +func (e *goEngine) checkRenoReordering(conn *GoConn, addend uint32) { + if !e.limitRenoSacked(conn) { + return + } + conn.reordering = min(conn.flight.packetsOut+addend, goMaxReordering) + conn.reorderingSeen++ +} + +func (e *goEngine) addRenoSack(conn *GoConn, numDupack uint32) { + if numDupack == 0 { + return + } + prior := conn.renoSacked + conn.renoSacked += numDupack + conn.flight.sackedOut = conn.renoSacked + e.checkRenoReordering(conn, 0) + if conn.renoSacked > prior { + conn.countDelivered(conn.renoSacked - prior) + } +} + +func (e *goEngine) removeRenoSacks(conn *GoConn, acked uint32) { + if acked > 0 { + conn.countDelivered(max(acked-min(acked, conn.renoSacked), 1)) + if acked-1 >= conn.renoSacked { + conn.renoSacked = 0 + } else { + conn.renoSacked -= acked - 1 + } + } + conn.flight.sackedOut = conn.renoSacked + e.checkRenoReordering(conn, acked) +} + +func (e *goEngine) processProbeAck(conn *GoConn, ackOffset uint64, flags goAckFlags) { + if ackOffset < conn.probeHighSeq { + return + } + switch { + case !conn.probeRetransmitted: + conn.probeHighSeq = 0 + case flags&goAckDSACKProbe != 0: + conn.probeHighSeq = 0 + case ackOffset > conn.probeHighSeq: + e.initWindowReduction(conn) + conn.setCAState(goCongestionCWR) + e.endWindowReduction(conn) + e.tryKeepOpen(conn) + case flags&(goAckUnackedAdvanced|goAckNotDuplicate|goAckDataSacked) == 0: + conn.probeHighSeq = 0 + } +} + +func (e *goEngine) markLost(conn *GoConn, descriptor *goSentDescriptor) { + if descriptor.flags&goDescriptorSacked != 0 || descriptor.endOffset > conn.transmittedTail.Load() { + return + } + if descriptor.flags&goDescriptorLost != 0 { + if descriptor.flags&goDescriptorRetransmitOut != 0 { + descriptor.flags &^= goDescriptorRetransmitOut + conn.flight.retransmitOut-- + conn.lost++ + } + return + } + descriptor.flags |= goDescriptorLost + conn.flight.lostOut++ + conn.lost++ +} + +func (e *goEngine) accountSplit(conn *GoConn, flags uint16) { + conn.packetCreditBase.Add(^uint32(0)) + conn.flight.packetsOut++ + if flags&goDescriptorRetransmitted != 0 { + conn.flight.retransmitted++ + } + if flags&goDescriptorSacked != 0 { + conn.flight.sackedOut++ + return + } + if flags&goDescriptorLost != 0 { + conn.flight.lostOut++ + } + if flags&goDescriptorRetransmitOut != 0 { + conn.flight.retransmitOut++ + } +} + +func (c *GoConn) descriptorSentTime(descriptor *goSentDescriptor) int64 { + now := c.engine.coarseTime.Load() + units := (now - c.epoch) / goTimeUnit + age := int64(int32(units) - descriptor.sentAt) + return c.epoch + (units-age)*goTimeUnit +} + +func (c *GoConn) spuriousRetransmit(descriptor *goSentDescriptor) bool { + return descriptor.flags&goDescriptorRetransmitted != 0 && c.echoedTimestampBefore(goTimestampAt(c.descriptorSentTime(descriptor))) +} + +func (e *goEngine) rackAdvance(conn *GoConn, descriptor *goSentDescriptor, endOffset uint64) { + if descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + return + } + roundTrip := goStampMicros(conn.stamp(e.coarseTime.Load()) - descriptor.sentAt) + if roundTrip < int64(conn.roundTripMin.get()) && descriptor.flags&goDescriptorRetransmitted != 0 { + return + } + conn.rackAdvanced = true + conn.rackRoundTripMicros = roundTrip + if conn.rackEndOffset == 0 || goStampAfter(descriptor.sentAt, conn.rackStamp, endOffset, conn.rackEndOffset) { + conn.rackStamp = descriptor.sentAt + conn.rackEndOffset = endOffset + } +} + +func (e *goEngine) rackReorderWindow(conn *GoConn) int64 { + if conn.reorderingSeen == 0 { + if conn.congestionState >= goCongestionRecovery { + return 0 + } + if conn.flight.sackedOut >= conn.reordering { + return 0 + } + } + return min(int64(conn.roundTripMin.get()>>2)*int64(conn.reorderWindowSteps), int64(conn.smoothedRoundTrip)) +} + +func (e *goEngine) rackDescriptorTimeout(conn *GoConn, descriptor *goSentDescriptor, reorderWindow int64, now int64) int64 { + return conn.rackRoundTripMicros + reorderWindow - goStampMicros(conn.stamp(now)-descriptor.sentAt) +} + +func (e *goEngine) rackDetectLoss(conn *GoConn) int64 { + if conn.rackEndOffset == 0 { + return 0 + } + now := e.coarseTime.Load() + reorderWindow := e.rackReorderWindow(conn) + timeout := int64(0) + ordered := conn.flight.retransmitted == 0 + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.flags&goDescriptorSacked != 0 { + continue + } + if descriptor.flags&goDescriptorLost != 0 && descriptor.flags&goDescriptorRetransmitOut == 0 { + continue + } + if !goStampAfter(conn.rackStamp, descriptor.sentAt, conn.rackEndOffset, descriptor.endOffset) { + if ordered { + break + } + continue + } + remaining := e.rackDescriptorTimeout(conn, descriptor, reorderWindow, now) + if remaining <= 0 { + e.markLost(conn, descriptor) + } else { + timeout = max(timeout, remaining) + } + } + return timeout +} + +func (e *goEngine) rackMarkLost(conn *GoConn) bool { + if !conn.rackAdvanced { + return false + } + conn.rackAdvanced = false + timeout := e.rackDetectLoss(conn) + if timeout == 0 { + return false + } + conn.reorderDeadline = e.coarseTime.Load() + (timeout+goRackTimeoutMinMicros)*int64(time.Microsecond) + e.rearmTimer(conn) + return true +} + +func (e *goEngine) expireReorder(conn *GoConn) { + conn.reorderDeadline = 0 + if !conn.hasOutstandingData() { + e.rearmTimer(conn) + return + } + e.drainDescriptors(conn) + e.summarizeFlight(conn) + priorInFlight := conn.flight.inFlight() + priorLost := conn.lost + e.rackDetectLoss(conn) + if priorInFlight != conn.flight.inFlight() { + if conn.congestionState != goCongestionRecovery { + e.enterRecovery(conn) + if conn.congestion.congControl == nil { + e.reduceWindow(conn, 1, conn.lost-priorLost, 0) + } + } + e.xmitRetransmitQueue(conn) + conn.publishPermit(conn.sendUnacked.Load()) + } + if conn.retransmitDeadline == 0 { + e.rearmRetransmit(conn) + } else { + e.rearmTimer(conn) + } +} + +func (e *goEngine) rackUpdateReorderWindow(conn *GoConn, sample *goRateSample) { + if !conn.sackPermitted || !sample.valid { + return + } + if int32(sample.priorDelivered-conn.rackLastDelivered) < 0 { + conn.rackDSACKSeen = false + } + if conn.rackDSACKSeen { + conn.reorderWindowSteps = min(0xff, conn.reorderWindowSteps+1) + conn.rackDSACKSeen = false + conn.rackLastDelivered = conn.delivered + conn.reorderWindowPersist = goRackRecoveryThreshold + } else if conn.reorderWindowPersist == 0 { + conn.reorderWindowSteps = 1 + } +} + +func (e *goEngine) identifyPacketLoss(conn *GoConn, flags goAckFlags) { + entries := conn.scoreboard.entries + if len(entries) == 0 { + return + } + if !conn.sackPermitted { + if (conn.congestionState < goCongestionRecovery && conn.renoSacked >= conn.reordering) || (conn.congestionState == goCongestionRecovery && flags&goAckUnackedAdvanced != 0) { + if entries[0].flags&goDescriptorLost == 0 { + e.markLost(conn, &entries[0]) + } + } + } else { + e.rackMarkLost(conn) + } +} + +func (e *goEngine) timeToRecover(conn *GoConn) bool { + return conn.flight.lostOut != 0 +} + +func (e *goEngine) processLoss(conn *GoConn, flags *goAckFlags, numDupack uint32) uint8 { + unacked := conn.sendUnacked.Load() + sent := conn.sentTail.Load() + recovered := unacked >= conn.highSeq + if *flags&goAckUnackedAdvanced != 0 && e.tryUndoLoss(conn, *flags, false) { + return goRexmitNone + } + if conn.frto { + if *flags&goAckOriginalSacked != 0 && e.tryUndoLoss(conn, *flags, true) { + return goRexmitNone + } + if sent > conn.highSeq { + if *flags&goAckDataSacked != 0 || numDupack > 0 { + conn.frto = false + } + } else if *flags&goAckUnackedAdvanced != 0 && !recovered { + conn.highSeq = sent + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + if conn.hasDataWaiting() && windowEnd > sent { + return goRexmitNew + } + conn.frto = false + } + } + if recovered { + e.tryUndoRecovery(conn, *flags) + return goRexmitNone + } + if !conn.sackPermitted { + if sent > conn.highSeq && numDupack > 0 { + e.addRenoSack(conn, numDupack) + } else if *flags&goAckUnackedAdvanced != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + } + return goRexmitLost +} + +func (e *goEngine) fastretransAlert(conn *GoConn, priorUnacked uint64, numDupack uint32, flags *goAckFlags) uint8 { + unacked := conn.sendUnacked.Load() + if conn.flight.packetsOut == 0 && conn.renoSacked != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + if *flags&goAckReneging != 0 && *flags&goAckUnackedAdvanced != 0 { + delay := max(int64(conn.smoothedRoundTrip)/2, int64(10*time.Millisecond/time.Microsecond)) + conn.retransmitDeadline = e.coarseTime.Load() + delay*int64(time.Microsecond) + e.rearmTimer(conn) + *flags |= goAckKeepTimer + return goRexmitNone + } + if conn.congestionState == goCongestionOpen { + conn.retransmitStamp = 0 + } else if unacked >= conn.highSeq { + switch conn.congestionState { + case goCongestionCWR: + if unacked != conn.highSeq { + e.endWindowReduction(conn) + conn.setCAState(goCongestionOpen) + } + case goCongestionRecovery: + if !conn.sackPermitted { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + if e.tryUndoRecovery(conn, *flags) { + return goRexmitNone + } + e.endWindowReduction(conn) + } + } + switch conn.congestionState { + case goCongestionRecovery: + if *flags&goAckUnackedAdvanced == 0 { + if !conn.sackPermitted { + e.addRenoSack(conn, numDupack) + } + } else if e.tryUndoPartial(conn, *flags, priorUnacked) { + return goRexmitNone + } + if e.tryUndoDSACK(conn) { + e.tryToOpen(conn) + } + e.identifyPacketLoss(conn, *flags) + if conn.congestionState != goCongestionRecovery { + if !e.timeToRecover(conn) { + return goRexmitNone + } + e.enterRecovery(conn) + } + return goRexmitLost + case goCongestionLoss: + rexmit := e.processLoss(conn, flags, numDupack) + e.identifyPacketLoss(conn, *flags) + if conn.congestionState != goCongestionOpen { + return rexmit + } + } + if !conn.sackPermitted { + if *flags&goAckUnackedAdvanced != 0 { + conn.renoSacked = 0 + conn.flight.sackedOut = 0 + } + e.addRenoSack(conn, numDupack) + } + if conn.congestionState <= goCongestionDisorder { + e.tryUndoDSACK(conn) + } + e.identifyPacketLoss(conn, *flags) + if !e.timeToRecover(conn) { + e.tryToOpen(conn) + return goRexmitNone + } + e.enterRecovery(conn) + return goRexmitLost +} + +func (e *goEngine) xmitRecovery(conn *GoConn, rexmit uint8) { + if rexmit == goRexmitNone || rexmit == goRexmitNew { + return + } + e.xmitRetransmitQueue(conn) +} + +func (e *goEngine) xmitRetransmitQueue(conn *GoConn) { + if conn.flight.packetsOut == 0 { + return + } + start := conn.sendUnacked.Load() + sent := 0 + for index := 0; index < len(conn.scoreboard.entries); index++ { + descriptor := &conn.scoreboard.entries[index] + end := descriptor.endOffset + if conn.congestionWindow <= conn.flight.inFlight() || conn.flight.retransmitOut >= conn.flight.lostOut { + return + } + if descriptor.flags&(goDescriptorSacked|goDescriptorRetransmitOut) != 0 || descriptor.flags&goDescriptorLost == 0 { + start = end + continue + } + if conn.congestion.pacing { + stamp := conn.pacingStamp.Load() + if stamp > e.coarseTime.Load()+goWheelTick { + conn.pacingDeadline = stamp - goWheelTick + e.rearmTimer(conn) + return + } + } + if e.transmitRetransmit(conn, start, int(end-start), false) == 0 { + return + } + sent++ + if sent == goRecoveryBurst { + return + } + start = conn.scoreboard.entries[index].endOffset + } +} + +func (e *goEngine) transmitRetransmit(conn *GoConn, offset uint64, length int, probe bool) int { + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + limit := min(conn.transmittedTail.Load(), windowEnd) + if conn.peerWindow == 0 || offset >= limit { + return 0 + } + length = int(min(uint64(length), uint64(conn.effectiveMSS.Load()), limit-offset)) + for _, descriptor := range conn.scoreboard.entries { + if descriptor.endOffset > offset { + length = min(length, int(descriptor.endOffset-offset)) + break + } + } + segment := goSegment{offset: offset, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} + if !e.writeConnControl(conn, &segment) { + return 0 + } + conn.totalRetransmits++ + conn.advancePacing(e.coarseTime.Load(), length) + end := offset + uint64(length) + marked := e.markRetransmitted(conn, offset, end, probe) + if !probe && marked > 0 { + conn.flight.retransmitOut += marked + if conn.inWindowReduction() { + conn.reductionRetransmits += marked + } + if conn.undoRetransmits < 0 { + conn.undoRetransmits = 0 + } + conn.undoRetransmits += int32(marked) + } + if conn.retransmitStamp == 0 { + conn.retransmitStamp = goTimestampAt(e.coarseTime.Load()) + } + return length +} + +func (e *goEngine) markRetransmitted(conn *GoConn, start uint64, end uint64, probe bool) uint32 { + previous := conn.sendUnacked.Load() + stamp := conn.stamp(e.coarseTime.Load()) + marked := uint32(0) + appLimited := conn.appLimited.Load() != 0 + scoreboard := &conn.scoreboard + for index := 0; index < len(scoreboard.entries); index++ { + descriptor := &scoreboard.entries[index] + descriptorEnd := descriptor.endOffset + if previous >= end { + break + } + if descriptorEnd <= start { + previous = descriptorEnd + continue + } + if previous < start { + flags := descriptor.flags + scoreboard.split(index, start) + e.accountSplit(conn, flags) + index++ + descriptor = &scoreboard.entries[index] + } + if end < descriptorEnd { + flags := descriptor.flags + scoreboard.split(index, end) + e.accountSplit(conn, flags) + descriptor = &scoreboard.entries[index] + descriptorEnd = end + } + if descriptor.flags&goDescriptorRetransmitted == 0 { + conn.flight.retransmitted++ + } + descriptor.flags |= goDescriptorRetransmitted + descriptor.flags &^= goDescriptorDropped | goDescriptorNoSample + if !probe && descriptor.flags&goDescriptorRetransmitOut == 0 { + descriptor.flags |= goDescriptorRetransmitOut + marked++ + } + descriptor.sentAt = stamp + descriptor.delivered = conn.delivered + descriptor.deliveredStamp = conn.deliveredStamp + descriptor.firstSentStamp = conn.firstSentStamp.Load() + descriptor.flags |= goDescriptorRated + if appLimited { + descriptor.flags |= goDescriptorAppLimited + } else { + descriptor.flags &^= goDescriptorAppLimited + } + previous = descriptorEnd + } + return marked +} + +func (c *GoConn) publishPermit(ackOffset uint64) { + unacked := c.sendUnacked.Load() + limit := max(c.flight.packetsEnd, unacked) + inFlight := c.flight.inFlight() + if c.congestionWindow > inFlight { + limit += uint64(c.congestionWindow-inFlight) * uint64(c.effectiveMSS.Load()) + } + windowEnd := ackOffset + c.peerWindow + permit := max(min(limit, windowEnd), unacked) + if limit <= windowEnd { + permit |= goPermitWindowBit + } + c.sendPacketPermit.Store(c.packetCreditBase.Load() + c.congestionWindow + c.flight.leftOut() - c.flight.retransmitOut) + c.sendPermit.Store(permit) + c.wakeTransmitter() + c.wakeWriter() +} + +func (c *GoConn) gsoMaxSize(mss uint32) uint32 { + if c.engine.platformIO.transmitSegmentOffload() { + return max(goGSOMaxPayload/mss*mss, mss) + } + return max(0xffff/mss*mss, mss) +} + +func (e *goEngine) updateFrameLimit(conn *GoConn) { + rate := conn.pacingRate.Load() + if rate == 0 { + conn.frameLimit.Store(0) + return + } + bytes := rate >> goPacingShift + shift := conn.roundTripMin.get() >> goTSORoundTripShift + gsoMax := uint64(conn.gsoMaxSize(conn.effectiveMSS.Load())) + if shift < 64 { + bytes += gsoMax >> shift + } + bytes = min(bytes, gsoMax) + minSegments := uint32(goMinTSOSegments) + if conn.congestion.minTSOSegments != nil { + minSegments = conn.congestion.minTSOSegments(conn) + } + mss := uint64(conn.effectiveMSS.Load()) + segments := max(uint32(bytes/mss), minSegments) + conn.frameLimit.Store(uint32(min(uint64(segments)*mss, gsoMax))) +} + +func (e *goEngine) handlePacingRequest(conn *GoConn) { + if conn.dead { + return + } + conn.pacingDeadline = conn.pacingRequest.Load() + e.rearmTimer(conn) +} + +func (e *goEngine) expirePacing(conn *GoConn) { + conn.pacingDeadline = 0 + e.rearmTimer(conn) + if conn.flight.lostOut > conn.flight.retransmitOut && conn.congestionWindow > conn.flight.inFlight() { + e.xmitRetransmitQueue(conn) + conn.publishPermit(conn.sendUnacked.Load()) + return + } + conn.wakeTransmitter() +} + +func (e *goEngine) armProbe(conn *GoConn) { + if conn.probeAttempts >= goProbeAttempts || !conn.hasOutstanding() || !conn.sackPermitted || (conn.congestionState != goCongestionOpen && conn.congestionState != goCongestionCWR) { + conn.probeDeadline = 0 + return + } + timeout := int32(goProbeNoSampleMicros) + if conn.smoothedRoundTrip > 0 { + timeout = max(2*conn.smoothedRoundTrip, goProbeFloorMicros) + } + if conn.sentTail.Load()-conn.sendUnacked.Load() <= 2*uint64(conn.effectiveMSS.Load()) { + timeout += goProbeDelayedAck + } + if timeout >= conn.retransmitTimeout { + conn.probeDeadline = 0 + return + } + conn.probeDeadline = e.coarseTime.Load() + int64(timeout)*int64(time.Microsecond) +} + +func (e *goEngine) expireProbe(conn *GoConn, now int64) { + conn.probeDeadline = 0 + if !conn.hasOutstanding() { + e.rearmTimer(conn) + return + } + e.drainDescriptors(conn) + if conn.probeHighSeq != 0 { + deadline := conn.retransmitDeadline + if len(conn.scoreboard.entries) > 0 { + elapsed := goStampMicros(conn.stamp(now) - conn.scoreboard.entries[0].sentAt) + deadline = now + max(int64(conn.retransmitTimeout)-elapsed, 1)*int64(time.Microsecond) + } + if deadline == 0 { + deadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + } + conn.retransmitDeadline = deadline + e.rearmTimer(conn) + return + } + if conn.flightHeadQueued() { + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + return + } + conn.probeAttempts++ + e.summarizeFlight(conn) + sent := conn.sentTail.Load() + committed := min(sent, conn.transmittedTail.Load()) + unacked := conn.sendUnacked.Load() + pending := conn.bufferedTail.Load() - sent + windowEnd := uint64(max(conn.windowLeft2+int64(conn.peerWindow), 0)) + conn.access.Lock() + writing := conn.writing + conn.access.Unlock() + switch { + case pending > 0 && committed == sent && windowEnd > sent && !writing: + length := min(pending, uint64(conn.effectiveMSS.Load()), windowEnd-sent) + conn.probeRetransmitted = false + conn.probeHighSeq = sent + length + permit := conn.sendPermit.Load() + if packets := conn.dataSegmentsOut.Load(); int32(conn.sendPacketPermit.Load()-packets) <= 0 { + conn.sendPacketPermit.Store(packets + 1) + } + if permit&^goPermitWindowBit < sent+length { + conn.sendPermit.Store(sent + length | permit&goPermitWindowBit) + } + conn.wakeTransmitter() + case committed > unacked: + entries := conn.scoreboard.entries + for len(entries) > 0 && entries[len(entries)-1].endOffset > committed { + entries = entries[:len(entries)-1] + } + start := unacked + end := committed + if len(entries) > 0 { + last := len(entries) - 1 + end = entries[last].endOffset + start = conn.scoreboard.startOf(last, unacked) + if end < committed { + start = end + end = committed + } + } + if end-start > uint64(conn.effectiveMSS.Load()) { + start = end - uint64(conn.effectiveMSS.Load()) + } + if e.transmitRetransmit(conn, start, int(end-start), true) > 0 { + conn.probeRetransmitted = true + conn.probeHighSeq = committed + } + case conn.finSent && !conn.finAcked: + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + if e.writeConnControl(conn, &segment) { + conn.probeRetransmitted = true + conn.probeHighSeq = conn.sendNext() + } + } + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.armProbe(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) expireRetransmit(conn *GoConn, now int64) { + if !conn.hasOutstanding() { + conn.retransmitDeadline = 0 + e.rearmTimer(conn) + return + } + if conn.persistNeeded() { + e.updatePersist(conn) + return + } + conn.probeDeadline = 0 + conn.probeAttempts = 0 + conn.reorderDeadline = 0 + if conn.hasOutstandingData() && conn.sendUnacked.Load() >= conn.transmittedTail.Load() || conn.flightHeadQueued() { + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + return + } + if conn.retransmitAttempts+1 >= goRetransmitLimit && conn.persistDeadline == 0 { + conn.retransmitDeadline = 0 + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: retransmit limit"), goDeathImmediate) + return + } + e.drainDescriptors(conn) + e.enterLoss(conn) + conn.retransmitAttempts++ + unacked := conn.sendUnacked.Load() + if conn.hasOutstandingData() { + length := int(min(conn.sentTail.Load()-unacked, uint64(conn.effectiveMSS.Load()))) + e.transmitRetransmit(conn, unacked, length, false) + } else { + e.retransmitFrom(conn, unacked) + } + conn.retransmitTimeout = min(conn.retransmitTimeout*2, goRTOMaxMicros) + conn.retransmitDeadline = now + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.rearmTimer(conn) + e.updateFrameLimit(conn) + conn.publishPermit(unacked) +} diff --git a/stack_go_tcp_bbr.go b/stack_go_tcp_bbr.go new file mode 100644 index 00000000..91a782a7 --- /dev/null +++ b/stack_go_tcp_bbr.go @@ -0,0 +1,590 @@ +package tun + +import ( + "math/bits" + "math/rand/v2" + "time" +) + +// Mirrors net/ipv4/tcp_bbr.c with HZ = 1000. + +const ( + goBBRBandwidthScale = 24 + goBBRBandwidthUnit = 1 << goBBRBandwidthScale + goBBRScale = 8 + goBBRUnit = 1 << goBBRScale + goBBRCycleLength = 8 +) + +const ( + goBBRModeStartup uint8 = iota + goBBRModeDrain + goBBRModeProbeBandwidth + goBBRModeProbeRoundTrip +) + +const ( + goBBRBandwidthRounds = goBBRCycleLength + 2 + goBBRMinRoundTripWindowMs = 10 * 1000 + goBBRProbeRoundTripModeMs = 200 + goBBRMinTSORate = 1200000 + goBBRPacingMarginPercent = 1 + goBBRHighGain = goBBRUnit*2885/1000 + 1 + goBBRDrainGain = goBBRUnit * 1000 / 2885 + goBBRWindowGain = goBBRUnit * 2 + goBBRCycleRandom = 7 + goBBRWindowMinTarget = 4 + goBBRFullBandwidthThresh = goBBRUnit * 5 / 4 + goBBRFullBandwidthCount = 3 + goBBRLongTermMinRounds = 4 + goBBRLongTermLossThresh = 50 + goBBRLongTermBandwidthRatio = goBBRUnit / 8 + goBBRLongTermBandwidthDiff = 4000 / 8 + goBBRLongTermMaxRounds = 48 + goBBRExtraAckedGain = goBBRUnit + goBBRExtraAckedWindowRounds = 5 + goBBRAckEpochResetThresh = 1 << 20 + goBBRExtraAckedMaxMicros = 100 * 1000 +) + +var goBBRPacingGains = [goBBRCycleLength]uint32{ + goBBRUnit * 5 / 4, + goBBRUnit * 3 / 4, + goBBRUnit, goBBRUnit, goBBRUnit, + goBBRUnit, goBBRUnit, goBBRUnit, +} + +type goBBRState struct { + minRoundTripMicros uint32 + minRoundTripStamp uint32 + probeRoundTripDone uint32 + bandwidth goMinMax + roundTripCount uint32 + nextRoundDelivered uint32 + cycleStamp int64 + mode uint8 + previousCAState uint8 + packetConservation bool + roundStart bool + idleRestart bool + probeRoundTripRoundDone bool + longTermSampling bool + longTermRounds uint32 + longTermUseBandwidth bool + longTermBandwidth uint32 + longTermLastDelivered uint32 + longTermLastStamp int64 + longTermLastLost uint32 + pacingGain uint32 + windowGain uint32 + fullBandwidthReached bool + fullBandwidthCount uint32 + cycleIndex uint32 + seenRoundTrip bool + priorWindow uint32 + fullBandwidth uint32 + ackEpochStamp int64 + extraAcked [2]uint32 + ackEpochAcked uint32 + extraAckedWindowRounds uint32 + extraAckedWindowIndex int +} + +func goBBRJiffies(c *GoConn) uint32 { + return uint32(c.engine.coarseTime.Load() / int64(time.Millisecond)) +} + +func goBBRAfter(a uint32, b uint32) bool { + return int32(a-b) > 0 +} + +func (s *goBBRState) currentBandwidth() uint32 { + if s.longTermUseBandwidth { + return s.longTermBandwidth + } + return s.bandwidth.get() +} + +func (s *goBBRState) extraAckedMax() uint32 { + return max(s.extraAcked[0], s.extraAcked[1]) +} + +func goBBRRateBytesPerSecond(c *GoConn, rate uint64, gain uint32) uint64 { + rate *= uint64(c.effectiveMSS.Load()) + rate *= uint64(gain) + rate >>= goBBRScale + rate *= uint64(time.Second/time.Microsecond) / 100 * (100 - goBBRPacingMarginPercent) + return rate >> goBBRBandwidthScale +} + +func goBBRInitPacingRateFromRoundTrip(c *GoConn, s *goBBRState) { + roundTrip := uint64(time.Millisecond / time.Microsecond) + if c.smoothedRoundTrip > 0 { + roundTrip = uint64(max(c.smoothedRoundTrip, 1)) + s.seenRoundTrip = true + } + bandwidth := uint64(c.congestionWindow) * goBBRBandwidthUnit / roundTrip + c.pacingRate.Store(goBBRRateBytesPerSecond(c, bandwidth, goBBRHighGain)) +} + +func goBBRSetPacingRate(c *GoConn, s *goBBRState, bandwidth uint32, gain uint32) { + rate := goBBRRateBytesPerSecond(c, uint64(bandwidth), gain) + if !s.seenRoundTrip && c.smoothedRoundTrip > 0 { + goBBRInitPacingRateFromRoundTrip(c, s) + } + if s.fullBandwidthReached || rate > c.pacingRate.Load() { + c.pacingRate.Store(rate) + } +} + +func goBBRMinTSOSegments(c *GoConn) uint32 { + if c.pacingRate.Load() < goBBRMinTSORate>>3 { + return 1 + } + return 2 +} + +func goBBRTSOSegmentsGoal(c *GoConn) uint32 { + bytes := min(c.pacingRate.Load()>>goPacingShift, uint64(0xffff-1-goHeaderScratchSize)) + segments := max(uint32(bytes/uint64(c.effectiveMSS.Load())), goBBRMinTSOSegments(c)) + return min(segments, 0x7f) +} + +func (s *goBBRState) saveWindow(c *GoConn) { + if s.previousCAState < goCongestionRecovery && s.mode != goBBRModeProbeRoundTrip { + s.priorWindow = c.congestionWindow + } else { + s.priorWindow = max(s.priorWindow, c.congestionWindow) + } +} + +func goBBRTxStart(c *GoConn, _ int64) { + s := c.congestionPrivate.(*goBBRState) + if c.appLimited.Load() != 0 { + s.idleRestart = true + s.ackEpochStamp = c.engine.coarseTime.Load() + s.ackEpochAcked = 0 + switch s.mode { + case goBBRModeProbeBandwidth: + goBBRSetPacingRate(c, s, s.currentBandwidth(), goBBRUnit) + case goBBRModeProbeRoundTrip: + s.checkProbeRoundTripDone(c) + } + } +} + +func (s *goBBRState) bandwidthDelayProduct(bandwidth uint32, gain uint32) uint32 { + if s.minRoundTripMicros == ^uint32(0) { + return goInitialWindow + } + w := uint64(bandwidth) * uint64(s.minRoundTripMicros) + return uint32((((w * uint64(gain)) >> goBBRScale) + goBBRBandwidthUnit - 1) / goBBRBandwidthUnit) +} + +func (s *goBBRState) quantizationBudget(c *GoConn, window uint32) uint32 { + window += 3 * goBBRTSOSegmentsGoal(c) + window = (window + 1) &^ 1 + if s.mode == goBBRModeProbeBandwidth && s.cycleIndex == 0 { + window += 2 + } + return window +} + +func (s *goBBRState) inflight(c *GoConn, bandwidth uint32, gain uint32) uint32 { + return s.quantizationBudget(c, s.bandwidthDelayProduct(bandwidth, gain)) +} + +func (s *goBBRState) packetsInNetAtDeparture(c *GoConn, inflightNow uint32) uint32 { + now := c.engine.coarseTime.Load() + departure := max(c.pacingStamp.Load(), now) + intervalMicros := uint64(departure-now) / uint64(time.Microsecond) + intervalDelivered := uint32(uint64(s.currentBandwidth()) * intervalMicros >> goBBRBandwidthScale) + inflightAtDeparture := inflightNow + if s.pacingGain > goBBRUnit { + inflightAtDeparture += goBBRTSOSegmentsGoal(c) + } + if intervalDelivered >= inflightAtDeparture { + return 0 + } + return inflightAtDeparture - intervalDelivered +} + +func (s *goBBRState) ackAggregationWindow() uint32 { + if goBBRExtraAckedGain == 0 || !s.fullBandwidthReached { + return 0 + } + maxAggregation := uint32(uint64(s.currentBandwidth()) * goBBRExtraAckedMaxMicros / goBBRBandwidthUnit) + aggregation := (goBBRExtraAckedGain * s.extraAckedMax()) >> goBBRScale + return min(aggregation, maxAggregation) +} + +func (s *goBBRState) setWindowToRecoverOrRestore(c *GoConn, sample *goRateSample, acked uint32) (uint32, bool) { + previousState := s.previousCAState + state := c.congestionState + window := c.congestionWindow + inFlight := c.flight.inFlight() + if sample.losses > 0 { + window = uint32(max(int64(window)-int64(sample.losses), 1)) + } + if state == goCongestionRecovery && previousState != goCongestionRecovery { + s.packetConservation = true + s.nextRoundDelivered = c.delivered + window = inFlight + acked + } else if previousState >= goCongestionRecovery && state < goCongestionRecovery { + window = max(window, s.priorWindow) + s.packetConservation = false + } + s.previousCAState = state + if s.packetConservation { + return max(window, inFlight+acked), true + } + return window, false +} + +func (s *goBBRState) setWindow(c *GoConn, sample *goRateSample, acked uint32, bandwidth uint32, gain uint32) { + window := c.congestionWindow + if acked != 0 { + var conserving bool + window, conserving = s.setWindowToRecoverOrRestore(c, sample, acked) + if !conserving { + target := s.bandwidthDelayProduct(bandwidth, gain) + target += s.ackAggregationWindow() + target = s.quantizationBudget(c, target) + if s.fullBandwidthReached { + window = min(window+acked, target) + } else if window < target || c.delivered < goInitialWindow { + window += acked + } + window = max(window, goBBRWindowMinTarget) + } + } + c.setCongestionWindow(window) + if s.mode == goBBRModeProbeRoundTrip { + c.setCongestionWindow(min(c.congestionWindow, goBBRWindowMinTarget)) + } +} + +func (s *goBBRState) isNextCyclePhase(c *GoConn, sample *goRateSample) bool { + fullLength := c.deliveredTime-s.cycleStamp > int64(s.minRoundTripMicros)*int64(time.Microsecond) + if s.pacingGain == goBBRUnit { + return fullLength + } + inflight := s.packetsInNetAtDeparture(c, sample.priorInFlight) + bandwidth := s.bandwidth.get() + if s.pacingGain > goBBRUnit { + return fullLength && (sample.losses > 0 || inflight >= s.inflight(c, bandwidth, s.pacingGain)) + } + return fullLength || inflight <= s.inflight(c, bandwidth, goBBRUnit) +} + +func (s *goBBRState) advanceCyclePhase(c *GoConn) { + s.cycleIndex = (s.cycleIndex + 1) & (goBBRCycleLength - 1) + s.cycleStamp = c.deliveredTime +} + +func (s *goBBRState) updateCyclePhase(c *GoConn, sample *goRateSample) { + if s.mode == goBBRModeProbeBandwidth && s.isNextCyclePhase(c, sample) { + s.advanceCyclePhase(c) + } +} + +func (s *goBBRState) resetProbeBandwidthMode(c *GoConn) { + s.mode = goBBRModeProbeBandwidth + s.cycleIndex = goBBRCycleLength - 1 - rand.Uint32N(goBBRCycleRandom) + s.advanceCyclePhase(c) +} + +func (s *goBBRState) resetMode(c *GoConn) { + if !s.fullBandwidthReached { + s.mode = goBBRModeStartup + } else { + s.resetProbeBandwidthMode(c) + } +} + +func (s *goBBRState) resetLongTermInterval(c *GoConn) { + s.longTermLastStamp = c.deliveredTime + s.longTermLastDelivered = c.delivered + s.longTermLastLost = c.lost + s.longTermRounds = 0 +} + +func (s *goBBRState) resetLongTermSampling(c *GoConn) { + s.longTermBandwidth = 0 + s.longTermUseBandwidth = false + s.longTermSampling = false + s.resetLongTermInterval(c) +} + +func (s *goBBRState) longTermIntervalDone(c *GoConn, bandwidth uint32) { + if s.longTermBandwidth != 0 { + var diff uint32 + if bandwidth > s.longTermBandwidth { + diff = bandwidth - s.longTermBandwidth + } else { + diff = s.longTermBandwidth - bandwidth + } + if uint64(diff)*goBBRUnit <= uint64(goBBRLongTermBandwidthRatio)*uint64(s.longTermBandwidth) || + goBBRRateBytesPerSecond(c, uint64(diff), goBBRUnit) <= goBBRLongTermBandwidthDiff { + s.longTermBandwidth = (bandwidth + s.longTermBandwidth) >> 1 + s.longTermUseBandwidth = true + s.pacingGain = goBBRUnit + s.longTermRounds = 0 + return + } + } + s.longTermBandwidth = bandwidth + s.resetLongTermInterval(c) +} + +func (s *goBBRState) longTermSample(c *GoConn, sample *goRateSample) { + if s.longTermUseBandwidth { + if s.mode == goBBRModeProbeBandwidth && s.roundStart { + s.longTermRounds++ + if s.longTermRounds >= goBBRLongTermMaxRounds { + s.resetLongTermSampling(c) + s.resetProbeBandwidthMode(c) + } + } + return + } + if !s.longTermSampling { + if sample.losses == 0 { + return + } + s.resetLongTermInterval(c) + s.longTermSampling = true + } + if sample.appLimited { + s.resetLongTermSampling(c) + return + } + if s.roundStart { + s.longTermRounds++ + } + if s.longTermRounds < goBBRLongTermMinRounds { + return + } + if s.longTermRounds > 4*goBBRLongTermMinRounds { + s.resetLongTermSampling(c) + return + } + if sample.losses == 0 { + return + } + lost := c.lost - s.longTermLastLost + delivered := c.delivered - s.longTermLastDelivered + if delivered == 0 || uint64(lost)<= int64(^uint32(0))/1000 { + s.resetLongTermSampling(c) + return + } + bandwidth := uint64(delivered) * goBBRBandwidthUnit / (uint64(t) * 1000) + s.longTermIntervalDone(c, uint32(bandwidth)) +} + +func (s *goBBRState) updateBandwidth(c *GoConn, sample *goRateSample) { + s.roundStart = false + if sample.delivered < 0 || sample.intervalMicros <= 0 { + return + } + if int32(sample.priorDelivered-s.nextRoundDelivered) >= 0 { + s.nextRoundDelivered = c.delivered + s.roundTripCount++ + s.roundStart = true + s.packetConservation = false + } + s.longTermSample(c, sample) + bandwidth := uint64(sample.delivered) * goBBRBandwidthUnit / uint64(sample.intervalMicros) + if !sample.appLimited || bandwidth >= uint64(s.bandwidth.get()) { + s.bandwidth.runningMax(goBBRBandwidthRounds, s.roundTripCount, uint32(min(bandwidth, uint64(^uint32(0))))) + } +} + +func (s *goBBRState) updateAckAggregation(c *GoConn, sample *goRateSample) { + if goBBRExtraAckedGain == 0 || sample.ackedSacked == 0 || sample.delivered < 0 || sample.intervalMicros <= 0 { + return + } + if s.roundStart { + s.extraAckedWindowRounds = min(0x1f, s.extraAckedWindowRounds+1) + if s.extraAckedWindowRounds >= goBBRExtraAckedWindowRounds { + s.extraAckedWindowRounds = 0 + s.extraAckedWindowIndex = 1 - s.extraAckedWindowIndex + s.extraAcked[s.extraAckedWindowIndex] = 0 + } + } + epochMicros := (c.deliveredTime - s.ackEpochStamp) / int64(time.Microsecond) + high, low := bits.Mul64(uint64(s.currentBandwidth()), uint64(max(epochMicros, 0))) + expectedAcked := uint32(goBBRAckEpochResetThresh) + if high == 0 && low < uint64(goBBRAckEpochResetThresh)*goBBRBandwidthUnit { + expectedAcked = uint32(low / goBBRBandwidthUnit) + } + if s.ackEpochAcked <= expectedAcked || s.ackEpochAcked+sample.ackedSacked >= goBBRAckEpochResetThresh { + s.ackEpochAcked = 0 + s.ackEpochStamp = c.deliveredTime + expectedAcked = 0 + } + s.ackEpochAcked = min(0xfffff, s.ackEpochAcked+sample.ackedSacked) + extraAcked := min(s.ackEpochAcked-expectedAcked, c.congestionWindow) + if extraAcked > s.extraAcked[s.extraAckedWindowIndex] { + s.extraAcked[s.extraAckedWindowIndex] = extraAcked + } +} + +func (s *goBBRState) checkFullBandwidthReached(sample *goRateSample) { + if s.fullBandwidthReached || !s.roundStart || sample.appLimited { + return + } + threshold := uint32(uint64(s.fullBandwidth) * goBBRFullBandwidthThresh >> goBBRScale) + if s.bandwidth.get() >= threshold { + s.fullBandwidth = s.bandwidth.get() + s.fullBandwidthCount = 0 + return + } + s.fullBandwidthCount++ + s.fullBandwidthReached = s.fullBandwidthCount >= goBBRFullBandwidthCount +} + +func (s *goBBRState) checkDrain(c *GoConn) { + if s.mode == goBBRModeStartup && s.fullBandwidthReached { + s.mode = goBBRModeDrain + c.slowStartThreshold = s.inflight(c, s.bandwidth.get(), goBBRUnit) + } + if s.mode == goBBRModeDrain && s.packetsInNetAtDeparture(c, c.flight.inFlight()) <= s.inflight(c, s.bandwidth.get(), goBBRUnit) { + s.resetProbeBandwidthMode(c) + } +} + +func (s *goBBRState) checkProbeRoundTripDone(c *GoConn) { + if s.probeRoundTripDone == 0 || !goBBRAfter(goBBRJiffies(c), s.probeRoundTripDone) { + return + } + s.minRoundTripStamp = goBBRJiffies(c) + c.setCongestionWindow(max(c.congestionWindow, s.priorWindow)) + s.resetMode(c) +} + +func (s *goBBRState) updateMinRoundTrip(c *GoConn, sample *goRateSample) { + now := goBBRJiffies(c) + filterExpired := goBBRAfter(now, s.minRoundTripStamp+goBBRMinRoundTripWindowMs) + if sample.roundTripMicros >= 0 && (uint32(sample.roundTripMicros) < s.minRoundTripMicros || (filterExpired && !sample.ackDelayed)) { + s.minRoundTripMicros = uint32(sample.roundTripMicros) + s.minRoundTripStamp = now + } + if goBBRProbeRoundTripModeMs > 0 && filterExpired && !s.idleRestart && s.mode != goBBRModeProbeRoundTrip { + s.mode = goBBRModeProbeRoundTrip + s.saveWindow(c) + s.probeRoundTripDone = 0 + } + if s.mode == goBBRModeProbeRoundTrip { + inFlight := c.flight.inFlight() + c.appLimited.Store(max(c.delivered+inFlight, 1)) + if s.probeRoundTripDone == 0 && inFlight <= goBBRWindowMinTarget { + s.probeRoundTripDone = now + goBBRProbeRoundTripModeMs + s.probeRoundTripRoundDone = false + s.nextRoundDelivered = c.delivered + } else if s.probeRoundTripDone != 0 { + if s.roundStart { + s.probeRoundTripRoundDone = true + } + if s.probeRoundTripRoundDone { + s.checkProbeRoundTripDone(c) + } + } + } + if sample.delivered > 0 { + s.idleRestart = false + } +} + +func (s *goBBRState) updateGains() { + switch s.mode { + case goBBRModeStartup: + s.pacingGain = goBBRHighGain + s.windowGain = goBBRHighGain + case goBBRModeDrain: + s.pacingGain = goBBRDrainGain + s.windowGain = goBBRHighGain + case goBBRModeProbeBandwidth: + if s.longTermUseBandwidth { + s.pacingGain = goBBRUnit + } else { + s.pacingGain = goBBRPacingGains[s.cycleIndex] + } + s.windowGain = goBBRWindowGain + case goBBRModeProbeRoundTrip: + s.pacingGain = goBBRUnit + s.windowGain = goBBRUnit + } +} + +func goBBRMain(c *GoConn, _ goAckFlags, sample *goRateSample) { + s := c.congestionPrivate.(*goBBRState) + s.updateBandwidth(c, sample) + s.updateAckAggregation(c, sample) + s.updateCyclePhase(c, sample) + s.checkFullBandwidthReached(sample) + s.checkDrain(c) + s.updateMinRoundTrip(c, sample) + s.updateGains() + bandwidth := s.currentBandwidth() + goBBRSetPacingRate(c, s, bandwidth, s.pacingGain) + s.setWindow(c, sample, sample.ackedSacked, bandwidth, s.windowGain) +} + +func goBBRInit(c *GoConn) { + s := new(goBBRState) + c.congestionPrivate = s + c.slowStartThreshold = goInfiniteSlowStartThreshold + s.nextRoundDelivered = c.delivered + s.previousCAState = goCongestionOpen + s.minRoundTripMicros = c.roundTripMin.get() + s.minRoundTripStamp = goBBRJiffies(c) + s.bandwidth.reset(s.roundTripCount, 0) + goBBRInitPacingRateFromRoundTrip(c, s) + s.resetLongTermSampling(c) + s.mode = goBBRModeStartup + s.ackEpochStamp = c.engine.coarseTime.Load() +} + +func goBBRUndoWindow(c *GoConn) uint32 { + s := c.congestionPrivate.(*goBBRState) + s.fullBandwidth = 0 + s.fullBandwidthCount = 0 + s.resetLongTermSampling(c) + return c.congestionWindow +} + +func goBBRSlowStartThreshold(c *GoConn) uint32 { + s := c.congestionPrivate.(*goBBRState) + s.saveWindow(c) + return c.slowStartThreshold +} + +func goBBRSetState(c *GoConn, newState uint8) { + if newState == goCongestionLoss { + s := c.congestionPrivate.(*goBBRState) + s.previousCAState = goCongestionLoss + s.fullBandwidth = 0 + s.roundStart = true + s.longTermSample(c, &goRateSample{losses: 1}) + } +} + +var goBBROps = goCongestionOps{ + name: "bbr", + init: goBBRInit, + congControl: goBBRMain, + undoWindow: goBBRUndoWindow, + windowEventTxStart: goBBRTxStart, + slowStartThreshold: goBBRSlowStartThreshold, + minTSOSegments: goBBRMinTSOSegments, + setState: goBBRSetState, + pacing: true, +} diff --git a/stack_go_tcp_cong.go b/stack_go_tcp_cong.go new file mode 100644 index 00000000..cf5fbde3 --- /dev/null +++ b/stack_go_tcp_cong.go @@ -0,0 +1,691 @@ +package tun + +import ( + "time" + + E "github.com/sagernet/sing/common/exceptions" +) + +// Mirrors net/ipv4/tcp_cong.c and the congestion control core of net/ipv4/tcp_input.c. + +const ( + goCongestionOpen uint8 = iota + goCongestionDisorder + goCongestionCWR + goCongestionRecovery + goCongestionLoss +) + +const ( + goCongestionEventWindowRestart uint8 = iota + goCongestionEventCompleteCWR + goCongestionEventLoss +) + +const ( + goInfiniteSlowStartThreshold = 0x7fffffff + goInitialReordering = 3 + goMaxReordering = 300 + goMinRoundTripWindow = 300 * time.Second + goPacingSlowStartRatio = 200 + goPacingAvoidanceRatio = 120 + goMinTSOSegments = 2 + goTSORoundTripShift = 9 + goPacingShift = 10 + goPacingUnpacedSegments = 10 + goRackRecoveryThreshold = 16 + goRackTimeoutMinMicros = 2000 +) + +type goAckFlags uint32 + +const ( + goAckData goAckFlags = 1 << iota + goAckWindowUpdate + goAckDataAcked + goAckRetransmittedAcked + goAckDataSacked + goAckOriginalSacked + goAckUnackedAdvanced + goAckDSACK + goAckMaybeDelayed + goAckDSACKProbe + goAckReneging + goAckKeepTimer +) + +const ( + goAckNotDuplicate = goAckData | goAckWindowUpdate | goAckDataAcked + goAckAlert = goAckDataSacked | goAckDSACK + goAckForwardProgress = goAckDataAcked | goAckDataSacked +) + +const ( + goRexmitNone uint8 = iota + goRexmitLost + goRexmitNew +) + +type goAckSample struct { + packetsAcked uint32 + roundTripMicros int64 + inFlightBytes uint64 +} + +type goRateSample struct { + valid bool + priorStamp int32 + priorDelivered uint32 + delivered int64 + intervalMicros int64 + sendIntervalMicros int64 + receiveIntervalMicros int64 + roundTripMicros int64 + losses uint32 + ackedSacked uint32 + priorInFlight uint32 + lastEndOffset uint64 + appLimited bool + retransmitted bool + ackDelayed bool +} + +type goFlightSummary struct { + packetsEnd uint64 + packetsOut uint32 + sackedOut uint32 + lostOut uint32 + retransmitOut uint32 + retransmitted uint32 +} + +func (s *goFlightSummary) inFlight() uint32 { + return s.packetsOut - s.sackedOut - s.lostOut + s.retransmitOut +} + +func (s *goFlightSummary) leftOut() uint32 { + return s.sackedOut + s.lostOut +} + +type goCongestionOps struct { + name string + init func(conn *GoConn) + release func(conn *GoConn) + slowStartThreshold func(conn *GoConn) uint32 + congAvoid func(conn *GoConn, acked uint32) + congControl func(conn *GoConn, flags goAckFlags, sample *goRateSample) + undoWindow func(conn *GoConn) uint32 + setState func(conn *GoConn, state uint8) + windowEvent func(conn *GoConn, event uint8) + windowEventTxStart func(conn *GoConn, idleNanos int64) + packetsAcked func(conn *GoConn, sample *goAckSample) + minTSOSegments func(conn *GoConn) uint32 + pacing bool +} + +var goCongestionRegistry = map[string]*goCongestionOps{ + goRenoOps.name: &goRenoOps, + goCubicOps.name: &goCubicOps, + goBBROps.name: &goBBROps, +} + +func goLookupCongestionControl(name string) (*goCongestionOps, error) { + if name == "" { + name = goCubicOps.name + } + ops, found := goCongestionRegistry[name] + if !found { + return nil, E.New("go: unknown congestion control: ", name) + } + return ops, nil +} + +func (c *GoConn) initCongestionControl(ops *goCongestionOps) { + c.congestion = ops + c.congestionWindow = goInitialWindow + c.slowStartThreshold = goInfiniteSlowStartThreshold + c.congestionClamp = max(uint32(goMaxCongestionSize/uint64(c.effectiveMSS.Load())), 2) + c.reordering = goInitialReordering + c.reorderWindowSteps = 1 + c.deliveredTime = c.engine.coarseTime.Load() + c.undoRetransmits = -1 + c.priorSlowStartThreshold = 0 + c.roundTripMin.reset(0, ^uint32(0)) + if ops.init != nil { + ops.init(c) + } +} + +func (c *GoConn) releaseCongestionControl() { + if c.congestion != nil && c.congestion.release != nil { + c.congestion.release(c) + } + c.congestionPrivate = nil +} + +func (c *GoConn) setCAState(state uint8) { + if c.congestion.setState != nil { + c.congestion.setState(c, state) + } + c.congestionState = state +} + +func (c *GoConn) congestionEvent(event uint8) { + if c.congestion.windowEvent != nil { + c.congestion.windowEvent(c, event) + } +} + +func (c *GoConn) inSlowStart() bool { + return c.congestionWindow < c.slowStartThreshold +} + +func (c *GoConn) inWindowReduction() bool { + return c.congestionState == goCongestionCWR || c.congestionState == goCongestionRecovery +} + +func (c *GoConn) currentSlowStartThreshold() uint32 { + if c.inWindowReduction() { + return c.slowStartThreshold + } + return max(c.slowStartThreshold, c.congestionWindow>>1+c.congestionWindow>>2) +} + +func (c *GoConn) setCongestionWindow(window uint32) { + c.congestionWindow = min(max(window, 1), c.congestionClamp) +} + +func (c *GoConn) isWindowLimited() bool { + if c.windowLimited { + return true + } + if c.inSlowStart() { + return c.congestionWindow < 2*c.maxPacketsOut + } + return false +} + +func (c *GoConn) slowStart(acked uint32) uint32 { + window := min(c.congestionWindow+acked, c.slowStartThreshold) + acked -= window - c.congestionWindow + c.setCongestionWindow(window) + return acked +} + +func (c *GoConn) congAvoidAI(w uint32, acked uint32) { + if c.congestionWindowCount >= w { + c.congestionWindowCount = 0 + c.setCongestionWindow(c.congestionWindow + 1) + } + c.congestionWindowCount += acked + if c.congestionWindowCount >= w { + delta := c.congestionWindowCount / w + c.congestionWindowCount -= delta * w + c.setCongestionWindow(c.congestionWindow + delta) + } +} + +func goRenoCongAvoid(c *GoConn, acked uint32) { + if !c.isWindowLimited() { + return + } + if c.inSlowStart() { + acked = c.slowStart(acked) + if acked == 0 { + return + } + } + c.congAvoidAI(c.congestionWindow, acked) +} + +func goRenoSlowStartThreshold(c *GoConn) uint32 { + return max(c.congestionWindow>>1, 2) +} + +func goRenoUndoWindow(c *GoConn) uint32 { + return max(c.congestionWindow, c.priorCongestionWindow) +} + +var goRenoOps = goCongestionOps{ + name: "reno", + slowStartThreshold: goRenoSlowStartThreshold, + congAvoid: goRenoCongAvoid, + undoWindow: goRenoUndoWindow, +} + +func (e *goEngine) initUndo(conn *GoConn) { + conn.undoMarker = conn.sendUnacked.Load() + conn.undoRetransmits = int32(conn.flight.retransmitOut) + if conn.probeHighSeq != 0 && conn.probeRetransmitted { + conn.undoRetransmits++ + } + if conn.undoRetransmits == 0 { + conn.undoRetransmits = -1 + } +} + +func (e *goEngine) initWindowReduction(conn *GoConn) { + conn.highSeq = conn.sentTail.Load() + conn.probeHighSeq = 0 + conn.congestionWindowCount = 0 + conn.priorCongestionWindow = conn.congestionWindow + conn.reductionDelivered = 0 + conn.reductionRetransmits = 0 + conn.reductionSegmentsOut = conn.dataSegmentsOut.Load() + conn.slowStartThreshold = conn.congestion.slowStartThreshold(conn) +} + +func (c *GoConn) reductionOut() uint32 { + return c.reductionRetransmits + c.dataSegmentsOut.Load() - c.reductionSegmentsOut +} + +func (e *goEngine) reduceWindow(conn *GoConn, newlyAckedSacked uint32, newlyLost uint32, flags goAckFlags) { + if newlyAckedSacked == 0 || conn.priorCongestionWindow == 0 { + return + } + inFlight := conn.flight.inFlight() + conn.reductionDelivered += newlyAckedSacked + out := conn.reductionOut() + var sendCount int64 + delta := int64(conn.slowStartThreshold) - int64(inFlight) + if delta < 0 { + dividend := uint64(conn.slowStartThreshold)*uint64(conn.reductionDelivered) + uint64(conn.priorCongestionWindow) - 1 + sendCount = int64(dividend/uint64(conn.priorCongestionWindow)) - int64(out) + } else { + sendCount = max(int64(conn.reductionDelivered)-int64(out), int64(newlyAckedSacked)) + if flags&goAckUnackedAdvanced != 0 && newlyLost == 0 { + sendCount++ + } + sendCount = min(delta, sendCount) + } + if out == 0 { + sendCount = max(sendCount, 1) + } else { + sendCount = max(sendCount, 0) + } + conn.setCongestionWindow(inFlight + uint32(sendCount)) +} + +func (e *goEngine) endWindowReduction(conn *GoConn) { + if conn.congestion.congControl != nil { + return + } + if conn.slowStartThreshold < goInfiniteSlowStartThreshold && (conn.congestionState == goCongestionCWR || conn.undoMarker != 0) { + conn.setCongestionWindow(conn.slowStartThreshold) + conn.congestionWindowStamp = e.coarseTime.Load() + } + conn.congestionEvent(goCongestionEventCompleteCWR) +} + +func (e *goEngine) enterCWR(conn *GoConn) { + conn.priorSlowStartThreshold = 0 + if conn.congestionState < goCongestionCWR { + conn.undoMarker = 0 + e.initWindowReduction(conn) + conn.setCAState(goCongestionCWR) + } +} + +func (e *goEngine) enterRecovery(conn *GoConn) { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + conn.priorSlowStartThreshold = 0 + e.initUndo(conn) + if !conn.inWindowReduction() { + conn.priorSlowStartThreshold = conn.currentSlowStartThreshold() + e.initWindowReduction(conn) + } + conn.setCAState(goCongestionRecovery) +} + +func (e *goEngine) enterLoss(conn *GoConn) { + newRecovery := conn.congestionState < goCongestionRecovery + unacked := conn.sendUnacked.Load() + sent := conn.sentTail.Load() + reneging := len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorSacked != 0 + if reneging { + conn.highestSacked = unacked + conn.sackReneging = true + } else if !conn.sackPermitted { + conn.renoSacked = 0 + } + now := e.coarseTime.Load() + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if reneging { + descriptor.flags &^= goDescriptorSacked + } else if index != 0 && e.rackDescriptorTimeout(conn, descriptor, 0, now) > 0 { + continue + } + e.markLost(conn, descriptor) + } + e.summarizeFlight(conn) + if conn.congestionState <= goCongestionDisorder || unacked >= conn.highSeq || (conn.congestionState == goCongestionLoss && conn.retransmitAttempts == 0) { + conn.priorSlowStartThreshold = conn.currentSlowStartThreshold() + conn.priorCongestionWindow = conn.congestionWindow + conn.slowStartThreshold = conn.congestion.slowStartThreshold(conn) + conn.congestionEvent(goCongestionEventLoss) + e.initUndo(conn) + } + conn.setCongestionWindow(conn.flight.inFlight() + 1) + conn.congestionWindowCount = 0 + conn.congestionWindowStamp = e.coarseTime.Load() + if conn.congestionState <= goCongestionDisorder && conn.flight.sackedOut >= goInitialReordering { + conn.reordering = min(conn.reordering, goInitialReordering) + } + conn.setCAState(goCongestionLoss) + conn.highSeq = sent + conn.probeHighSeq = 0 + conn.frto = conn.sackPermitted && (newRecovery || conn.retransmitAttempts > 0) +} + +func (e *goEngine) anyRetransmitDone(conn *GoConn) bool { + if conn.flight.retransmitOut > 0 { + return true + } + return len(conn.scoreboard.entries) > 0 && conn.scoreboard.entries[0].flags&goDescriptorRetransmitted != 0 +} + +func (c *GoConn) packetDelayed(flags goAckFlags) bool { + if c.retransmitStamp != 0 { + return c.echoedTimestampBefore(c.retransmitStamp) + } + if !c.sackPermitted && c.sendUnacked.Load() >= c.highSeq { + return false + } + return true +} + +func (c *GoConn) echoedTimestampBefore(stamp uint32) bool { + return c.timestampsEnabled && c.lastTimestampEcho != 0 && int32(c.lastTimestampEcho-stamp) < 0 +} + +func (c *GoConn) mayUndo(flags goAckFlags) bool { + return c.undoMarker != 0 && (c.undoRetransmits == 0 || c.packetDelayed(flags)) +} + +func (e *goEngine) undoWindowReduction(conn *GoConn, unmarkLoss bool) { + if unmarkLoss { + for index := range conn.scoreboard.entries { + conn.scoreboard.entries[index].flags &^= goDescriptorLost + } + e.summarizeFlight(conn) + } + if conn.priorSlowStartThreshold != 0 { + conn.setCongestionWindow(conn.congestion.undoWindow(conn)) + if conn.priorSlowStartThreshold > conn.slowStartThreshold { + conn.slowStartThreshold = conn.priorSlowStartThreshold + } + } + conn.congestionWindowStamp = e.coarseTime.Load() + conn.undoMarker = 0 + conn.rackAdvanced = true +} + +func (e *goEngine) nonSackPreventingReopen(conn *GoConn) bool { + if !conn.sackPermitted && conn.sendUnacked.Load() == conn.highSeq { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + return true + } + return false +} + +func (e *goEngine) tryUndoRecovery(conn *GoConn, flags goAckFlags) bool { + if conn.mayUndo(flags) { + e.undoWindowReduction(conn, false) + } else if conn.reorderWindowPersist > 0 { + conn.reorderWindowPersist-- + } + if e.nonSackPreventingReopen(conn) { + return true + } + conn.setCAState(goCongestionOpen) + conn.sackReneging = false + return false +} + +func (e *goEngine) tryUndoDSACK(conn *GoConn) bool { + if conn.undoMarker != 0 && conn.undoRetransmits == 0 { + conn.reorderWindowPersist = min(goRackRecoveryThreshold, conn.reorderWindowPersist+1) + e.undoWindowReduction(conn, false) + return true + } + return false +} + +func (e *goEngine) tryUndoLoss(conn *GoConn, flags goAckFlags, frtoUndo bool) bool { + if !frtoUndo && !conn.mayUndo(flags) { + return false + } + e.undoWindowReduction(conn, true) + conn.retransmitAttempts = 0 + if e.nonSackPreventingReopen(conn) { + return true + } + if frtoUndo || conn.sackPermitted { + conn.setCAState(goCongestionOpen) + conn.sackReneging = false + } + return true +} + +func (e *goEngine) tryUndoPartial(conn *GoConn, flags goAckFlags, priorUnacked uint64) bool { + if conn.undoMarker == 0 || !conn.packetDelayed(flags) { + return false + } + e.checkSackReordering(conn, priorUnacked) + if conn.flight.retransmitOut > 0 { + return true + } + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + e.undoWindowReduction(conn, true) + e.tryKeepOpen(conn) + return false +} + +func (e *goEngine) tryKeepOpen(conn *GoConn) { + state := goCongestionOpen + if conn.flight.leftOut() != 0 || e.anyRetransmitDone(conn) { + state = goCongestionDisorder + } + if conn.congestionState != state { + conn.setCAState(state) + conn.highSeq = conn.sentTail.Load() + } +} + +func (e *goEngine) tryToOpen(conn *GoConn) { + if !e.anyRetransmitDone(conn) { + conn.retransmitStamp = 0 + } + if conn.congestionState != goCongestionCWR { + e.tryKeepOpen(conn) + } +} + +func (e *goEngine) checkSackReordering(conn *GoConn, lowOffset uint64) { + fack := conn.highestSacked + if lowOffset >= fack { + return + } + metric := fack - lowOffset + mss := uint64(conn.effectiveMSS.Load()) + if metric > uint64(conn.reordering)*mss { + conn.reordering = uint32(min((metric+mss-1)/mss, goMaxReordering)) + } + conn.reorderingSeen++ +} + +func (e *goEngine) mayRaiseWindow(conn *GoConn, flags goAckFlags) bool { + if conn.reordering > goInitialReordering { + return flags&goAckForwardProgress != 0 + } + return flags&goAckDataAcked != 0 +} + +func (e *goEngine) congestionControl(conn *GoConn, ackedSacked uint32, flags goAckFlags, sample *goRateSample) { + if conn.congestion.congControl != nil { + conn.congestion.congControl(conn, flags, sample) + return + } + if conn.inWindowReduction() { + e.reduceWindow(conn, ackedSacked, sample.losses, flags) + } else if e.mayRaiseWindow(conn, flags) { + conn.congestion.congAvoid(conn, ackedSacked) + conn.congestionWindowStamp = e.coarseTime.Load() + } + e.updatePacingRate(conn) +} + +func (e *goEngine) updatePacingRate(conn *GoConn) { + rate := uint64(conn.effectiveMSS.Load()) * (uint64(time.Second/time.Microsecond) / 100) + if conn.congestionWindow < conn.slowStartThreshold/2 { + rate *= goPacingSlowStartRatio + } else { + rate *= goPacingAvoidanceRatio + } + rate *= uint64(max(conn.congestionWindow, conn.flight.packetsOut)) + if conn.smoothedRoundTrip > 0 { + rate /= uint64(conn.smoothedRoundTrip) + } + conn.pacingRate.Store(rate) +} + +func (e *goEngine) validateWindow(conn *GoConn) { + limited := conn.windowLimitedSince.Swap(false) + peak := conn.peakFlight.Swap(0) + if peak == 0 && !limited { + return + } + peakPackets := uint32(peak) + if conn.sendUnacked.Load() >= conn.windowUsageSeq || limited || (!conn.windowLimited && peakPackets > conn.maxPacketsOut) { + conn.windowLimited = limited + conn.maxPacketsOut = peakPackets + conn.windowUsageSeq = conn.sentTail.Load() + } + now := e.coarseTime.Load() + if conn.isWindowLimited() { + conn.congestionWindowUsed = 0 + conn.congestionWindowStamp = now + return + } + if peakPackets > conn.congestionWindowUsed { + conn.congestionWindowUsed = peakPackets + } + if conn.congestion.congControl == nil && now-conn.congestionWindowStamp >= int64(conn.retransmitTimeout)*int64(time.Microsecond) { + e.applicationLimitedWindow(conn, now) + } +} + +func (e *goEngine) armIdleRestart(conn *GoConn) { + if conn.congestion.congControl != nil || conn.congestionState != goCongestionOpen || conn.congestionWindow <= goInitialWindow { + conn.idleDeadline = 0 + return + } + conn.idleDeadline = e.coarseTime.Load() + int64(conn.retransmitTimeout)*int64(time.Microsecond) +} + +func (e *goEngine) expireIdleRestart(conn *GoConn, now int64) { + conn.idleDeadline = 0 + if conn.hasOutstanding() || conn.congestionState != goCongestionOpen || conn.congestion.congControl != nil { + e.rearmTimer(conn) + return + } + conn.congestionEvent(goCongestionEventWindowRestart) + conn.slowStartThreshold = conn.currentSlowStartThreshold() + restart := min(uint32(goInitialWindow), conn.congestionWindow) + conn.setCongestionWindow(max(conn.congestionWindow>>1, restart)) + conn.congestionWindowStamp = now + conn.congestionWindowUsed = 0 + e.summarizeFlight(conn) + conn.publishPermit(conn.sendUnacked.Load()) + e.armIdleRestart(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) applicationLimitedWindow(conn *GoConn, now int64) { + if conn.congestionState == goCongestionOpen { + used := max(conn.congestionWindowUsed, uint32(goInitialWindow)) + if used < conn.congestionWindow { + conn.slowStartThreshold = conn.currentSlowStartThreshold() + conn.setCongestionWindow((conn.congestionWindow + used) >> 1) + } + conn.congestionWindowUsed = 0 + } + conn.congestionWindowStamp = now +} + +type goMinMaxSample struct { + time uint32 + value uint32 +} + +// Mirrors lib/win_minmax.c. +type goMinMax struct { + samples [3]goMinMaxSample +} + +func (m *goMinMax) get() uint32 { + return m.samples[0].value +} + +func (m *goMinMax) reset(time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + m.samples[0] = sample + m.samples[1] = sample + m.samples[2] = sample + return value +} + +func (m *goMinMax) subwindowUpdate(window uint32, sample goMinMaxSample) uint32 { + dt := sample.time - m.samples[0].time + if dt > window { + m.samples[0] = m.samples[1] + m.samples[1] = m.samples[2] + m.samples[2] = sample + if sample.time-m.samples[0].time > window { + m.samples[0] = m.samples[1] + m.samples[1] = m.samples[2] + m.samples[2] = sample + } + } else if m.samples[1].time == m.samples[0].time && dt > window/4 { + m.samples[2] = sample + m.samples[1] = sample + } else if m.samples[2].time == m.samples[1].time && dt > window/2 { + m.samples[2] = sample + } + return m.samples[0].value +} + +func (m *goMinMax) runningMax(window uint32, time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + if value >= m.samples[0].value || time-m.samples[2].time > window { + return m.reset(time, value) + } + if value >= m.samples[1].value { + m.samples[2] = sample + m.samples[1] = sample + } else if value >= m.samples[2].value { + m.samples[2] = sample + } + return m.subwindowUpdate(window, sample) +} + +func (m *goMinMax) runningMin(window uint32, time uint32, value uint32) uint32 { + sample := goMinMaxSample{time: time, value: value} + if value <= m.samples[0].value || time-m.samples[2].time > window { + return m.reset(time, value) + } + if value <= m.samples[1].value { + m.samples[2] = sample + m.samples[1] = sample + } else if value <= m.samples[2].value { + m.samples[2] = sample + } + return m.subwindowUpdate(window, sample) +} diff --git a/stack_go_tcp_cubic.go b/stack_go_tcp_cubic.go new file mode 100644 index 00000000..bc22403d --- /dev/null +++ b/stack_go_tcp_cubic.go @@ -0,0 +1,294 @@ +package tun + +import ( + "math/bits" + "time" +) + +// Mirrors net/ipv4/tcp_cubic.c with HZ = 1000. + +const ( + goCubicBetaScale = 1024 + goCubicHZ = 10 + goCubicBeta = 717 + goCubicScale = 41 + goCubicHystartAckTrain = 1 + goCubicHystartDelay = 2 + goCubicHystartDetect = goCubicHystartAckTrain | goCubicHystartDelay + goCubicHystartMinSamples = 8 + goCubicHystartDelayMin = 4000 + goCubicHystartDelayMax = 16000 + goCubicHystartLowWindow = 16 + goCubicHystartAckDelta = 2000 + goCubicJiffiesPerSecond = 1000 + goCubicUpdateInterval = goCubicJiffiesPerSecond / 32 +) + +var ( + goCubicBetaScaleFactor = uint32(8 * (goCubicBetaScale + goCubicBeta) / 3 / (goCubicBetaScale - goCubicBeta)) + goCubicRoundTripScale = uint64(goCubicScale * 10) + goCubicFactor = (uint64(1) << (10 + 3*goCubicHZ)) / (goCubicScale * 10) +) + +type goCubicState struct { + count uint32 + lastMaxWindow uint32 + lastWindow uint32 + lastTime uint32 + originPoint uint32 + timeToOrigin uint32 + minDelay uint32 + epochStart uint32 + ackCount uint32 + renoWindow uint32 + sampleCount uint8 + found bool + roundStart int64 + endOffset uint64 + lastAck int64 + currentRoundTrip uint32 +} + +func goCubicClamp(value uint32) uint32 { + return min(max(value, goCubicHystartDelayMin), goCubicHystartDelayMax) +} + +func (s *goCubicState) reset() { + s.count = 0 + s.lastMaxWindow = 0 + s.lastWindow = 0 + s.lastTime = 0 + s.originPoint = 0 + s.timeToOrigin = 0 + s.minDelay = 0 + s.epochStart = 0 + s.ackCount = 0 + s.renoWindow = 0 + s.found = false +} + +func goCubicJiffies(c *GoConn) uint32 { + return uint32(c.engine.coarseTime.Load()/int64(time.Millisecond)) + 1 +} + +func goCubicMicros(c *GoConn) int64 { + return c.engine.coarseTime.Load() / int64(time.Microsecond) +} + +func (s *goCubicState) hystartReset(c *GoConn) { + now := goCubicMicros(c) + s.roundStart = now + s.lastAck = now + s.endOffset = c.sentTail.Load() + s.currentRoundTrip = ^uint32(0) + s.sampleCount = 0 +} + +func goCubicInit(c *GoConn) { + state := new(goCubicState) + state.reset() + state.hystartReset(c) + c.congestionPrivate = state +} + +func goCubicTxStart(c *GoConn, idleNanos int64) { + state := c.congestionPrivate.(*goCubicState) + delta := idleNanos / int64(time.Millisecond) + if state.epochStart != 0 && delta > 0 { + now := goCubicJiffies(c) + state.epochStart += uint32(delta) + if int32(state.epochStart-now) > 0 { + state.epochStart = now + } + } +} + +var goCubicRootTable = [64]uint8{ + 0, 54, 54, 54, 118, 118, 118, 118, + 123, 129, 134, 138, 143, 147, 151, 156, + 157, 161, 164, 168, 170, 173, 176, 179, + 181, 185, 187, 190, 192, 194, 197, 199, + 200, 202, 204, 206, 209, 211, 213, 215, + 217, 219, 221, 222, 224, 225, 227, 229, + 231, 232, 234, 236, 237, 239, 240, 242, + 244, 245, 246, 248, 250, 251, 252, 254, +} + +func goCubicRoot(a uint64) uint32 { + b := uint32(bits.Len64(a)) + if b < 7 { + return (uint32(goCubicRootTable[a]) + 35) >> 6 + } + b = ((b * 84) >> 8) - 1 + shift := a >> (b * 3) + x := ((uint32(goCubicRootTable[shift]) + 10) << b) >> 6 + x = 2*x + uint32(a/(uint64(x)*uint64(x-1))) + x = (x * 341) >> 10 + return x +} + +func (s *goCubicState) update(c *GoConn, window uint32, acked uint32) { + now := goCubicJiffies(c) + s.ackCount += acked + if s.lastWindow == window && int32(now-s.lastTime) <= goCubicUpdateInterval { + return + } + if s.epochStart == 0 || now != s.lastTime { + s.lastWindow = window + s.lastTime = now + if s.epochStart == 0 { + s.epochStart = now + s.ackCount = acked + s.renoWindow = window + if s.lastMaxWindow <= window { + s.timeToOrigin = 0 + s.originPoint = window + } else { + s.timeToOrigin = goCubicRoot(goCubicFactor * uint64(s.lastMaxWindow-window)) + s.originPoint = s.lastMaxWindow + } + } + t := uint64(now - s.epochStart) + t += (uint64(s.minDelay) + 999) / 1000 + t <<= goCubicHZ + t /= goCubicJiffiesPerSecond + var offset uint64 + if t < uint64(s.timeToOrigin) { + offset = uint64(s.timeToOrigin) - t + } else { + offset = t - uint64(s.timeToOrigin) + } + delta := uint32((goCubicRoundTripScale * offset * offset * offset) >> (10 + 3*goCubicHZ)) + var target uint32 + if t < uint64(s.timeToOrigin) { + target = s.originPoint - delta + } else { + target = s.originPoint + delta + } + if target > window { + s.count = window / (target - window) + } else { + s.count = 100 * window + } + if s.lastMaxWindow == 0 && s.count > 20 { + s.count = 20 + } + } + friendlyDelta := (window * goCubicBetaScaleFactor) >> 3 + for s.ackCount > friendlyDelta { + s.ackCount -= friendlyDelta + s.renoWindow++ + } + if s.renoWindow > window { + maxCount := window / (s.renoWindow - window) + if s.count > maxCount { + s.count = maxCount + } + } + s.count = max(s.count, 2) +} + +func goCubicCongAvoid(c *GoConn, acked uint32) { + state := c.congestionPrivate.(*goCubicState) + if !c.isWindowLimited() { + return + } + if c.inSlowStart() { + acked = c.slowStart(acked) + if acked == 0 { + return + } + } + state.update(c, c.congestionWindow, acked) + c.congAvoidAI(state.count, acked) +} + +func goCubicSlowStartThreshold(c *GoConn) uint32 { + state := c.congestionPrivate.(*goCubicState) + state.epochStart = 0 + window := c.congestionWindow + if window < state.lastMaxWindow { + state.lastMaxWindow = (window * (goCubicBetaScale + goCubicBeta)) / (2 * goCubicBetaScale) + } else { + state.lastMaxWindow = window + } + return max((window*goCubicBeta)/goCubicBetaScale, 2) +} + +func goCubicSetState(c *GoConn, newState uint8) { + if newState == goCongestionLoss { + state := c.congestionPrivate.(*goCubicState) + state.reset() + state.hystartReset(c) + } +} + +func goCubicHystartAckDelay(c *GoConn) uint32 { + rate := c.pacingRate.Load() + if rate == 0 { + return 0 + } + return uint32(min(uint64(time.Second/time.Microsecond)/1000, uint64(c.gsoMaxSize(c.effectiveMSS.Load()))*4*uint64(time.Second/time.Microsecond)/rate)) +} + +func (s *goCubicState) hystartUpdate(c *GoConn, delay uint32) { + if c.sendUnacked.Load() > s.endOffset { + s.hystartReset(c) + } + if c.congestionWindow < goCubicHystartLowWindow { + return + } + if goCubicHystartDetect&goCubicHystartAckTrain != 0 { + now := goCubicMicros(c) + if now-s.lastAck <= goCubicHystartAckDelta { + s.lastAck = now + threshold := int64(s.minDelay + goCubicHystartAckDelay(c)) + if !c.congestion.pacing { + threshold >>= 1 + } + if now-s.roundStart > threshold { + s.found = true + c.slowStartThreshold = c.congestionWindow + } + } + } + if goCubicHystartDetect&goCubicHystartDelay != 0 { + if s.currentRoundTrip > delay { + s.currentRoundTrip = delay + } + if s.sampleCount < goCubicHystartMinSamples { + s.sampleCount++ + } else if s.currentRoundTrip > s.minDelay+goCubicClamp(s.minDelay>>3) { + s.found = true + c.slowStartThreshold = c.congestionWindow + } + } +} + +func goCubicAcked(c *GoConn, sample *goAckSample) { + state := c.congestionPrivate.(*goCubicState) + if sample.roundTripMicros < 0 { + return + } + if state.epochStart != 0 && int32(goCubicJiffies(c)-state.epochStart) < goCubicJiffiesPerSecond { + return + } + delay := uint32(max(sample.roundTripMicros, 1)) + if state.minDelay == 0 || state.minDelay > delay { + state.minDelay = delay + } + if !state.found && c.inSlowStart() { + state.hystartUpdate(c, delay) + } +} + +var goCubicOps = goCongestionOps{ + name: "cubic", + init: goCubicInit, + slowStartThreshold: goCubicSlowStartThreshold, + congAvoid: goCubicCongAvoid, + setState: goCubicSetState, + undoWindow: goRenoUndoWindow, + windowEventTxStart: goCubicTxStart, + packetsAcked: goCubicAcked, +} diff --git a/stack_go_tcp_input.go b/stack_go_tcp_input.go new file mode 100644 index 00000000..46c5c161 --- /dev/null +++ b/stack_go_tcp_input.go @@ -0,0 +1,1562 @@ +package tun + +import ( + "encoding/binary" + "hash/maphash" + "math" + "net" + "syscall" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" +) + +// goRTOFloorMicros is above the delayed acknowledgement timer of every kernel that can sit on +// the other side of the device: Linux holds a lone segment's ACK for 40 ms, Darwin for 100 ms and +// Windows for 200 ms, and Windows itself floors its own RTO at 300 ms. +const ( + goRTOFloorMicros = 300000 + goRTOInitialMicros = 1000000 + goRTOMaxMicros = 60000000 + goRTOGranularity = int32(goWheelTick / int64(time.Microsecond)) + goSynAckRetransmit = 500 * time.Millisecond + goSynAckAttempts = 5 + goFinLinger = 5 * time.Second + goAbortLinger = time.Second + goRetransmitLimit = 8 + goProbeFloorMicros = 50000 + goProbeDelayedAck = 200000 + goProbeNoSampleMicros = 1000000 + goProbeAttempts = 1 + goKeepaliveInterval = 75 * time.Second + goKeepaliveCount = 9 + goFinWait2Timeout = 60 * time.Second + goAckRampBytes = 256 << 10 + goDelayedAckInterval = 25 * time.Millisecond + goDyingLeakTimeout = 30 * time.Second + goMaxCongestionSize = 4 << 20 + goResetBurstLimit = 32 + goRecoveryBurst = 64 + goInitialWindow = 10 + goAckCoalesceBytes = 256 << 10 +) + +func (p *forwardPacket) isPureTCPSyn() bool { + return p.protocol == uint8(header.TCPProtocolNumber) && + p.tcpFlags&header.TCPFlagSyn != 0 && p.tcpFlags&header.TCPFlagAck == 0 +} + +func (e *goEngine) initialSequence(key flowKey, now int64) uint32 { + return uint32(maphash.Comparable(e.sequenceSeed, key)) + uint32(now/goTimeUnit) +} + +func (e *goEngine) demuxTCP(packet []byte, parsed *forwardPacket) { + if len(parsed.transport) < header.TCPMinimumSize { + return + } + if parsed.isPureTCPSyn() { + e.handleTCPSyn(parsed) + return + } + if parsed.tcpFlags&header.TCPFlagRst != 0 { + return + } + e.answerNoFlow(parsed) +} + +func (e *goEngine) answerNoFlow(parsed *forwardPacket) { + if e.resetBurst >= goResetBurstLimit { + return + } + e.resetBurst++ + e.controlIdent++ + reply, ok := goBuildNoFlowReset(e.controlScratch[:], parsed, uint16(e.controlIdent)) + if !ok { + return + } + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write reset")) + } +} + +func goBuildNoFlowReset(scratch []byte, parsed *forwardPacket, ident uint16) ([]byte, bool) { + tcpHdr := header.TCP(parsed.transport) + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return nil, false + } + ipHeaderLength := goNetworkHeaderLength(parsed.ipVersion) + packet := scratch[:ipHeaderLength+header.TCPMinimumSize] + replyTCP := header.TCP(packet[ipHeaderLength:]) + encodeResetTCP(replyTCP, tcpHdr) + pseudoSum := goEncodeNetworkHeader(packet, parsed.ipVersion, header.TCPProtocolNumber, parsed.destination.Addr(), parsed.source.Addr(), header.TCPMinimumSize, ident) + replyTCP.SetChecksum(^replyTCP.CalculateChecksum(pseudoSum)) + return packet, true +} + +func (e *goEngine) handleTCPSyn(parsed *forwardPacket) { + key := parsed.flowKey() + tcpHdr := header.TCP(parsed.transport) + clientISN := tcpHdr.SequenceNumber() + existing := e.flows[key] + if existing != nil { + if existing.state == goTCPSynSent { + return + } + if existing.clientISN == clientISN { + e.handleDuplicateSyn(existing) + return + } + if !existing.dead && existing.state != goTCPSynReceived { + e.markAck(existing, true) + return + } + if !existing.dead { + e.detachConn(existing, E.New("go: connection replaced by new SYN"), goDeathImmediate) + } + e.removeFlow(existing) + } + listener := e.lookupListener(parsed.destination) + if listener == nil && e.stack.handler == nil { + e.answerNoFlow(parsed) + return + } + if len(e.flows) >= e.flowCapacity && !e.evictFlow() { + return + } + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return + } + if listener != nil && !listener.reserve() { + return + } + synOptions := header.ParseSynOptions(tcpHdr[header.TCPMinimumSize:dataOffset], false) + now := e.coarseTime.Load() + conn := new(GoConn) + source := M.SocksaddrFromNetIP(parsed.source) + destination := M.SocksaddrFromNetIP(parsed.destination) + conn.initialize(e, key, source, destination) + conn.ipVersion = parsed.ipVersion + conn.clientISN = clientISN + conn.sendISN = e.initialSequence(key, now) + conn.state = goTCPSynReceived + conn.sackPermitted = synOptions.SACKPermitted + if synOptions.WS >= 0 { + conn.localWindowShift = goLocalWindowShift + conn.peerWindowShift = uint8(synOptions.WS) + } + localMSS := e.localMSS(parsed.ipVersion) + conn.peerMSS = synOptions.MSS + effectiveMSS := min(synOptions.MSS, localMSS) + if synOptions.TS { + conn.timestampsEnabled = true + conn.tsRecent.Store(synOptions.TSVal) + effectiveMSS -= goTimestampOptionLength + } + conn.effectiveMSS.Store(uint32(effectiveMSS)) + e.initializeSession(conn, now, tcpHdr.WindowSize()) + conn.buildHandshake(localMSS, synOptions.WS >= 0, synOptions.SACKPermitted, header.TCPFlagSyn|header.TCPFlagAck, clientISN+1) + conn.keyed = true + e.stack.directory.insert(key, conn) + if listener != nil { + conn.listener = listener + conn.socket = true + e.handleEngage(conn) + return + } + go e.stack.handler.NewConnectionEx(e.stack.ctx, conn, source, destination, nil) +} + +func (e *goEngine) initializeSession(conn *GoConn, now int64, peerWindow uint16) { + conn.receiveCapacity = min(goReceiveCapacityBase, conn.receiveWindowBound()) + conn.receiveNext = 1 + conn.lastAckSent = 1 + conn.lastActivity = now + conn.publishedEdge = 1 + conn.receiveCapacity + conn.initCongestionControl(e.stack.congestion) + conn.retransmitTimeout = goRTOInitialMicros + conn.peerWindow = uint64(peerWindow) + conn.maxPeerWindow = conn.peerWindow + conn.lastPeerWindow = peerWindow + conn.consumedTail.Store(1) + conn.receiveAvailable.Store(1) + conn.receiveNextAck.Store(1) + conn.receiveEdge.Store(conn.publishedEdge) + conn.sentEdge = conn.publishedEdge + conn.dataSentEdge.Store(conn.publishedEdge) + conn.receiveSpaceConsumed = 1 + conn.receiveSpaceStamp = now + conn.sentTail.Store(1) + conn.transmittedTail.Store(1) + conn.bufferedTail.Store(1) + conn.sendUnacked.Store(1) + conn.sendReleased.Store(1) + conn.sendPermit.Store(1 + min(uint64(conn.congestionWindow)*uint64(conn.effectiveMSS.Load()), conn.peerWindow)) + conn.sendPacketPermit.Store(conn.congestionWindow) + conn.refreshWindowUpdateAt() +} + +func (e *goEngine) localMSS(ipVersion uint8) uint16 { + mss := e.platformIO.mtu() - goNetworkHeaderLength(ipVersion) - header.TCPMinimumSize + return uint16(min(max(mss, header.TCPMinimumSendMSS), header.TCPMaximumMSS)) +} + +func (e *goEngine) evictFlow() bool { + visited := 0 + for _, conn := range e.flows { + if conn.dead { + e.removeFlow(conn) + return true + } + visited++ + if visited >= 64 { + break + } + } + for _, conn := range e.flows { + if conn.state == goTCPSynReceived { + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + e.removeFlow(conn) + return true + } + visited++ + if visited >= 128 { + break + } + } + return false +} + +func (e *goEngine) handleDuplicateSyn(conn *GoConn) { + switch { + case conn.dead: + if conn.deathClass == goDeathAbortLinger { + e.sendReset(conn) + } + case conn.phase == goPhaseEngaged: + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) + } + } +} + +func (e *goEngine) inputTCP(conn *GoConn, parsed *forwardPacket) { + tcpHdr := header.TCP(parsed.transport) + dataOffset := int(tcpHdr.DataOffset()) + if dataOffset < header.TCPMinimumSize || dataOffset > len(tcpHdr) { + return + } + if conn.dead { + e.inputTombstone(conn, parsed) + return + } + conn.lastActivity = e.coarseTime.Load() + if conn.keepaliveProbes != 0 { + conn.keepaliveProbes = 0 + e.armKeepalive(conn) + e.rearmTimer(conn) + } + if conn.state == goTCPSynSent { + e.inputSynSent(conn, parsed, tcpHdr, dataOffset) + return + } + payload := tcpHdr[dataOffset:] + flags := tcpHdr.Flags() + segOffset := conn.receiveOffset(tcpHdr.SequenceNumber()) + segmentLength := int64(len(payload)) + if flags&header.TCPFlagSyn != 0 { + segmentLength++ + } + if flags&header.TCPFlagFin != 0 { + segmentLength++ + } + if flags&header.TCPFlagRst != 0 { + e.handleReset(conn, segOffset) + return + } + if len(payload) > 0 && flags&header.TCPFlagSyn == 0 { + conn.recordDuplicate(segOffset, segOffset+int64(len(payload))) + } + var ( + timestampValue uint32 + timestampEcho uint32 + timestampRecent uint32 + hasTimestamp bool + ) + if conn.timestampsEnabled { + timestampValue, timestampEcho, hasTimestamp = goParseTimestampOption(tcpHdr[header.TCPMinimumSize:dataOffset]) + timestampRecent = conn.tsRecent.Load() + } + if hasTimestamp && int32(timestampValue-timestampRecent) < 0 { + e.markAck(conn, true) + return + } + if !conn.segmentAcceptable(segOffset, segmentLength) { + e.markAck(conn, true) + return + } + if hasTimestamp && timestampValue != timestampRecent && segOffset <= int64(conn.lastAckSent) { + conn.tsRecent.Store(timestampValue) + } + if flags&header.TCPFlagSyn != 0 { + segOffset++ + if len(payload) == 0 && flags&header.TCPFlagFin == 0 { + e.markAck(conn, true) + return + } + } + if flags&header.TCPFlagAck == 0 { + return + } + if !e.processAck(conn, tcpHdr, segOffset, len(payload), flags, timestampEcho, hasTimestamp) { + return + } + if conn.dead { + return + } + if len(payload) > 0 || flags&header.TCPFlagFin != 0 { + e.deliverSegment(conn, segOffset, payload, flags&header.TCPFlagFin != 0) + } + e.maybeSendFin(conn) + if conn.splice != nil { + e.spliceAfterInput(conn) + } +} + +func (e *goEngine) inputTombstone(conn *GoConn, parsed *forwardPacket) { + if parsed.tcpFlags&header.TCPFlagRst != 0 { + e.removeFlow(conn) + return + } + switch conn.deathClass { + case goDeathFinLinger: + if parsed.tcpFlags&header.TCPFlagFin != 0 { + e.sendAck(conn) + } + case goDeathAbortLinger: + e.sendReset(conn) + } +} + +func (e *goEngine) handleReset(conn *GoConn, segOffset int64) { + if segOffset == int64(conn.receiveNext) { + e.detachConn(conn, errGoReset, goDeathImmediate) + return + } + if conn.state != goTCPSynReceived && conn.segmentAcceptable(segOffset, 0) { + e.markAck(conn, true) + } +} + +func (c *GoConn) receiveOffset(sequence uint32) int64 { + current := uint32(uint64(c.clientISN) + c.receiveNext) + return int64(c.receiveNext) + int64(int32(sequence-current)) +} + +func (c *GoConn) recordDuplicate(start int64, end int64) { + if start < 0 || end <= start { + return + } + switch { + case uint64(end) <= c.receiveNext: + case uint64(start) < c.receiveNext: + end = int64(c.receiveNext) + default: + if c.oooRanges == nil { + return + } + overlapStart, overlapEnd, overlapped := c.oooRanges.overlap(uint64(start), uint64(end)) + if !overlapped { + return + } + start = int64(overlapStart) + end = int64(overlapEnd) + } + c.dsackStart = uint64(start) + c.dsackEnd = uint64(end) +} + +func (c *GoConn) sendOffset(acknowledgement uint32) int64 { + unacked := c.sendUnacked.Load() + current := uint32(uint64(c.sendISN) + unacked) + return int64(unacked) + int64(int32(acknowledgement-current)) +} + +func (c *GoConn) segmentAcceptable(segOffset int64, segmentLength int64) bool { + receiveNext := int64(c.receiveNext) + edge := int64(c.publishedEdge) + if segmentLength == 0 { + if edge == receiveNext { + return segOffset == receiveNext + } + return segOffset >= receiveNext && segOffset <= edge + } + if edge == receiveNext { + return false + } + return segOffset < edge && segOffset+segmentLength > receiveNext +} + +func (e *goEngine) establishConn(conn *GoConn) { + conn.state = goTCPEstablished + conn.handshakeDeadline = 0 + conn.access.Lock() + conn.phase = goPhaseEstablished + conn.access.Unlock() + e.armKeepalive(conn) + e.rearmTimer(conn) + close(conn.establishedSignal) + if conn.listener != nil { + conn.listener.deliver(conn) + } + e.wokeHandlerThisBurst = true +} + +func goParseTimestampOption(options []byte) (uint32, uint32, bool) { + for index := 0; index < len(options); { + switch options[index] { + case header.TCPOptionEOL: + return 0, 0, false + case header.TCPOptionNOP: + index++ + case header.TCPOptionTS: + if index+header.TCPOptionTSLength > len(options) || options[index+1] != header.TCPOptionTSLength { + return 0, 0, false + } + return binary.BigEndian.Uint32(options[index+2:]), binary.BigEndian.Uint32(options[index+6:]), true + default: + if index+2 > len(options) { + return 0, 0, false + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) { + return 0, 0, false + } + index += optionLength + } + } + return 0, 0, false +} + +type goRawSackBlock struct { + start int64 + end int64 +} + +func goParseSackBlocks(tcpHdr header.TCP, dataOffset int, conn *GoConn, blocks *[goMaxSackBlocks]goRawSackBlock) int { + options := tcpHdr[header.TCPMinimumSize:dataOffset] + count := 0 + for index := 0; index < len(options); { + switch options[index] { + case header.TCPOptionEOL: + return count + case header.TCPOptionNOP: + index++ + case header.TCPOptionSACK: + if index+2 > len(options) { + return count + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) || (optionLength-2)%8 != 0 { + return count + } + for offset := index + 2; offset+8 <= index+optionLength && count < goMaxSackBlocks; offset += 8 { + start := conn.sendOffset(binary.BigEndian.Uint32(options[offset:])) + end := conn.sendOffset(binary.BigEndian.Uint32(options[offset+4:])) + if end <= start { + continue + } + blocks[count] = goRawSackBlock{start: start, end: end} + count++ + } + index += optionLength + default: + if index+2 > len(options) { + return count + } + optionLength := int(options[index+1]) + if optionLength < 2 || index+optionLength > len(options) { + return count + } + index += optionLength + } + } + return count +} + +func goIsDuplicateSack(blocks []goRawSackBlock, ackOffset int64) bool { + first := blocks[0] + if first.end <= ackOffset { + return true + } + if len(blocks) > 1 { + second := blocks[1] + return second.start <= first.start && first.end <= second.end + } + return false +} + +func goSelectSackBlocks(raw []goRawSackBlock, unacked uint64, sent uint64, blocks *[goMaxSackBlocks]goSackBlock) int { + count := 0 + for _, block := range raw { + if block.start <= int64(unacked) || block.end <= block.start || block.end > int64(sent) { + continue + } + blocks[count] = goSackBlock{start: uint64(block.start), end: uint64(block.end)} + count++ + } + return count +} + +func (c *GoConn) releaseTransmitted(unacked uint64) { + edge := min(unacked, c.transmittedTail.Load()) + if edge <= c.sendReleased.Load() { + return + } + c.transmitStore.releaseBelow(edge) + if edge == c.bufferedTail.Load() { + c.transmitStore.releaseDrained(edge) + c.scoreboard.trim() + } + c.sendReleased.Store(edge) +} + +func (c *GoConn) releaseIdleDescriptors() { + if len(c.scoreboard.entries) != 0 || !c.transmitAccess.TryLock() { + return + } + if c.sendUnacked.Load() == c.sentTail.Load() && !c.blockedValid { + c.descriptors.release() + } + c.transmitAccess.Unlock() + if c.nextTransmitLength(c.effectiveMSS.Load()) != 0 { + c.wakeTransmitter() + } +} + +func (c *GoConn) releaseDrainedSlabs() { + consumed := c.consumedTail.Load() + if consumed == c.receiveAvailable.Load() && (c.oooRanges == nil || c.oooRanges.count == 0) { + c.receiveChain.releaseDrained(consumed) + } + buffered := c.bufferedTail.Load() + if min(c.sendUnacked.Load(), c.transmittedTail.Load()) == buffered { + c.transmitStore.releaseDrained(buffered) + } +} + +func (c *GoConn) wakeTransmitter() { + if c.splice != nil { + c.engine.spliceResume(c) + return + } + if c.nextTransmitLength(c.effectiveMSS.Load()) == 0 { + return + } + if goEngineTransmits { + if c.engine.inlineTransmitBudget <= 0 { + c.engine.postMessage(&c.transmitMessage) + } else { + c.engine.transmitOnEngine(c, c.engine.inlineTransmitBudget) + } + return + } + c.wakeTransmitterGoroutine() + c.engine.wokeHandlerThisBurst = true +} + +func (c *GoConn) wakeWriter() { + required := c.writerWaiting.Load() + if required == 0 || !c.writeReady(int(required)) { + return + } + c.signalWriter() +} + +func (c *GoConn) signalWriter() { + if c.writeSignal.notify() { + c.engine.wokeHandlerThisBurst = true + } +} + +func (c *GoConn) signalReader() { + if c.readSignal.notify() { + c.engine.wokeHandlerThisBurst = true + } +} + +func (e *goEngine) deliverSegment(conn *GoConn, segOffset int64, payload []byte, fin bool) { + if conn.finReceived { + e.markAck(conn, true) + return + } + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + receiveNext := int64(conn.receiveNext) + edge := int64(conn.publishedEdge) + start := max(segOffset, receiveNext) + end := min(segOffset+int64(len(payload)), edge) + finOffset := segOffset + int64(len(payload)) + if end <= start { + if fin && finOffset == receiveNext { + e.acceptFin(conn) + return + } + if fin && finOffset > receiveNext && finOffset < edge { + if conn.oooRanges == nil { + conn.oooRanges = new(goRangeSet) + } + conn.oooRanges.insert(uint64(finOffset), uint64(finOffset), true) + } + e.markAck(conn, true) + return + } + data := payload[start-segOffset : end-segOffset] + if start == receiveNext { + e.deliverInOrder(conn, data) + if fin && finOffset == int64(conn.receiveNext) { + e.acceptFin(conn) + return + } + e.mergeOutOfOrder(conn) + e.measureReceiveRoundTrip(conn) + conn.ackPending += uint64(len(data)) + conn.ackCovered += uint64(len(data)) + e.markAck(conn, conn.ackPending >= conn.ackThreshold()) + return + } + if conn.discardReceive { + e.markAck(conn, true) + return + } + if conn.oooRanges == nil { + conn.oooRanges = new(goRangeSet) + } + if conn.oooRanges.bytes()+uint64(len(data)) > conn.receiveCapacity { + e.markAck(conn, true) + return + } + conn.receiveChain.reserve(uint64(start), len(data)) + conn.receiveChain.writeAt(uint64(start), data) + if !conn.oooRanges.insert(uint64(start), uint64(end), fin && finOffset == end) { + e.markAck(conn, true) + return + } + e.markAck(conn, true) +} + +func (e *goEngine) deliverInOrder(conn *GoConn, data []byte) { + if conn.discardReceive { + conn.receiveNext += uint64(len(data)) + conn.receiveNextAck.Store(conn.receiveNext) + conn.publishReceiveWindow() + return + } + if conn.splice != nil { + e.spliceDeliver(conn, data) + return + } + direct := 0 + taken := false + if conn.receiveAvailable.Load() == conn.consumedTail.Load() { + target := conn.postedTarget.Load() + if target != nil && target != &goTargetFilled && conn.postedTarget.CompareAndSwap(target, &goTargetTaken) { + direct = copy(target.target, data) + target.filled = direct + taken = true + } + } + if direct < len(data) { + conn.receiveChain.reserve(conn.receiveNext+uint64(direct), len(data)-direct) + conn.receiveChain.writeAt(conn.receiveNext+uint64(direct), data[direct:]) + } + conn.receiveNext += uint64(len(data)) + conn.receiveNextAck.Store(conn.receiveNext) + conn.receiveAvailable.Store(conn.receiveNext) + conn.publishReceiveWindow() + if taken { + conn.postedTarget.Store(&goTargetFilled) + } + if conn.readerParked.Load() || taken { + conn.signalReader() + } +} + +func (e *goEngine) mergeOutOfOrder(conn *GoConn) { + if conn.oooRanges == nil { + return + } + for { + front, ok := conn.oooRanges.first() + if !ok || front.start > conn.receiveNext { + return + } + if front.end > conn.receiveNext { + conn.receiveNext = front.end + conn.receiveNextAck.Store(conn.receiveNext) + if !conn.discardReceive { + conn.receiveAvailable.Store(conn.receiveNext) + if conn.readerParked.Load() { + conn.signalReader() + } + } + conn.publishReceiveWindow() + } + fin := front.fin + conn.oooRanges.removeBelow(conn.receiveNext) + e.markAck(conn, true) + if fin { + e.acceptFin(conn) + return + } + } +} + +func (e *goEngine) acceptFin(conn *GoConn) { + if conn.finReceived { + e.markAck(conn, true) + return + } + conn.access.Lock() + conn.finReceived = true + conn.access.Unlock() + conn.oooRanges = nil + conn.receiveNext++ + conn.receiveNextAck.Store(conn.receiveNext) + conn.publishReceiveWindow() + if conn.readerParked.Load() { + conn.signalReader() + } + switch conn.state { + case goTCPEstablished: + conn.state = goTCPCloseWait + case goTCPFinWait1: + if conn.finAcked { + conn.state = goTCPFinWait2 + conn.finWait2Since = e.coarseTime.Load() + e.markAck(conn, true) + e.finishClose(conn) + return + } + conn.state = goTCPClosing + case goTCPFinWait2: + e.markAck(conn, true) + e.finishClose(conn) + return + } + e.markAck(conn, true) +} + +func (e *goEngine) advanceFinState(conn *GoConn) { + if !conn.finSent || !conn.finAcked { + return + } + switch conn.state { + case goTCPFinWait1: + if conn.finReceived { + e.finishClose(conn) + return + } + conn.state = goTCPFinWait2 + conn.finWait2Since = e.coarseTime.Load() + e.armKeepalive(conn) + e.rearmTimer(conn) + case goTCPClosing, goTCPLastAck: + e.finishClose(conn) + } +} + +func (e *goEngine) finishClose(conn *GoConn) { + conn.state = goTCPClosed + e.detachConn(conn, nil, goDeathFinLinger) +} + +func (c *GoConn) publishReceiveWindow() { + base := c.consumedTail.Load() + if c.discardReceive { + base = c.receiveNext + } + edge := base + c.receiveCapacity + if edge > c.publishedEdge && (base >= c.receiveNext || edge-c.advertisedEdge() >= c.windowUpdateThreshold()) { + c.publishedEdge = edge + c.receiveEdge.Store(edge) + } + c.refreshWindowUpdateAt() +} + +func (c *GoConn) advertisedEdge() uint64 { + return max(c.sentEdge, c.dataSentEdge.Load()) +} + +func (c *GoConn) windowUpdateThreshold() uint64 { + return min(uint64(c.effectiveMSS.Load()), c.receiveCapacity/2) +} + +func (c *GoConn) refreshWindowUpdateAt() { + edge := c.advertisedEdge() + capacity := int64(c.receiveCapacity) + at := int64(edge) + int64(c.windowUpdateThreshold()) - capacity + available := c.receiveAvailable.Load() + if available < edge { + window := int64(edge - available) + if 2*window > capacity { + c.windowUpdateAt.Store(math.MaxUint64) + return + } + at = max(at, int64(available)+2*window-capacity) + } + c.windowUpdateAt.Store(uint64(max(at, 0))) +} + +func (e *goEngine) measureReceiveRoundTrip(conn *GoConn) { + now := e.coarseTime.Load() + if conn.receiveRoundTripMark == 0 { + conn.receiveRoundTripMark = conn.receiveNext + conn.receiveCapacity + conn.receiveRoundTripStamp = now + return + } + if conn.receiveNext < conn.receiveRoundTripMark { + return + } + sample := max(int32(min((now-conn.receiveRoundTripStamp)/int64(time.Microsecond), goRTOMaxMicros)), 1) + if conn.receiveRoundTrip == 0 || sample < conn.receiveRoundTrip { + conn.receiveRoundTrip = sample + } + conn.receiveRoundTripMark = conn.receiveNext + conn.receiveCapacity + conn.receiveRoundTripStamp = now +} + +func (e *goEngine) moderateReceiveCapacity(conn *GoConn) { + if conn.receiveRoundTrip == 0 { + return + } + now := e.coarseTime.Load() + if now-conn.receiveSpaceStamp < int64(conn.receiveRoundTrip)*int64(time.Microsecond) { + return + } + consumed := conn.consumedTail.Load() + copied := consumed - conn.receiveSpaceConsumed + previous := conn.receiveSpaceCopied + conn.receiveSpaceConsumed = consumed + conn.receiveSpaceCopied = copied + conn.receiveSpaceStamp = now + capacity := min(max(2*max(copied, previous), conn.receiveCapacity), e.receiveCapacityLimit(conn)) + if capacity == conn.receiveCapacity { + return + } + conn.receiveCapacity = capacity +} + +func (e *goEngine) receiveCapacityLimit(conn *GoConn) uint64 { + bound := conn.receiveWindowBound() + share := e.slabPool.shareBytes() + if share == 0 { + return bound + } + transmitHeld := conn.transmitStore.chain.heldBytes() + if share <= transmitHeld { + return min(goReceiveCapacityBase, bound) + } + return min(max(share-transmitHeld, goReceiveCapacityBase), bound) +} + +func (e *goEngine) markAck(conn *GoConn, forced bool) { + if forced { + conn.ackForced = true + } + if conn.ackDirty { + return + } + conn.ackDirty = true + conn.ackNext = e.ackList + e.ackList = conn +} + +func (e *goEngine) drainAckList(now int64, allowDefer bool) { + var kept *GoConn + for conn := e.ackList; conn != nil; { + next := conn.ackNext + conn.ackNext = nil + if allowDefer && !conn.ackForced && conn.ackPending < conn.ackThreshold() && conn.ackCovered >= goAckRampBytes { + conn.ackNext = kept + kept = conn + conn = next + continue + } + conn.ackDirty = false + if !e.sendAck(conn) { + conn.ackDirty = true + conn.ackNext = kept + kept = conn + } + conn = next + } + e.ackList = kept + if kept != nil && e.delayedAckTickNode.slot == nil { + e.wheel.schedule(&e.delayedAckTickNode, now+int64(goDelayedAckInterval)) + } +} + +func (e *goEngine) sendAck(conn *GoConn) bool { + e.moderateReceiveCapacity(conn) + conn.receiveChain.releaseBelow(conn.consumedTail.Load()) + conn.publishReceiveWindow() + + segment := goSegment{offset: conn.sendNext(), flags: header.TCPFlagAck} + if conn.sackPermitted && (conn.dsackEnd != 0 || conn.oooRanges != nil && conn.oooRanges.count > 0) { + count := 0 + if conn.dsackEnd != 0 { + e.sackScratch[0] = goSackBlock{start: conn.dsackStart, end: conn.dsackEnd} + count = 1 + } + if conn.oooRanges != nil { + count = conn.oooRanges.blocks(&e.sackScratch, count) + } + segment.sackBlocks = e.sackScratch[:count] + } + if !e.writeConnControl(conn, &segment) { + return false + } + conn.refreshWindowUpdateAt() + conn.dsackStart = 0 + conn.dsackEnd = 0 + conn.ackPending = 0 + conn.ackForced = false + conn.lastAckSent = conn.receiveNext + return true +} + +func (e *goEngine) sendReset(conn *GoConn) bool { + if conn.state == goTCPSynSent { + return true + } + segment := goSegment{offset: conn.sendNext(), flags: header.TCPFlagRst | header.TCPFlagAck} + return e.writeConnControl(conn, &segment) +} + +func (e *goEngine) writeConnControl(conn *GoConn, segment *goSegment) bool { + frame, _ := conn.buildFrame(e.controlScratch[:], e.controlSegments[:0], segment, &conn.transmitStore, false, true) + e.controlSegments = frame + err := e.platformIO.writeFrame(frame, ForwardFrameMeta{}) + clear(frame) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write control frame")) + return false + } + return true +} + +func (e *goEngine) maybeSendFin(conn *GoConn) { + if !conn.finPending || conn.finSent || conn.handshaking() || conn.dead { + return + } + conn.access.Lock() + writing := conn.writing + conn.access.Unlock() + if writing { + return + } + buffered := conn.bufferedTail.Load() + if conn.sentTail.Load() != buffered || conn.transmittedTail.Load() != buffered { + return + } + conn.finOffset = buffered + conn.finSent = true + segment := goSegment{offset: conn.finOffset, flags: header.TCPFlagFin | header.TCPFlagAck} + e.writeConnControl(conn, &segment) + switch conn.state { + case goTCPCloseWait: + conn.state = goTCPLastAck + default: + conn.state = goTCPFinWait1 + } + e.rearmRetransmit(conn) +} + +func (e *goEngine) handleCloseRequest(conn *GoConn) { + conn.access.Lock() + abort := conn.abort || conn.splicePending != nil + discard := conn.userClosed || conn.abort + if conn.dead && discard { + conn.drainable = false + } + err := conn.err + conn.access.Unlock() + if conn.dead { + if discard { + e.spliceDetach(conn, err) + } + return + } + if conn.state == goTCPSynSent { + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + return + } + if abort || conn.splice != nil { + e.sendReset(conn) + e.detachConn(conn, net.ErrClosed, goDeathAbortLinger) + return + } + conn.finPending = true + e.maybeSendFin(conn) + if conn.dead { + return + } + e.armKeepalive(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) handleReadShut(conn *GoConn) { + if conn.dead { + return + } + conn.discardReceive = true + conn.oooRanges = nil + if conn.readAccess.TryLock() { + conn.consumedTail.Store(conn.receiveAvailable.Load()) + conn.readAccess.Unlock() + conn.releaseDrainedSlabs() + } + conn.publishReceiveWindow() + if conn.readerParked.Load() { + conn.signalReader() + } +} + +func (e *goEngine) handleWindowUpdate(conn *GoConn) { + if conn.dead { + return + } + consumed := conn.consumedTail.Load() + windowUpdate := consumed >= conn.windowUpdateAt.Load() + conn.receiveChain.releaseBelow(consumed) + conn.releaseDrainedSlabs() + if !windowUpdate { + return + } + conn.publishReceiveWindow() + if conn.publishedEdge > conn.advertisedEdge() || conn.ackPending > 0 { + e.markAck(conn, true) + } +} + +func (e *goEngine) handleTransmitted(conn *GoConn) { + if conn.dead { + return + } + conn.releaseTransmitted(conn.sendUnacked.Load()) + conn.releaseIdleDescriptors() + conn.wakeWriter() +} + +func (e *goEngine) handleRetransmitArm(conn *GoConn) { + if conn.dead || conn.retransmitDeadline != 0 { + return + } + e.rearmRetransmit(conn) +} + +func (e *goEngine) handleDroppedFrames(conn *GoConn) { + if conn.dead { + return + } + e.drainDescriptors(conn) + lost := false + for index := range conn.scoreboard.entries { + descriptor := &conn.scoreboard.entries[index] + if descriptor.flags&goDescriptorDropped != 0 && descriptor.flags&(goDescriptorSacked|goDescriptorLost) == 0 { + e.markLost(conn, descriptor) + lost = true + } + } + e.refreshPacketsOut(conn) + if lost { + e.enterCWR(conn) + if conn.congestionState < goCongestionRecovery { + e.enterRecovery(conn) + } + e.xmitRetransmitQueue(conn) + } + e.rearmRetransmit(conn) + conn.publishPermit(conn.sendUnacked.Load()) +} + +type goBlockedWriters struct { + head *GoConn + tail *GoConn +} + +func (l *goBlockedWriters) push(conn *GoConn) { + conn.blockedOn = l + conn.blockedPrev = l.tail + conn.blockedNext = nil + if l.tail == nil { + l.head = conn + } else { + l.tail.blockedNext = conn + } + l.tail = conn +} + +func (l *goBlockedWriters) remove(conn *GoConn) { + if conn.blockedPrev == nil { + l.head = conn.blockedNext + } else { + conn.blockedPrev.blockedNext = conn.blockedNext + } + if conn.blockedNext == nil { + l.tail = conn.blockedPrev + } else { + conn.blockedNext.blockedPrev = conn.blockedPrev + } + conn.blockedOn = nil + conn.blockedPrev = nil + conn.blockedNext = nil +} + +func (e *goEngine) handleTransmitBlocked(conn *GoConn) { + if conn.dead { + conn.transmitSignal.notify() + return + } + waiters, err := e.platformIO.armTransmitWritable(conn) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: arm transmit writable")) + } + if conn.blockedOn == waiters && waiters != nil { + return + } + if conn.blockedOn != nil { + conn.blockedOn.remove(conn) + } + if waiters == nil { + e.releaseBlockedWriter(conn) + return + } + waiters.push(conn) +} + +func (e *goEngine) releaseBlockedWriters(waiters *goBlockedWriters) { + e.releasing = *waiters + *waiters = goBlockedWriters{} + for conn := e.releasing.head; conn != nil; conn = conn.blockedNext { + conn.blockedOn = &e.releasing + } + for e.releasing.head != nil { + conn := e.releasing.head + e.releasing.remove(conn) + e.releaseBlockedWriter(conn) + } +} + +func (e *goEngine) releaseBlockedWriter(conn *GoConn) { + if conn.splice != nil || goEngineTransmits { + e.retryBlockedOnEngine(conn) + } else { + conn.transmitSignal.notify() + e.wokeHandlerThisBurst = true + } +} + +func (c *GoConn) hasOutstandingData() bool { + return c.sendUnacked.Load() < c.sentTail.Load() +} + +func (c *GoConn) hasOutstanding() bool { + if c.handshaking() { + return false + } + return c.sendUnacked.Load() < c.sentTail.Load() || (c.finSent && !c.finAcked) +} + +func (e *goEngine) rearmRetransmit(conn *GoConn) { + if !conn.hasOutstanding() { + conn.retransmitDeadline = 0 + if conn.persistNeeded() { + e.updatePersist(conn) + } else { + e.rearmTimer(conn) + } + return + } + if conn.persistNeeded() { + e.updatePersist(conn) + return + } + conn.retransmitDeadline = e.coarseTime.Load() + int64(conn.retransmitTimeout)*int64(time.Microsecond) + e.armProbe(conn) + e.rearmTimer(conn) +} + +func (c *GoConn) persistNeeded() bool { + return c.peerWindow == 0 && (c.bufferedTail.Load() > c.sendUnacked.Load() || (c.finSent && !c.finAcked)) +} + +func (e *goEngine) updatePersist(conn *GoConn) { + if conn.persistNeeded() { + conn.retransmitDeadline = 0 + conn.probeDeadline = 0 + if conn.persistDeadline == 0 { + conn.persistAttempts = 0 + conn.persistDeadline = e.coarseTime.Load() + int64(conn.retransmitTimeout)*int64(time.Microsecond) + } + e.rearmTimer(conn) + return + } + if conn.persistDeadline != 0 { + conn.persistDeadline = 0 + conn.persistAttempts = 0 + if conn.hasOutstanding() { + e.retransmitFrom(conn, conn.sendUnacked.Load()) + e.rearmRetransmit(conn) + } else { + e.rearmTimer(conn) + } + } +} + +func (e *goEngine) rearmTimer(conn *GoConn) { + next := int64(0) + for _, deadline := range [9]int64{conn.retransmitDeadline, conn.probeDeadline, conn.persistDeadline, conn.lingerDeadline, conn.handshakeDeadline, conn.idleDeadline, conn.pacingDeadline, conn.reorderDeadline, conn.keepaliveDeadline} { + if deadline == 0 { + continue + } + if next == 0 || deadline < next { + next = deadline + } + } + if next == 0 { + e.wheel.cancel(&conn.timerNode) + return + } + e.wheel.schedule(&conn.timerNode, next) +} + +func (e *goEngine) fireConnTimer(conn *GoConn, now int64) { + if conn.lingerDeadline != 0 && conn.lingerDeadline <= now { + conn.lingerDeadline = 0 + e.removeFlow(conn) + return + } + if conn.handshakeDeadline != 0 && conn.handshakeDeadline <= now { + e.expireHandshake(conn, now) + return + } + if conn.pacingDeadline != 0 && conn.pacingDeadline <= now { + e.expirePacing(conn) + return + } + if conn.idleDeadline != 0 && conn.idleDeadline <= now { + e.expireIdleRestart(conn, now) + return + } + if conn.reorderDeadline != 0 && conn.reorderDeadline <= now { + e.expireReorder(conn) + return + } + if conn.persistDeadline != 0 && conn.persistDeadline <= now { + e.expirePersist(conn, now) + return + } + if conn.probeDeadline != 0 && conn.probeDeadline <= now { + e.expireProbe(conn, now) + return + } + if conn.retransmitDeadline != 0 && conn.retransmitDeadline <= now { + e.expireRetransmit(conn, now) + return + } + if conn.keepaliveDeadline != 0 && conn.keepaliveDeadline <= now { + e.expireKeepalive(conn, now) + return + } + e.rearmTimer(conn) +} + +func (e *goEngine) expireHandshake(conn *GoConn, now int64) { + if conn.state == goTCPSynSent { + e.retransmitSyn(conn, now) + return + } + conn.handshakeAttempts++ + if conn.handshakeAttempts >= goSynAckAttempts { + conn.handshakeDeadline = 0 + e.detachConn(conn, E.New("go: handshake timeout"), goDeathAbortLinger) + return + } + err := goIgnoreDropped(conn.engine.platformIO.writePacket(conn.handshakeImage[:conn.handshakeLength], ForwardFrameMeta{})) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: resend SYN-ACK")) + } + conn.handshakeDeadline = now + int64(goSynAckRetransmit)< conn.consumedTail.Load() + conn.dead = true + established := conn.phase == goPhaseEstablished + conn.access.Unlock() + conn.deathClass = class + conn.state = goTCPClosed + if class == goDeathFinLinger && conn.splice != nil && !conn.splice.uploadClosed { + e.spliceMarkDirty(conn) + } else { + e.spliceDetach(conn, err) + } + if !established { + close(conn.establishedSignal) + if conn.listener != nil { + conn.listener.abandon() + } + } + close(conn.closeSignal) + conn.retransmitDeadline = 0 + conn.probeDeadline = 0 + conn.persistDeadline = 0 + conn.handshakeDeadline = 0 + conn.idleDeadline = 0 + conn.pacingDeadline = 0 + conn.reorderDeadline = 0 + conn.keepaliveDeadline = 0 + conn.finPending = false + switch class { + case goDeathFinLinger: + conn.lingerDeadline = e.coarseTime.Load() + int64(goFinLinger) + case goDeathAbortLinger: + conn.lingerDeadline = e.coarseTime.Load() + int64(goAbortLinger) + default: + conn.lingerDeadline = 0 + } + e.rearmTimer(conn) + if class == goDeathImmediate { + e.removeFlowKey(conn) + } + if !conn.onDyingList { + conn.onDyingList = true + conn.dyingSince = e.coarseTime.Load() + conn.dyingNext = e.dyingList + e.dyingList = conn + } + if conn.blockedOn != nil { + conn.blockedOn.remove(conn) + conn.transmitSignal.notify() + } + conn.wakeUser() + e.wokeHandlerThisBurst = true +} + +func (e *goEngine) removeFlow(conn *GoConn) { + if !conn.dead { + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + return + } + conn.lingerDeadline = 0 + e.wheel.cancel(&conn.timerNode) + e.removeFlowKey(conn) +} + +func (e *goEngine) removeFlowKey(conn *GoConn) { + if !conn.keyed { + return + } + conn.keyed = false + e.stack.directory.remove(conn.key, conn) + if e.dispatchStage == nil { + return + } + reason := FlowCloseReset + if conn.finReceived && conn.finSent { + reason = FlowCloseFinished + } + e.dispatchStage.teardownFlow(conn.key, reason) +} + +func (e *goEngine) reapDying() { + var kept *GoConn + now := e.coarseTime.Load() + for conn := e.dyingList; conn != nil; { + next := conn.dyingNext + conn.dyingNext = nil + if conn.reclaim() { + conn.onDyingList = false + } else { + if now-conn.dyingSince > int64(goDyingLeakTimeout) { + conn.dyingSince = now + e.stack.logger.Warn("go: connection stuck in teardown: ", conn.local) + } + conn.dyingNext = kept + kept = conn + } + conn = next + } + e.dyingList = kept +} + +func (c *GoConn) reclaim() bool { + c.access.Lock() + blocked := c.spliced || c.transmitterRunning || (c.drainable && !c.userClosed && c.receiveAvailable.Load() > c.consumedTail.Load()) + c.access.Unlock() + if blocked || !c.readAccess.TryLock() { + return false + } + defer c.readAccess.Unlock() + if !c.writeAccess.TryLock() { + return false + } + defer c.writeAccess.Unlock() + if !c.transmitAccess.TryLock() { + return false + } + defer c.transmitAccess.Unlock() + c.access.Lock() + c.released = true + c.access.Unlock() + c.engine.platformIO.flush() + c.receiveChain.releaseAll() + c.transmitStore.releaseAll() + c.blockedValid = false + c.blockedFrame = nil + c.transmitSegments = nil + c.oooRanges = nil + c.descriptors.release() + c.scoreboard.entries = nil + c.releaseCongestionControl() + if c.receiveTarget.buffer != nil { + c.receiveTarget.buffer.Release() + c.receiveTarget.buffer = nil + } + return true +} + +func (e *goEngine) closeAllFlows() []*GoConn { + var unreset []*GoConn + for _, conn := range e.flows { + if !conn.dead && !e.sendReset(conn) { + unreset = append(unreset, conn) + } + e.detachConn(conn, net.ErrClosed, goDeathImmediate) + e.removeFlowKey(conn) + } + return unreset +} + +func (e *goEngine) handleKeepaliveUpdate(conn *GoConn) { + if conn.dead { + return + } + e.armKeepalive(conn) + e.rearmTimer(conn) +} + +func (e *goEngine) armKeepalive(conn *GoConn) { + conn.access.Lock() + userClosed := conn.userClosed + enabled := conn.keepaliveEnabled + idleThreshold := conn.keepaliveIdle + conn.access.Unlock() + switch { + case conn.state == goTCPFinWait2 && userClosed: + conn.keepaliveDeadline = conn.finWait2Since + int64(goFinWait2Timeout) + case enabled && !conn.handshaking(): + conn.keepaliveProbes = 0 + conn.keepaliveDeadline = conn.lastActivity + int64(idleThreshold) + default: + conn.keepaliveDeadline = 0 + } +} + +func (e *goEngine) expireKeepalive(conn *GoConn, now int64) { + conn.access.Lock() + userClosed := conn.userClosed + enabled := conn.keepaliveEnabled + idleThreshold := conn.keepaliveIdle + interval := conn.keepaliveInterval + count := conn.keepaliveCount + conn.access.Unlock() + if conn.state == goTCPFinWait2 && userClosed { + if now-conn.finWait2Since >= int64(goFinWait2Timeout) { + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: orphaned FIN_WAIT_2"), goDeathImmediate) + return + } + conn.keepaliveDeadline = conn.finWait2Since + int64(goFinWait2Timeout) + e.rearmTimer(conn) + return + } + if !enabled { + conn.keepaliveProbes = 0 + conn.keepaliveDeadline = 0 + e.rearmTimer(conn) + return + } + idle := now - conn.lastActivity + if idle < int64(idleThreshold) { + conn.keepaliveProbes = 0 + conn.keepaliveDeadline = conn.lastActivity + int64(idleThreshold) + e.rearmTimer(conn) + return + } + if conn.hasOutstanding() { + conn.keepaliveDeadline = now + int64(idleThreshold) + e.rearmTimer(conn) + return + } + if conn.keepaliveProbes >= count { + e.sendReset(conn) + e.detachConn(conn, E.Cause(syscall.ETIMEDOUT, "go: keepalive timeout"), goDeathImmediate) + return + } + conn.keepaliveProbes++ + segment := goSegment{offset: conn.sendNext() - 1, flags: header.TCPFlagAck} + e.writeConnControl(conn, &segment) + conn.keepaliveDeadline = now + int64(interval) + e.rearmTimer(conn) +} + +func (e *goEngine) reclaim() { + e.slabPool.purge() + for _, conn := range e.flows { + conn.scoreboard.trim() + } +} diff --git a/stack_go_tcp_output.go b/stack_go_tcp_output.go new file mode 100644 index 00000000..e7af6dab --- /dev/null +++ b/stack_go_tcp_output.go @@ -0,0 +1,1014 @@ +package tun + +import ( + "context" + "net" + "net/netip" + "os" + "time" + + "github.com/sagernet/sing-tun/gtcpip" + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing-tun/gtcpip/seqnum" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" +) + +const ( + goHeaderScratchSize = 128 + goGSOMaxPayload = 61440 + goEngineInlineBurst = 256 << 10 + goTransmitReadAhead = 64 << 10 +) + +const goTimestampOptionLength = 2 + header.TCPOptionTSLength + +// include/uapi/linux/virtio_net.h +const ( + goGSOTypeIPv4 uint8 = 1 + goGSOTypeIPv6 uint8 = 4 +) + +type goSegment struct { + offset uint64 + length int + flags header.TCPFlags + sackBlocks []goSackBlock + gsoSize uint16 +} + +func (c *GoConn) Write(p []byte) (int, error) { + err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) + if err != nil { + return 0, err + } + c.writeAccess.Lock() + var n int + if c.enterWriter() { + n, err = c.writeLocked(p) + c.exitWriter() + } else { + err = net.ErrClosed + } + c.writeAccess.Unlock() + c.engine.reapAfterExit() + return n, err +} + +func (c *GoConn) WriteBuffer(buffer *buf.Buffer) error { + err := c.awaitHandshake(context.Background(), c.writeDeadline.Wait()) + if err != nil { + buffer.Release() + return err + } + c.writeAccess.Lock() + if c.enterWriter() { + err = c.writeBufferLocked(buffer) + c.exitWriter() + } else { + buffer.Release() + err = net.ErrClosed + } + c.writeAccess.Unlock() + c.engine.reapAfterExit() + return err +} + +func (c *GoConn) enterWriter() bool { + c.access.Lock() + defer c.access.Unlock() + if c.dead || c.spliced || c.splicePending != nil { + return false + } + c.writing = true + return true +} + +func (c *GoConn) exitWriter() { + c.access.Lock() + c.writing = false + dead := c.dead + finRequested := c.finRequested + c.access.Unlock() + if !dead { + sent := c.sentTail.Load() + if c.bufferedTail.Load() > sent { + c.transmitPublished() + } else { + c.checkAppLimited(0, c.sendPermit.Load()&^goPermitWindowBit, sent) + } + } + if finRequested { + c.engine.postMessage(&c.closeMessage) + } +} + +func (c *GoConn) writeLocked(p []byte) (int, error) { + total := 0 + for { + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + capacity := c.transmitCapacity + c.access.Unlock() + if dead { + return total, c.closeError() + } + if writeShut { + return total, net.ErrClosed + } + select { + case <-c.writeDeadline.Wait(): + return total, os.ErrDeadlineExceeded + default: + } + if len(p) == 0 { + return total, nil + } + budget := c.writeBudget(len(p), capacity) + if budget <= 0 { + err := c.parkWriter(1) + if err != nil { + return total, err + } + continue + } + buffered := c.bufferedTail.Load() + c.transmitStore.reserve(buffered, budget) + c.transmitStore.writeAt(buffered, p[:budget]) + c.publishBuffered(buffered + uint64(budget)) + total += budget + p = p[budget:] + if len(p) == 0 { + return total, nil + } + } +} + +func (c *GoConn) writeBufferLocked(buffer *buf.Buffer) error { + length := buffer.Len() + for { + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + capacity := c.transmitCapacity + c.access.Unlock() + if dead { + buffer.Release() + return c.closeError() + } + if writeShut { + buffer.Release() + return net.ErrClosed + } + if uint64(length) > capacity { + defer buffer.Release() + _, err := c.writeLocked(buffer.Bytes()) + return err + } + select { + case <-c.writeDeadline.Wait(): + buffer.Release() + return os.ErrDeadlineExceeded + default: + } + if length == 0 { + buffer.Release() + return nil + } + budget := c.writeBudget(length, capacity) + if budget < length { + if budget > 0 && c.storeEmpty() { + defer buffer.Release() + _, err := c.writeLocked(buffer.Bytes()) + return err + } + err := c.parkWriter(length) + if err != nil { + buffer.Release() + return err + } + continue + } + buffered := c.bufferedTail.Load() + if !c.transmitStore.adopt(buffered, buffer) { + defer buffer.Release() + _, err := c.writeLocked(buffer.Bytes()) + return err + } + c.publishBuffered(buffered + uint64(length)) + return nil + } +} + +func (c *GoConn) FrontHeadroom() int { + return header.IPv6MinimumSize + header.TCPMinimumSize + goTimestampOptionLength +} + +func (c *GoConn) WriterMTU() int { + return int(c.gsoMaxSize(c.effectiveMSS.Load())) +} + +func (c *GoConn) publishBuffered(tail uint64) { + c.bufferedTail.Store(tail) + c.transmitPublished() +} + +func (c *GoConn) transmitPublished() { + if goEngineTransmits { + c.engine.postMessage(&c.transmitMessage) + return + } + for { + if !c.transmitAccess.TryLock() { + c.transmitSignal.notify() + return + } + _, result, armed := c.transmitLoop(false, 0) + c.transmitAccess.Unlock() + c.afterTransmit(armed) + switch result { + case goTransmitBlocked: + c.wakeTransmitterGoroutine() + return + case goTransmitPaced, goTransmitThrottled: + return + } + if c.closed() || c.nextTransmitLength(c.effectiveMSS.Load()) == 0 { + return + } + } +} + +func (c *GoConn) afterTransmit(armed bool) { + if armed { + c.engine.postMessage(&c.retransmitMessage) + } + c.access.Lock() + finRequested := c.finRequested + c.access.Unlock() + if finRequested && c.sentTail.Load() == c.bufferedTail.Load() { + c.engine.postMessage(&c.closeMessage) + } +} + +func (e *goEngine) transmitOnEngine(conn *GoConn, budget int) { + if !conn.transmitAccess.TryLock() { + conn.transmitSignal.notify() + return + } + written, result, armed := conn.transmitLoop(false, budget) + conn.transmitAccess.Unlock() + if budget > 0 { + e.inlineTransmitBudget = budget - written + } + if conn.retransmitDeadline == 0 && (armed || conn.hasOutstanding()) { + e.rearmRetransmit(conn) + } + e.maybeSendFin(conn) + switch result { + case goTransmitBlocked: + e.handleTransmitBlocked(conn) + case goTransmitDone: + if !conn.dead && conn.nextTransmitLength(conn.effectiveMSS.Load()) != 0 { + e.postMessage(&conn.transmitMessage) + } + } +} + +func (e *goEngine) handleTransmitRequest(conn *GoConn) { + if conn.dead { + return + } + if conn.splice != nil { + e.spliceResume(conn) + return + } + e.transmitOnEngine(conn, 0) +} + +func (e *goEngine) retryBlockedOnEngine(conn *GoConn) { + if conn.dead { + return + } + if !conn.transmitAccess.TryLock() { + conn.transmitSignal.notify() + return + } + if !conn.blockedValid { + conn.transmitAccess.Unlock() + return + } + segment := conn.blockedSegment + end := segment.offset + uint64(segment.length) + err := conn.transmitFrame(&segment) + switch err { + case nil: + case errGoTransmitBlocked: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) + conn.transmitAccess.Unlock() + e.handleTransmitBlocked(conn) + return + case errGoFrameDropped: + conn.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmitted(end) + conn.transmitAccess.Unlock() + e.handleDroppedFrames(conn) + return + default: + conn.transmitAccess.Unlock() + e.abortConn(conn, E.Cause(err, "go: write tun")) + return + } + if conn.congestion.pacing { + conn.advancePacing(e.coarseTime.Load(), segment.length) + } + conn.blockedValid = false + conn.blockedFrame = nil + conn.transmitted(end) + conn.transmitAccess.Unlock() + if conn.splice != nil { + e.spliceResume(conn) + if conn.splice != nil && conn.splice.downloadClosed { + e.maybeSendFin(conn) + e.spliceMaybeFinish(conn) + } + return + } + e.transmitOnEngine(conn, 0) +} + +func (c *GoConn) wakeTransmitterGoroutine() { + c.access.Lock() + spawn := !c.transmitterRunning && !c.dead + if spawn { + c.transmitterRunning = true + } + c.access.Unlock() + if spawn { + go c.runTransmitter() + return + } + c.transmitSignal.notify() +} + +func (c *GoConn) hasDataWaiting() bool { + return c.bufferedTail.Load() > c.sentTail.Load() || c.writerWaiting.Load() != 0 +} + +func (c *GoConn) runTransmitter() { + defer func() { + c.access.Lock() + c.transmitterRunning = false + c.access.Unlock() + c.engine.reapAfterExit() + }() + for { + if c.closed() { + return + } + if c.transmitAccess.TryLock() { + _, _, armed := c.transmitLoop(true, 0) + c.transmitAccess.Unlock() + c.afterTransmit(armed) + } + select { + case <-c.transmitSignal: + case <-c.closeSignal: + return + } + } +} + +func (c *GoConn) nextTransmitLength(mss uint32) int { + sent := c.sentTail.Load() + pending := c.bufferedTail.Load() - sent + if pending == 0 { + return 0 + } + permit := c.sendPermit.Load() + if permit&^goPermitWindowBit <= sent { + if permit&goPermitWindowBit != 0 { + c.windowLimitedSince.Store(true) + } + return 0 + } + permit &^= goPermitWindowBit + packets := int32(c.sendPacketPermit.Load() - c.dataSegmentsOut.Load()) + if packets <= 0 { + c.windowLimitedSince.Store(true) + return 0 + } + length := int(min(pending, permit-sent, uint64(packets)*uint64(mss))) + if pending < uint64(mss) && sent > c.sendUnacked.Load() { + c.access.Lock() + held := c.writing || c.nagle && !c.finRequested + c.access.Unlock() + if held { + return 0 + } + } + // Windows delays the ACK of a lone segment by 40 ms and its receive window auto-tuning never + // grows while every window is drained by one segment, so a window narrower than two segments is + // always covered by two frames. + allowance := permit - c.sendUnacked.Load() + if allowance < 2*uint64(mss) && packets > 1 { + length = min(length, int((allowance+1)/2)) + } + return c.frameLength(length, mss) +} + +func (c *GoConn) writeBudget(pending int, capacity uint64) int { + released := c.sendReleased.Load() + buffered := c.bufferedTail.Load() + stored := buffered - released + if stored >= capacity { + return 0 + } + if stored >= goTransmitReadAhead { + share := c.engine.slabPool.shareBytes() + if share > 0 && c.slabHolder.heldBytes() >= share { + return 0 + } + } + unacked := c.sendUnacked.Load() + permit := max(c.sendPermit.Load()&^goPermitWindowBit, unacked) + limit := min(permit+max(permit-unacked, goTransmitReadAhead), released+capacity) + budget := int64(limit) - int64(buffered) + if pending > 0 { + budget = min(budget, int64(pending)) + } + if budget <= 0 { + return 0 + } + return int(budget) +} + +func (c *GoConn) storeEmpty() bool { + return c.bufferedTail.Load() == c.sendReleased.Load() +} + +func (c *GoConn) loadTransmitCapacity() uint64 { + c.access.Lock() + defer c.access.Unlock() + return c.transmitCapacity +} + +func (c *GoConn) writeReady(required int) bool { + budget := c.writeBudget(required, c.loadTransmitCapacity()) + return budget >= required || (budget > 0 && c.storeEmpty()) +} + +func (c *GoConn) parkWriter(required int) error { + c.writerWaiting.Store(int32(required)) + c.writeSignal.drain() + if c.writeReady(required) { + c.writerWaiting.Store(0) + return nil + } + c.access.Lock() + dead := c.dead + writeShut := c.writeShut + c.access.Unlock() + if dead { + c.writerWaiting.Store(0) + return c.closeError() + } + if writeShut { + c.writerWaiting.Store(0) + return net.ErrClosed + } + deadlineSignal := c.writeDeadline.Wait() + select { + case <-c.writeSignal: + case <-deadlineSignal: + c.writerWaiting.Store(0) + return os.ErrDeadlineExceeded + case <-c.closeSignal: + c.writerWaiting.Store(0) + return c.closeError() + } + c.writerWaiting.Store(0) + if c.closed() { + return c.closeError() + } + return nil +} + +func (c *GoConn) parkTransmitBlocked() error { + c.engine.postMessage(&c.blockedMessage) + select { + case <-c.transmitSignal: + case <-c.closeSignal: + return c.closeError() + } + if c.closed() { + return c.closeError() + } + return nil +} + +type goTransmitResult uint8 + +const ( + goTransmitDone goTransmitResult = iota + goTransmitBlocked + goTransmitPaced + goTransmitThrottled +) + +func (c *GoConn) transmitLoop(transmitter bool, budget int) (int, goTransmitResult, bool) { + if transmitter { + defer c.engine.platformIO.flush() + } + written := 0 + armed := false + if c.blockedValid { + if !transmitter { + return written, goTransmitBlocked, armed + } + if !c.retryBlocked() { + return written, goTransmitDone, armed + } + } + paced := c.congestion.pacing + result := goTransmitDone + for !c.closed() { + if budget > 0 && written >= budget { + break + } + mss := c.effectiveMSS.Load() + length := c.nextTransmitLength(mss) + if length == 0 { + sent := c.sentTail.Load() + pending := c.bufferedTail.Load() - sent + permit := c.sendPermit.Load() + c.checkAppLimited(pending, permit&^goPermitWindowBit, sent) + if pending > 0 && permit&goPermitWindowBit == 0 && sent == c.sendUnacked.Load() { + armed = true + } + break + } + var now int64 + if paced { + now = int64(time.Since(c.engine.epoch)) + if stamp := c.pacingStamp.Load(); c.dataSegmentsOut.Load() >= goPacingUnpacedSegments && stamp > now+goWheelTick { + c.requestPacing(stamp - goWheelTick) + result = goTransmitPaced + break + } + } + if c.queueThrottle(length) { + result = goTransmitThrottled + break + } + written += length + sent := c.sentTail.Load() + segment := goSegment{offset: sent, length: length, flags: header.TCPFlagAck | header.TCPFlagPsh} + if length > int(mss) { + segment.gsoSize = uint16(mss) + } + var flags uint16 + unacked := c.sendUnacked.Load() + flightEmpty := sent == unacked + if flightEmpty { + flags |= goDescriptorTxStart + armed = true + } + if !paced { + now = int64(time.Since(c.engine.epoch)) + } + c.pushDescriptors(sent, length, mss, c.stamp(now), flags, flightEmpty) + c.sentTail.Store(sent + uint64(length)) + if !armed && c.sendUnacked.Load() == sent { + armed = true + } + creditBase := c.packetCreditBase.Load() + flight := uint64(c.dataSegmentsOut.Load() - creditBase) + if flight > c.peakFlight.Load() { + c.peakFlight.Store(flight) + } + permit := c.sendPermit.Load() + if int32(c.sendPacketPermit.Load()-c.dataSegmentsOut.Load()) <= 0 || + sent+uint64(length) >= permit&^goPermitWindowBit && permit&goPermitWindowBit != 0 { + c.windowLimitedSince.Store(true) + } + err := c.transmitFrame(&segment) + if paced && err != errGoTransmitBlocked { + c.advancePacing(now, length) + } + switch err { + case nil: + c.transmitted(sent + uint64(length)) + case errGoTransmitBlocked: + c.amendDescriptors(segment.offset, segment.length, goDescriptorNoSample) + c.blockedSegment = segment + c.blockedValid = true + if !transmitter { + return written, goTransmitBlocked, armed + } + if !c.retryBlocked() { + return written, goTransmitDone, armed + } + case errGoFrameDropped: + c.amendDescriptors(segment.offset, segment.length, goDescriptorDropped) + c.transmitted(sent + uint64(length)) + c.engine.postMessage(&c.droppedMessage) + default: + if !c.closed() { + c.fail(E.Cause(err, "go: write tun")) + } + return written, goTransmitDone, armed + } + } + return written, result, armed +} + +func (c *GoConn) retryBlocked() bool { + for { + parkErr := c.parkTransmitBlocked() + if parkErr != nil { + return false + } + end := c.blockedSegment.offset + uint64(c.blockedSegment.length) + err := c.transmitFrame(&c.blockedSegment) + switch err { + case nil: + case errGoTransmitBlocked: + c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorNoSample) + continue + case errGoFrameDropped: + c.amendDescriptors(c.blockedSegment.offset, c.blockedSegment.length, goDescriptorDropped) + default: + if !c.closed() { + c.fail(E.Cause(err, "go: write tun")) + } + return false + } + if c.congestion.pacing { + c.advancePacing(int64(time.Since(c.engine.epoch)), c.blockedSegment.length) + } + c.blockedValid = false + c.transmitted(end) + if err == errGoFrameDropped { + c.engine.postMessage(&c.droppedMessage) + } + return true + } +} + +func (c *GoConn) amendDescriptors(offset uint64, length int, flags uint16) { + c.descriptors.push(goSentDescriptor{endOffset: offset + uint64(length), sentAt: int32(length), flags: goDescriptorAmend | flags}) +} + +func (c *GoConn) pushDescriptors(offset uint64, length int, mss uint32, sentAt int32, flags uint16, flightEmpty bool) { + delivered, deliveredStamp, firstSentStamp, appLimited := c.snapshotDelivery(sentAt, flightEmpty) + flags |= goDescriptorRated + if appLimited { + flags |= goDescriptorAppLimited + } + step := int(mss) + for length > 0 { + span := min(length, step) + offset += uint64(span) + length -= span + c.dataSegmentsOut.Add(1) + c.descriptors.push(goSentDescriptor{ + endOffset: offset, + sentAt: sentAt, + deliveredStamp: deliveredStamp, + firstSentStamp: firstSentStamp, + delivered: delivered, + flags: flags, + }) + flags &^= goDescriptorTxStart + } +} + +func (c *GoConn) frameLength(pending int, mss uint32) int { + limit := int(mss) + if c.engine.platformIO.transmitSegmentOffload() { + limit = int(c.gsoMaxSize(mss)) + if frameLimit := int(c.frameLimit.Load()); frameLimit > 0 { + limit = min(limit, max(frameLimit, int(mss))) + } + } + return min(pending, limit) +} + +func (c *GoConn) advancePacing(now int64, length int) { + rate := c.pacingRate.Load() + if rate == 0 || c.dataSegmentsOut.Load() < goPacingUnpacedSegments { + c.pacingStamp.Store(now) + return + } + for { + prior := c.pacingStamp.Load() + stamp := max(prior, now) + lengthNanos := int64(uint64(length) * uint64(time.Second) / rate) + credit := stamp - prior + lengthNanos -= min(lengthNanos/2, credit) + if c.pacingStamp.CompareAndSwap(prior, stamp+lengthNanos) { + return + } + } +} + +const goQueueLimitShift = 8 + +func (c *GoConn) queueThrottle(length int) bool { + limit := max(2*int64(length), int64(c.pacingRate.Load()>>goQueueLimitShift)) + if c.queuedBytes.Load()+int64(length) <= limit || c.engine.platformIO.transmitBacklogBelowBatch(c) { + return false + } + if c.sentTail.Load() == c.sendUnacked.Load() { + return false + } + c.queueThrottled.Store(true) + if c.queuedBytes.Load()+int64(length) <= limit { + c.queueThrottled.Store(false) + return false + } + return true +} + +func (c *GoConn) frameEnqueued(length int) { + c.queuedBytes.Add(int64(length)) +} + +func (c *GoConn) frameDequeued(length int, segmentEnd uint64) { + c.queuedBytes.Add(-int64(length)) + if segmentEnd > c.departedTail.Load() { + c.departedTail.Store(segmentEnd) + } + if c.queueThrottled.Swap(false) { + c.engine.postMessage(&c.throttleMessage) + } +} + +func (c *GoConn) flightHeadQueued() bool { + return c.queuedBytes.Load() > 0 && c.sendUnacked.Load() >= c.departedTail.Load() +} + +func (c *GoConn) requestPacing(deadline int64) { + c.pacingRequest.Store(deadline) + c.engine.postMessage(&c.pacingMessage) +} + +func (c *GoConn) transmitFrame(segment *goSegment) error { + var ( + frame [][]byte + meta ForwardFrameMeta + ) + if c.blockedValid && c.blockedFrame != nil && segment.offset == c.blockedSegment.offset && segment.length == c.blockedSegment.length { + frame, meta = c.blockedFrame, c.blockedMeta + } else { + offload := c.engine.platformIO.transmitChecksumOffload() + buffer, payload, contiguous := c.transmitStore.leadingRun(segment.offset, segment.length) + if contiguous && buffer.Start() >= c.FrontHeadroom() { + var packet []byte + packet, meta = c.buildContiguousFrame(buffer, payload, segment, offload) + c.transmitSegments = append(c.transmitSegments[:0], packet) + frame = c.transmitSegments + } else { + frame, meta = c.buildFrame(c.transmitScratch[:], c.transmitSegments[:0], segment, &c.transmitStore, offload, false) + c.transmitSegments = frame + } + } + err := c.engine.platformIO.writeData(frame, meta, c, segment.offset+uint64(segment.length)) + switch err { + case nil, errGoFrameDropped: + c.blockedFrame = nil + clear(c.transmitSegments[:cap(c.transmitSegments)]) + case errGoTransmitBlocked: + c.blockedFrame = frame + c.blockedMeta = meta + } + return err +} + +type goFrameLayout struct { + ipHeaderLength int + tcpHeaderLength int + timestampLength int + sackBlocks []goSackBlock +} + +func (l *goFrameLayout) headerLength() int { + return l.ipHeaderLength + l.tcpHeaderLength +} + +func (c *GoConn) frameLayout(segment *goSegment) goFrameLayout { + var layout goFrameLayout + if c.timestampsEnabled { + layout.timestampLength = goTimestampOptionLength + } + layout.sackBlocks = segment.sackBlocks + if len(layout.sackBlocks) > goMaxSackBlocksSent { + layout.sackBlocks = layout.sackBlocks[:goMaxSackBlocksSent] + } + sackLength := 0 + if len(layout.sackBlocks) > 0 { + sackLength = 4 + 8*len(layout.sackBlocks) + } + layout.tcpHeaderLength = header.TCPMinimumSize + layout.timestampLength + sackLength + layout.ipHeaderLength = goNetworkHeaderLength(c.ipVersion) + return layout +} + +func (c *GoConn) encodeHeaders(packet []byte, layout *goFrameLayout, segment *goSegment, control bool) uint16 { + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + receiveAck := c.receiveNextAck.Load() + edge := c.receiveEdge.Load() + var window uint16 + if edge > receiveAck { + window = uint16(min((edge-receiveAck)>>c.localWindowShift, 0xffff)) + } + if control { + c.sentEdge = max(c.sentEdge, edge) + } else if edge > c.dataSentEdge.Load() { + c.dataSentEdge.Store(edge) + } + tcpHdr.Encode(&header.TCPFields{ + SrcPort: c.local.Port, + DstPort: c.peer.Port, + SeqNum: uint32(uint64(c.sendISN) + segment.offset), + AckNum: uint32(uint64(c.clientISN) + receiveAck), + DataOffset: uint8(layout.tcpHeaderLength), + Flags: segment.flags, + WindowSize: window, + }) + options := packet[layout.ipHeaderLength+header.TCPMinimumSize:] + if layout.timestampLength > 0 { + goEncodeTimestampOption(options, goTimestampAt(int64(time.Since(c.engine.epoch))), c.tsRecent.Load()) + } + if len(layout.sackBlocks) > 0 { + goEncodeSackOption(options[layout.timestampLength:], layout.sackBlocks, c.clientISN) + } + return goEncodeNetworkHeader(packet, c.ipVersion, header.TCPProtocolNumber, c.local.Addr, c.peer.Addr, layout.tcpHeaderLength+segment.length, uint16(c.ident.Add(1))) +} + +func goNetworkHeaderLength(ipVersion uint8) int { + if ipVersion == 4 { + return header.IPv4MinimumSize + } + return header.IPv6MinimumSize +} + +func goEncodeNetworkHeader(packet []byte, ipVersion uint8, protocol tcpip.TransportProtocolNumber, source netip.Addr, destination netip.Addr, transportLength int, ident uint16) uint16 { + if ipVersion == 4 { + ipHdr := header.IPv4(packet) + ipHdr.Encode(&header.IPv4Fields{ + TotalLength: uint16(header.IPv4MinimumSize + transportLength), + ID: ident, + TTL: synthesizedTTL, + Protocol: uint8(protocol), + SrcAddr: source, + DstAddr: destination, + }) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + return header.PseudoHeaderChecksum(protocol, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(transportLength)) + } + ipHdr := header.IPv6(packet) + ipHdr.Encode(&header.IPv6Fields{ + PayloadLength: uint16(transportLength), + TransportProtocol: protocol, + HopLimit: synthesizedTTL, + SrcAddr: source, + DstAddr: destination, + }) + return header.PseudoHeaderChecksum(protocol, ipHdr.SourceAddressSlice(), ipHdr.DestinationAddressSlice(), uint16(transportLength)) +} + +func (c *GoConn) offloadMeta(tcpHdr header.TCP, pseudoSum uint16, layout *goFrameLayout, segment *goSegment) ForwardFrameMeta { + tcpHdr.SetChecksum(pseudoSum) + meta := ForwardFrameMeta{ + needsChecksum: true, + checksumStart: uint16(layout.ipHeaderLength), + checksumOffset: header.TCPChecksumOffset, + } + if segment.gsoSize > 0 { + meta.gsoSize = segment.gsoSize + meta.gsoType = goGSOType(c.ipVersion) + } + return meta +} + +func (c *GoConn) buildFrame(scratch []byte, segments [][]byte, segment *goSegment, store *goTransmitStore, offload bool, control bool) ([][]byte, ForwardFrameMeta) { + layout := c.frameLayout(segment) + packet := scratch[:layout.headerLength()] + pseudoSum := c.encodeHeaders(packet, &layout, segment, control) + segments = append(segments, packet) + if segment.length > 0 { + segments = store.appendRuns(segments, segment.offset, segment.length) + } + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + if offload { + return segments, c.offloadMeta(tcpHdr, pseudoSum, &layout, segment) + } + tcpHdr.SetChecksum(0) + sum := checksum.Checksum(tcpHdr, pseudoSum) + sum = goChecksumRuns(sum, segments[1:], layout.tcpHeaderLength%2 == 1) + tcpHdr.SetChecksum(^sum) + return segments, ForwardFrameMeta{} +} + +func (c *GoConn) buildContiguousFrame(buffer *buf.Buffer, payload []byte, segment *goSegment, offload bool) ([]byte, ForwardFrameMeta) { + layout := c.frameLayout(segment) + headerLength := layout.headerLength() + packet := buffer.ExtendHeader(headerLength) + packet = packet[:headerLength+len(payload)] + pseudoSum := c.encodeHeaders(packet[:headerLength], &layout, segment, false) + tcpHdr := header.TCP(packet[layout.ipHeaderLength:]) + if offload { + return packet, c.offloadMeta(tcpHdr, pseudoSum, &layout, segment) + } + tcpHdr.SetChecksum(0) + tcpHdr.SetChecksum(^checksum.Checksum(packet[layout.ipHeaderLength:], pseudoSum)) + return packet, ForwardFrameMeta{} +} + +func goChecksumRuns(initial uint16, runs [][]byte, odd bool) uint16 { + sum := initial + for _, run := range runs { + if odd { + partial := checksum.Checksum(run, 0) + sum = checksum.Combine(sum, partial>>8|partial<<8) + } else { + sum = checksum.Checksum(run, sum) + } + if len(run)%2 == 1 { + odd = !odd + } + } + return sum +} + +func goTimestampAt(now int64) uint32 { + return uint32(now >> goTimestampShift) +} + +func goEncodeTimestampOption(options []byte, timestampValue uint32, timestampEcho uint32) { + options[0] = header.TCPOptionNOP + options[1] = header.TCPOptionNOP + header.EncodeTSOption(timestampValue, timestampEcho, options[2:goTimestampOptionLength]) +} + +func goEncodeSackOption(options []byte, blocks []goSackBlock, base uint32) { + var sackBlocks [goMaxSackBlocksSent]header.SACKBlock + for index, block := range blocks { + sackBlocks[index] = header.SACKBlock{ + Start: seqnum.Value(uint64(base) + block.start), + End: seqnum.Value(uint64(base) + block.end), + } + } + options[0] = header.TCPOptionNOP + options[1] = header.TCPOptionNOP + optionsLength := 2 + header.EncodeSACKBlocks(sackBlocks[:len(blocks)], options[2:]) + header.AddTCPOptionPadding(options, optionsLength) +} + +func (c *GoConn) buildHandshake(localMSS uint16, windowScale bool, sackPermitted bool, flags header.TCPFlags, ackNumber uint32) { + ipHeaderLength := goNetworkHeaderLength(c.ipVersion) + optionsStorage := c.handshakeImage[ipHeaderLength+header.TCPMinimumSize:] + optionsLength := header.EncodeMSSOption(uint32(localMSS), optionsStorage) + if windowScale { + optionsLength += header.EncodeWSOption(int(c.localWindowShift), optionsStorage[optionsLength:]) + } + if sackPermitted { + optionsLength += header.EncodeSACKPermittedOption(optionsStorage[optionsLength:]) + } + if c.timestampsEnabled { + optionsLength += header.EncodeTSOption(goTimestampAt(int64(time.Since(c.engine.epoch))), c.tsRecent.Load(), optionsStorage[optionsLength:]) + } + optionsLength += header.AddTCPOptionPadding(optionsStorage, optionsLength) + tcpHeaderLength := header.TCPMinimumSize + optionsLength + totalLength := ipHeaderLength + tcpHeaderLength + c.handshakeLength = uint8(totalLength) + packet := c.handshakeImage[:totalLength] + tcpHdr := header.TCP(packet[ipHeaderLength:]) + tcpHdr.Encode(&header.TCPFields{ + SrcPort: c.local.Port, + DstPort: c.peer.Port, + SeqNum: c.sendISN, + AckNum: ackNumber, + DataOffset: uint8(tcpHeaderLength), + Flags: flags, + WindowSize: uint16(min(c.receiveCapacity, 0xffff)), + }) + pseudoSum := goEncodeNetworkHeader(packet, c.ipVersion, header.TCPProtocolNumber, c.local.Addr, c.peer.Addr, tcpHeaderLength, uint16(c.ident.Add(1))) + tcpHdr.SetChecksum(0) + tcpHdr.SetChecksum(^checksum.Checksum(tcpHdr, pseudoSum)) +} + +var errGoTransmitBlocked = E.New("go: transmit blocked") + +func goGSOType(ipVersion uint8) uint8 { + if ipVersion == 4 { + return goGSOTypeIPv4 + } + return goGSOTypeIPv6 +} diff --git a/stack_go_tcp_rate.go b/stack_go_tcp_rate.go new file mode 100644 index 00000000..848528e5 --- /dev/null +++ b/stack_go_tcp_rate.go @@ -0,0 +1,126 @@ +package tun + +import "time" + +// Mirrors the delivery rate estimator of net/ipv4/tcp_input.c (formerly tcp_rate.c). + +func goStampMicros(units int32) int64 { + return int64(units) * goTimeUnit / int64(time.Microsecond) +} + +func goStampAfter(stamp1 int32, stamp2 int32, offset1 uint64, offset2 uint64) bool { + delta := stamp1 - stamp2 + return delta > 0 || (delta == 0 && offset1 > offset2) +} + +func goPackDelivery(delivered uint32, stamp int32) uint64 { + return uint64(delivered)<<32 | uint64(uint32(stamp)) +} + +func goUnpackDelivery(state uint64) (uint32, int32) { + return uint32(state >> 32), int32(uint32(state)) +} + +func (c *GoConn) snapshotDelivery(sentAt int32, flightEmpty bool) (uint32, int32, int32, bool) { + if flightEmpty { + for { + state := c.deliveryState.Load() + delivered, _ := goUnpackDelivery(state) + if c.deliveryState.CompareAndSwap(state, goPackDelivery(delivered, sentAt)) { + break + } + } + c.firstSentStamp.Store(sentAt) + } + delivered, deliveredStamp := goUnpackDelivery(c.deliveryState.Load()) + return delivered, deliveredStamp, c.firstSentStamp.Load(), c.appLimited.Load() != 0 +} + +func (e *goEngine) rateDescriptorDelivered(conn *GoConn, descriptor *goSentDescriptor, sample *goRateSample) { + if descriptor.flags&goDescriptorRated == 0 || descriptor.flags&(goDescriptorNoSample|goDescriptorDropped) != 0 { + return + } + if !sample.valid || goStampAfter(descriptor.sentAt, conn.firstSentStamp.Load(), descriptor.endOffset, sample.lastEndOffset) { + sample.valid = true + sample.priorDelivered = descriptor.delivered + sample.priorStamp = descriptor.deliveredStamp + sample.appLimited = descriptor.flags&goDescriptorAppLimited != 0 + sample.retransmitted = descriptor.flags&goDescriptorRetransmitted != 0 + sample.lastEndOffset = descriptor.endOffset + conn.firstSentStamp.Store(descriptor.sentAt) + sample.sendIntervalMicros = goStampMicros(descriptor.sentAt - descriptor.firstSentStamp) + } + if descriptor.flags&goDescriptorSacked != 0 { + descriptor.flags &^= goDescriptorRated + } +} + +func (e *goEngine) rateGenerate(conn *GoConn, sample *goRateSample, delivered uint32, lost uint32, now int64) { + appLimited := conn.appLimited.Load() + if appLimited != 0 && int32(conn.delivered-appLimited) > 0 { + conn.appLimited.CompareAndSwap(appLimited, 0) + } + stamp := conn.stamp(now) + if delivered > 0 { + conn.deliveredTime = now + conn.deliveredStamp = stamp + conn.publishDelivery() + } + sample.ackedSacked = delivered + sample.losses = lost + if !sample.valid || conn.sackReneging { + sample.delivered = -1 + sample.intervalMicros = -1 + return + } + sample.delivered = int64(conn.delivered - sample.priorDelivered) + sendMicros := sample.sendIntervalMicros + receiveMicros := goStampMicros(stamp - sample.priorStamp) + sample.intervalMicros = max(sendMicros, receiveMicros) + sample.receiveIntervalMicros = receiveMicros + if sample.intervalMicros < int64(conn.roundTripMin.get()) { + sample.intervalMicros = -1 + return + } + if !sample.appLimited || uint64(sample.delivered)*uint64(conn.rateIntervalMicros) >= uint64(conn.rateDelivered)*uint64(sample.intervalMicros) { + conn.rateDelivered = uint32(sample.delivered) + conn.rateIntervalMicros = uint32(sample.intervalMicros) + conn.rateAppLimited = sample.appLimited + } +} + +func (c *GoConn) countDelivered(packets uint32) { + c.delivered += packets + c.deliveredTime = c.engine.coarseTime.Load() + c.deliveredStamp = c.stamp(c.deliveredTime) + c.publishDelivery() +} + +func (c *GoConn) publishDelivery() { + for { + state := c.deliveryState.Load() + _, stamp := goUnpackDelivery(state) + if c.deliveredStamp-stamp > 0 { + stamp = c.deliveredStamp + } + if c.deliveryState.CompareAndSwap(state, goPackDelivery(c.delivered, stamp)) { + return + } + } +} + +func (c *GoConn) checkAppLimited(pending uint64, permit uint64, sent uint64) { + segments := c.dataSegmentsOut.Load() + if pending >= uint64(c.effectiveMSS.Load()) || permit <= sent || int32(c.sendPacketPermit.Load()-segments) <= 0 || c.writerWaiting.Load() != 0 { + return + } + c.access.Lock() + writing := c.writing + c.access.Unlock() + if writing { + return + } + inFlight := segments - c.packetCreditBase.Load() + delivered, _ := goUnpackDelivery(c.deliveryState.Load()) + c.appLimited.Store(max(delivered+inFlight, 1)) +} diff --git a/stack_go_udp.go b/stack_go_udp.go new file mode 100644 index 00000000..86760cd1 --- /dev/null +++ b/stack_go_udp.go @@ -0,0 +1,527 @@ +package tun + +import ( + "context" + "io" + "net/netip" + "os" + "runtime" + "slices" + "sync" + "sync/atomic" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +const ( + goPacketBatchSize = 32 + goUDPGSOType = 5 + goUDPChecksumOffset = 6 + // UDP_MAX_SEGMENTS in include/linux/udp.h bounds the segments of one USO packet. + goGSOMaxSegments = 128 +) + +type goUDPFrame struct { + header [header.IPv6MinimumSize + header.UDPMinimumSize]byte + length int + payload []byte + meta ForwardFrameMeta +} + +var goUDPFramePool = sync.Pool{ + New: func() any { + return new([goPacketBatchSize]goUDPFrame) + }, +} + +type goUDPUserData struct { + engine *goEngine + owner *goEngine + key udpNatSessionKey + packet []byte + meta ForwardFrameMeta + ipVersion uint8 + created *GoPacketConn +} + +func (e *goEngine) processUDPSegments(frame *goFrame) { + meta := frame.meta + packet := frame.buffer.Bytes() + parsed, parsedOK := parseForwardPacket(packet) + headerLength := int(meta.checksumStart) + header.UDPMinimumSize + if !parsedOK || parsed.protocol != uint8(header.UDPProtocolNumber) || parsed.fragment || meta.gsoSize == 0 || headerLength > len(packet) { + return + } + count := (len(packet) - headerLength + int(meta.gsoSize) - 1) / int(meta.gsoSize) + mtu := e.platformIO.mtu() + if count == 0 || count > goGSOMaxSegments || headerLength+int(meta.gsoSize) > mtu { + return + } + options := *e.packetReadOptions.Load() + e.gsoSlots.configure(goGSOMaxSegments, mtu+options.FrontHeadroom+options.RearHeadroom) + e.gsoSlots.wake(count) + for index := range count { + buffer := e.gsoSlots.slot(index) + buffer.Reset() + buffer.Resize(options.FrontHeadroom, 0) + buffer.Reserve(options.RearHeadroom) + e.gsoSegments[index] = buffer.FreeBytes() + } + count, err := GSOSplit(packet, GSOOptions{ + GSOType: GSOUDPL4, HdrLen: uint16(headerLength), GSOSize: meta.gsoSize, + CsumStart: meta.checksumStart, CsumOffset: meta.checksumOffset, NeedsCsum: meta.needsChecksum, + }, e.gsoSegments[:count], e.gsoSizes[:count], 0) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: split UDP segments")) + return + } + for index := range count { + buffer := e.gsoSlots.slot(index) + buffer.Truncate(e.gsoSizes[index]) + options.PostReturn(buffer) + e.gsoFrame = goFrame{buffer: buffer} + e.processFrame(&e.gsoFrame) + } + e.flushPacketUploads() +} + +func (e *goEngine) demuxUDP(packetBuffer *buf.Buffer, meta ForwardFrameMeta, parsed *forwardPacket) { + packet := packetBuffer.Bytes() + if len(parsed.transport) < header.UDPMinimumSize { + return + } + if e.udpNat == nil { + meta.completeChecksum(packet) + reply, ok := buildReject(parsed, 0) + if ok { + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write port unreachable")) + } + } + return + } + switch parsed.verdict.Action { + case ActionReject: + meta.completeChecksum(packet) + reply, ok := BuildICMPError(packet, ICMPErrorNoRoute, netip.Addr{}, 0, 0) + if ok { + err := e.platformIO.writeFrame(e.singleFrame(reply), ForwardFrameMeta{}) + if err != nil { + e.stack.logger.Trace(E.Cause(err, "go: write unreachable")) + } + } + return + case ActionDrop: + return + case ActionHijackDNS: + e.dispatcher.hijackDNSPacket(parsed) + return + } + payload := header.UDP(parsed.transport).Payload() + source := M.SocksaddrFromNetIP(parsed.source) + destination := M.SocksaddrFromNetIP(parsed.destination) + key := e.udpNat.sessionKey(source, destination) + userData := &e.udpUserData + *userData = goUDPUserData{ + engine: e, + key: key, + packet: packet, + meta: meta, + ipVersion: parsed.ipVersion, + } + conn, ok := e.udpNat.getOrCreate(key, source, destination, userData, func(conn *UDPNatConn) { + e.attachUDPSession(conn, userData.created, &parsed.verdict) + userData.created = nil + }) + if !ok { + if userData.owner != nil { + userData.owner.inject(packet, meta) + } + return + } + if writer, isGoPacketConn := conn.writer.(*GoPacketConn); isGoPacketConn { + if writer.engine != e { + writer.engine.inject(packet, meta) + return + } + if writer.splice != nil { + if len(e.packetFlows) < goReadBatch { + e.packetFlows[parsed.flowKey()] = writer + } + e.packetSpliceInput(writer, packetBuffer, parsed) + return + } + trackerPointer := writer.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountForward(len(packet)) + } + } + readWaitOptions := conn.loadReadWaitOptions() + buffer := readWaitOptions.NewBufferSize(len(payload)) + buffer.Write(payload) + readWaitOptions.PostReturn(buffer) + conn.enqueue(buffer, destination) + e.wokeHandlerThisBurst = true +} + +func (e *goEngine) packetSpliceInput(writer *GoPacketConn, packetBuffer *buf.Buffer, parsed *forwardPacket) { + trackerPointer := writer.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountForward(packetBuffer.Len()) + } + headerLength := header.IPv6MinimumSize + if parsed.ipVersion == 4 { + headerLength = int(header.IPv4(parsed.network).HeaderLength()) + } + payloadLength := len(header.UDP(parsed.transport).Payload()) + packetBuffer.Advance(headerLength + header.UDPMinimumSize) + packetBuffer.Truncate(payloadLength) + e.packetSpliceUpload(writer, packetBuffer, M.SocksaddrFromNetIP(parsed.destination)) +} + +func (e *goEngine) attachUDPSession(conn *UDPNatConn, writer *GoPacketConn, verdict *FlowVerdict) { + writer.conn.Store(conn) + if verdict.UDPTimeout > 0 { + conn.SetTimeout(verdict.UDPTimeout) + } + if verdict.NewTracker != nil { + tracker := verdict.NewTracker() + if tracker != nil { + writer.tracker.Store(&tracker) + tracker.AttachFlow(writer) + } + } +} + +func (s *Go) prepareUDPConnection(source M.Socksaddr, destination M.Socksaddr, userData any) (bool, context.Context, N.PacketWriter, N.CloseHandlerFunc) { + data := userData.(*goUDPUserData) + directory := &s.directory + if directory.udpFlows != nil { + directory.access.Lock() + defer directory.access.Unlock() + if owner := directory.udpFlows[data.key]; owner != nil { + conn := owner.conn.Load() + if conn == nil || !conn.isClosed() { + data.owner = owner.engine + return false, nil, nil, nil + } + } + } + writer := &GoPacketConn{ + engine: data.engine, + key: data.key, + platformIO: data.engine.platformIO, + checksumOffload: data.engine.platformIO.transmitChecksumOffload(), + snapshot: slices.Clone(data.packet), + snapshotMeta: data.meta, + } + writer.spliceMessage = goMessage{kind: goMessagePacketSplice, packet: writer} + writer.closeMessage = goMessage{kind: goMessagePacketClose, packet: writer} + writer.buildTemplate(data.ipVersion, source) + data.created = writer + if directory.udpFlows != nil { + directory.udpFlows[data.key] = writer + } + return true, s.ctx, writer, writer.handleSessionClose +} + +type goPacketCloseError struct { + err error +} + +type GoPacketConn struct { + engine *goEngine + key udpNatSessionKey + platformIO goPlatformIO + splice *goSplicePacket + splicePending atomic.Pointer[goSplicePacket] + spliceActive atomic.Bool + spliceCloseError atomic.Pointer[goPacketCloseError] + spliceMessage goMessage + closeMessage goMessage + ipVersion uint8 + templateLength int + checksumOffload bool + template [header.IPv6MinimumSize + header.UDPMinimumSize]byte + conn atomic.Pointer[UDPNatConn] + tracker atomic.Pointer[FlowTracker] + trackerClosed atomic.Bool + snapshotAccess sync.Mutex + snapshot []byte + snapshotMeta ForwardFrameMeta +} + +func (w *GoPacketConn) buildTemplate(ipVersion uint8, client M.Socksaddr) { + w.ipVersion = ipVersion + clientAddr := client.Addr + if ipVersion == 4 { + w.templateLength = header.IPv4MinimumSize + header.UDPMinimumSize + ipHdr := header.IPv4(w.template[:w.templateLength]) + ipHdr.Encode(&header.IPv4Fields{ + TotalLength: uint16(w.templateLength), + TTL: synthesizedTTL, + Protocol: uint8(header.UDPProtocolNumber), + SrcAddr: clientAddr, + DstAddr: clientAddr, + }) + header.UDP(ipHdr[header.IPv4MinimumSize:]).Encode(&header.UDPFields{ + DstPort: client.Port, + }) + } else { + w.templateLength = header.IPv6MinimumSize + header.UDPMinimumSize + ipHdr := header.IPv6(w.template[:w.templateLength]) + ipHdr.Encode(&header.IPv6Fields{ + PayloadLength: header.UDPMinimumSize, + TransportProtocol: header.UDPProtocolNumber, + HopLimit: synthesizedTTL, + SrcAddr: clientAddr, + DstAddr: clientAddr, + }) + header.UDP(ipHdr[header.IPv6MinimumSize:]).Encode(&header.UDPFields{ + DstPort: client.Port, + }) + } +} + +func (w *GoPacketConn) preparePacketHeader(packet []byte, payload []byte, destination M.Socksaddr, checksumOffload bool) (ForwardFrameMeta, error) { + maximumPayload := 65535 - header.UDPMinimumSize + if w.ipVersion == 4 { + maximumPayload -= header.IPv4MinimumSize + } + if len(payload) > maximumPayload { + return ForwardFrameMeta{}, errGoFrameDropped + } + if !destination.IsIP() || w.ipVersion == 4 && !destination.IsIPv4() { + return ForwardFrameMeta{}, E.New("go: invalid packet destination") + } + if w.ipVersion == 6 && destination.IsIPv4() { + destination = M.SocksaddrFrom(netip.AddrFrom16(destination.Addr.As16()), destination.Port) + } + var meta ForwardFrameMeta + copy(packet, w.template[:w.templateLength]) + udpLength := uint16(header.UDPMinimumSize + len(payload)) + var sourceAddress, destinationAddress []byte + if w.ipVersion == 4 { + ipHdr := header.IPv4(packet[:header.IPv4MinimumSize]) + ipHdr.SetTotalLength(uint16(w.templateLength + len(payload))) + ipHdr.SetSourceAddr(destination.Addr) + ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } else { + ipHdr := header.IPv6(packet[:header.IPv6MinimumSize]) + ipHdr.SetPayloadLength(udpLength) + ipHdr.SetSourceAddr(destination.Addr) + sourceAddress = ipHdr.SourceAddressSlice() + destinationAddress = ipHdr.DestinationAddressSlice() + } + udpHdr := header.UDP(packet[w.templateLength-header.UDPMinimumSize : w.templateLength]) + udpHdr.SetSourcePort(destination.Port) + udpHdr.SetLength(udpLength) + pseudoSum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, sourceAddress, destinationAddress, udpLength) + if checksumOffload { + udpHdr.SetChecksum(pseudoSum) + meta.needsChecksum = true + meta.checksumStart = uint16(w.templateLength - header.UDPMinimumSize) + meta.checksumOffset = goUDPChecksumOffset + } else { + sum := ^checksum.Checksum(payload, udpHdr.CalculateChecksum(pseudoSum)) + if sum == 0 { + sum = 0xffff + } + udpHdr.SetChecksum(sum) + } + trackerPointer := w.tracker.Load() + if trackerPointer != nil { + (*trackerPointer).CountReverse(w.templateLength + len(payload)) + } + return meta, nil +} + +func (w *GoPacketConn) FrontHeadroom() int { + return w.platformIO.transmitPrefix() + w.templateLength +} + +func (w *GoPacketConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + defer buffer.Release() + return w.transmit(buffer, destination) +} + +func (w *GoPacketConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + return nil, false + } + return w, true +} + +func (w *GoPacketConn) WritePacketBatch(buffers []*buf.Buffer, destinations []M.Socksaddr) error { + defer buf.ReleaseMulti(buffers) + if len(buffers) == 0 || len(buffers) != len(destinations) { + return os.ErrInvalid + } + frames := goUDPFramePool.Get().(*[goPacketBatchSize]goUDPFrame) + count := 0 + defer func() { + clear(frames[:count]) + goUDPFramePool.Put(frames) + }() + mtu := w.platformIO.mtu() + for index, buffer := range buffers { + if buffer.Len()+w.templateLength > mtu { + if count > 0 { + err := goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) + if err != nil { + return err + } + clear(frames[:count]) + count = 0 + } + err := w.transmit(buffer, destinations[index]) + if err != nil { + return err + } + continue + } + frame := &frames[count] + meta, err := w.preparePacketHeader(frame.header[:], buffer.Bytes(), destinations[index], w.checksumOffload) + if err == errGoFrameDropped { + continue + } else if err != nil { + return err + } + frame.length = w.templateLength + frame.payload = buffer.Bytes() + frame.meta = meta + count++ + if count == len(frames) { + err = goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) + if err != nil { + return err + } + clear(frames[:count]) + count = 0 + } + } + if count == 0 { + return nil + } + return goIgnoreDropped(w.platformIO.writePacketBatch(frames[:count])) +} + +func (w *GoPacketConn) transmit(buffer *buf.Buffer, destination M.Socksaddr) error { + payload := buffer.Bytes() + mtu := w.platformIO.mtu() + fragmented := w.templateLength+len(payload) > mtu + meta, err := w.preparePacketHeader(buffer.ExtendHeader(w.templateLength), payload, destination, w.checksumOffload && !fragmented) + if err != nil { + if w.ipVersion == 4 && destination.IsIPv6() { + return E.New("send IPv6 packet to IPv4 connection") + } + return goIgnoreDropped(err) + } + packet := buffer.Bytes() + if !fragmented { + return goIgnoreDropped(w.platformIO.writeDatagram(packet, meta)) + } + return goWriteFragmented(w.platformIO, &w.engine.stack.fragmentIdentification, packet, mtu) +} + +func goWriteFragmented(platformIO goPlatformIO, identification *atomic.Uint32, packet []byte, mtu int) error { + fragments, fragmentsBuilt := goFragmentPacket(identification, packet, mtu) + if !fragmentsBuilt { + return nil + } + var writeErr error + for _, fragment := range fragments { + writeErr = E.Errors(writeErr, goIgnoreDropped(platformIO.writeDatagram(fragment, ForwardFrameMeta{}))) + } + return writeErr +} + +func goFragmentPacket(identification *atomic.Uint32, packet []byte, mtu int) ([][]byte, bool) { + if header.IPVersion(packet) == header.IPv4Version { + ipHdr := header.IPv4(packet) + ipHdr.SetID(uint16(identification.Add(1))) + return fragmentIPv4Packet(ipHdr, uint32(mtu)) + } + return fragmentIPv6Packet(header.IPv6(packet), uint32(mtu), identification.Add(1)) +} + +func (w *GoPacketConn) HandshakeSuccess() error { + w.snapshotAccess.Lock() + w.snapshot = nil + w.snapshotAccess.Unlock() + return nil +} + +func (w *GoPacketConn) HandshakeFailure(err error) error { + w.snapshotAccess.Lock() + snapshot := w.snapshot + snapshotMeta := w.snapshotMeta + w.snapshot = nil + w.snapshotAccess.Unlock() + if snapshot == nil { + return os.ErrInvalid + } + snapshotMeta.completeChecksum(snapshot) + reply, ok := BuildICMPError(snapshot, ICMPErrorNoRoute, netip.Addr{}, 0, 0) + if !ok { + return nil + } + return goIgnoreDropped(w.platformIO.writePacket(reply, ForwardFrameMeta{})) +} + +func (w *GoPacketConn) closeSplice(err error) { + w.spliceCloseError.CompareAndSwap(nil, &goPacketCloseError{err: err}) + w.engine.postMessage(&w.closeMessage) +} + +func (w *GoPacketConn) Close() error { + directory := &w.engine.stack.directory + if directory.udpFlows != nil { + directory.access.Lock() + if directory.udpFlows[w.key] == w { + delete(directory.udpFlows, w.key) + } + directory.access.Unlock() + } + if w.splicePending.Load() != nil || w.spliceActive.Load() { + w.closeSplice(io.ErrClosedPipe) + } + return nil +} + +func (w *GoPacketConn) CloseFlow() { + conn := w.conn.Load() + if conn != nil { + conn.Close() + } +} + +func (w *GoPacketConn) handleSessionClose(err error) { + trackerPointer := w.tracker.Load() + if trackerPointer == nil { + return + } + if !w.trackerClosed.CompareAndSwap(false, true) { + return + } + if err != nil && !E.IsClosedOrCanceled(err) { + (*trackerPointer).CloseFlow(FlowCloseReset) + } else { + (*trackerPointer).CloseFlow(FlowCloseTimeout) + } +} + +var ( + _ N.FrontHeadroom = (*GoPacketConn)(nil) + _ N.HandshakeSuccess = (*GoPacketConn)(nil) + _ N.HandshakeFailure = (*GoPacketConn)(nil) + _ N.PacketBatchWriteCreator = (*GoPacketConn)(nil) +) diff --git a/stack_go_udp_socket.go b/stack_go_udp_socket.go new file mode 100644 index 00000000..abdb053b --- /dev/null +++ b/stack_go_udp_socket.go @@ -0,0 +1,462 @@ +package tun + +import ( + "context" + "io" + "net" + "net/netip" + "os" + "sync" + "syscall" + "time" + "unsafe" + + "github.com/sagernet/sing-tun/gtcpip/checksum" + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/buf" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/common/pipe" +) + +// SK_RMEM_MAX in include/net/sock.h is the initial receive budget of every socket, and +// SOCK_MIN_RCVBUF the floor SO_RCVBUF is clamped to. +const ( + goUDPReceiveCapacity = 212992 + goUDPReceiveMinimum = 2304 +) + +type goUDPDatagram struct { + buffer *buf.Buffer + source netip.AddrPort +} + +func (d goUDPDatagram) memoryCost() int { + return d.buffer.RawCap() + int(unsafe.Sizeof(d)) + int(unsafe.Sizeof(*d.buffer)) +} + +type GoUDPConn struct { + engine *goEngine + local netip.AddrPort + remote netip.AddrPort + ipVersion uint8 + connected bool + access sync.Mutex + queue []goUDPDatagram + queueHead int + queuedBytes int + receiveCapacity int + readWaitOptions N.ReadWaitOptions + closed bool + openErr error + pendingErr error + openSignal chan struct{} + closeSignal chan struct{} + closeDone chan struct{} + readSignal goSignal + readDeadline pipe.Deadline + writeDeadline pipe.Deadline + openMessage goMessage + closeMessage goMessage +} + +func (s *Go) DialUDP(local netip.AddrPort, destination netip.AddrPort) (*GoUDPConn, error) { + destination = netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !destination.IsValid() || destination.Port() == 0 { + return nil, errGoInvalidAddress + } + return s.openUDP(local, destination) +} + +func (s *Go) ListenUDP(local netip.AddrPort) (*GoUDPConn, error) { + return s.openUDP(local, netip.AddrPort{}) +} + +func (s *Go) openUDP(local netip.AddrPort, destination netip.AddrPort) (*GoUDPConn, error) { + local = netip.AddrPortFrom(local.Addr().Unmap(), local.Port()) + if !local.Addr().IsValid() || destination.IsValid() && local.Addr().Is4() != destination.Addr().Is4() { + return nil, errGoInvalidAddress + } + engine, err := s.selectEngine(local) + if err != nil { + return nil, err + } + socket := &GoUDPConn{ + engine: engine, + local: local, + remote: destination, + ipVersion: goIPVersion(local.Addr()), + connected: destination.IsValid(), + receiveCapacity: min(goUDPReceiveCapacity, goReceiveCapacityMax), + openSignal: make(chan struct{}), + closeSignal: make(chan struct{}), + closeDone: make(chan struct{}), + readSignal: make(goSignal, 1), + readDeadline: pipe.MakeDeadline(), + writeDeadline: pipe.MakeDeadline(), + } + socket.openMessage = goMessage{kind: goMessageUDPOpen, socket: socket} + socket.closeMessage = goMessage{kind: goMessageUDPClose, socket: socket} + engine.postMessage(&socket.openMessage) + select { + case <-socket.openSignal: + case <-s.ctx.Done(): + socket.Close() + return nil, E.Cause(context.Cause(s.ctx), "go: open UDP socket") + } + socket.access.Lock() + err = socket.openErr + socket.access.Unlock() + if err != nil { + return nil, err + } + return socket, nil +} + +func (e *goEngine) handleUDPOpen(socket *GoUDPConn) { + socket.access.Lock() + closed := socket.closed + socket.access.Unlock() + if closed { + close(socket.openSignal) + return + } + available := func(port uint16) bool { + local := netip.AddrPortFrom(socket.local.Addr(), port) + return e.udpSockets[local] == nil && (len(e.stack.engines) == 1 || e.stack.directory.lookupUDPSocket(local) == nil) + } + port := socket.local.Port() + if port == 0 { + allocated := false + port, allocated = goAllocatePort(available) + if !allocated { + socket.access.Lock() + socket.openErr = E.Cause(syscall.EADDRINUSE, "go: ephemeral ports exhausted") + socket.access.Unlock() + close(socket.openSignal) + return + } + } else if !available(port) { + socket.access.Lock() + socket.openErr = E.Cause(syscall.EADDRINUSE, "go: bind ", socket.local) + socket.access.Unlock() + close(socket.openSignal) + return + } + socket.local = netip.AddrPortFrom(socket.local.Addr(), port) + e.stack.directory.insertUDPSocket(socket) + close(socket.openSignal) +} + +func (e *goEngine) handleUDPClose(socket *GoUDPConn) { + e.stack.directory.removeUDPSocket(socket) + socket.drain() + close(socket.closeDone) +} + +func (e *goEngine) closeAllUDPSockets() { + for _, socket := range e.udpSockets { + e.stack.directory.removeUDPSocket(socket) + socket.markClosed(net.ErrClosed) + socket.drain() + } +} + +func (e *goEngine) inputUDPSocket(socket *GoUDPConn, parsed *forwardPacket) { + if socket.connected && parsed.source != socket.remote { + return + } + payload := header.UDP(parsed.transport).Payload() + socket.access.Lock() + if socket.closed || socket.queuedBytes >= socket.receiveCapacity { + socket.access.Unlock() + return + } + buffer := socket.readWaitOptions.NewBufferSize(len(payload)) + buffer.Write(payload) + socket.readWaitOptions.PostReturn(buffer) + datagram := goUDPDatagram{buffer: buffer, source: parsed.source} + socket.queue = append(socket.queue, datagram) + socket.queuedBytes += datagram.memoryCost() + socket.access.Unlock() + if socket.readSignal.notify() { + e.wokeHandlerThisBurst = true + } +} + +func (c *GoUDPConn) deliverError(err error) { + c.access.Lock() + c.pendingErr = err + c.access.Unlock() + c.readSignal.notify() +} + +func (c *GoUDPConn) InitializeReadWaiter(options N.ReadWaitOptions) bool { + c.access.Lock() + c.readWaitOptions = options + c.access.Unlock() + return false +} + +func (c *GoUDPConn) waitDatagram() (goUDPDatagram, error) { + for { + select { + case <-c.readDeadline.Wait(): + return goUDPDatagram{}, os.ErrDeadlineExceeded + default: + } + c.access.Lock() + if c.pendingErr != nil { + err := c.pendingErr + c.pendingErr = nil + c.access.Unlock() + return goUDPDatagram{}, err + } + if c.queueHead < len(c.queue) { + datagram := c.queue[c.queueHead] + c.queue[c.queueHead] = goUDPDatagram{} + c.queueHead++ + if c.queueHead == len(c.queue) { + c.queue = c.queue[:0] + c.queueHead = 0 + } else if c.queueHead >= len(c.queue)/2 { + remaining := copy(c.queue, c.queue[c.queueHead:]) + clear(c.queue[remaining:]) + c.queue = c.queue[:remaining] + c.queueHead = 0 + } + c.queuedBytes -= datagram.memoryCost() + datagram.buffer = c.readWaitOptions.Copy(datagram.buffer) + c.access.Unlock() + return datagram, nil + } + closed := c.closed + c.access.Unlock() + if closed { + return goUDPDatagram{}, c.closeError() + } + select { + case <-c.readSignal: + case <-c.closeSignal: + case <-c.readDeadline.Wait(): + return goUDPDatagram{}, os.ErrDeadlineExceeded + } + } +} + +func (c *GoUDPConn) WaitReadPacket() (*buf.Buffer, M.Socksaddr, error) { + datagram, err := c.waitDatagram() + if err != nil { + return nil, M.Socksaddr{}, err + } + return datagram.buffer, M.SocksaddrFromNetIP(datagram.source), nil +} + +func (c *GoUDPConn) ReadPacket(buffer *buf.Buffer) (M.Socksaddr, error) { + datagram, err := c.waitDatagram() + if err != nil { + return M.Socksaddr{}, err + } + defer datagram.buffer.Release() + if buffer.FreeLen() < datagram.buffer.Len() { + return M.SocksaddrFromNetIP(datagram.source), io.ErrShortBuffer + } + buffer.Write(datagram.buffer.Bytes()) + return M.SocksaddrFromNetIP(datagram.source), nil +} + +func (c *GoUDPConn) ReadFromUDPAddrPort(p []byte) (int, netip.AddrPort, error) { + datagram, err := c.waitDatagram() + if err != nil { + return 0, netip.AddrPort{}, err + } + defer datagram.buffer.Release() + return copy(p, datagram.buffer.Bytes()), datagram.source, nil +} + +func (c *GoUDPConn) ReadFrom(p []byte) (int, net.Addr, error) { + n, source, err := c.ReadFromUDPAddrPort(p) + if err != nil { + return 0, nil, err + } + return n, net.UDPAddrFromAddrPort(source), nil +} + +func (c *GoUDPConn) Read(p []byte) (int, error) { + n, _, err := c.ReadFromUDPAddrPort(p) + return n, err +} + +func (c *GoUDPConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + defer buffer.Release() + if !destination.IsIP() { + return E.Cause(os.ErrInvalid, "go: invalid packet destination") + } + return c.transmit(buffer.Bytes(), destination.AddrPort()) +} + +func (c *GoUDPConn) WriteTo(p []byte, addr net.Addr) (int, error) { + destination := M.SocksaddrFromNet(addr) + if !destination.IsIP() { + return 0, E.Cause(os.ErrInvalid, "go: invalid packet destination") + } + return c.WriteToUDPAddrPort(p, destination.AddrPort()) +} + +func (c *GoUDPConn) WriteToUDPAddrPort(p []byte, addr netip.AddrPort) (int, error) { + err := c.transmit(p, addr) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (c *GoUDPConn) Write(p []byte) (int, error) { + if !c.connected { + return 0, E.Cause(syscall.EDESTADDRREQ, "go: socket not connected") + } + return c.WriteToUDPAddrPort(p, c.remote) +} + +func (c *GoUDPConn) transmit(payload []byte, destination netip.AddrPort) error { + c.access.Lock() + closed := c.closed + c.access.Unlock() + if closed { + return c.closeError() + } + select { + case <-c.writeDeadline.Wait(): + return os.ErrDeadlineExceeded + default: + } + remote := netip.AddrPortFrom(destination.Addr().Unmap(), destination.Port()) + if !remote.IsValid() || remote.Addr().Is4() != c.local.Addr().Is4() || remote.Port() == 0 { + return E.Cause(syscall.EAFNOSUPPORT, "go: packet destination ", remote) + } + headerLength := goNetworkHeaderLength(c.ipVersion) + header.UDPMinimumSize + maximumPayload := 65535 - header.UDPMinimumSize + if c.ipVersion == 4 { + maximumPayload -= header.IPv4MinimumSize + } + if len(payload) > maximumPayload { + return E.Cause(syscall.EMSGSIZE, "go: packet payload ", len(payload)) + } + packet := buf.NewSize(headerLength + len(payload)) + defer packet.Release() + goEncodeUDPPacket(packet.Extend(headerLength+len(payload)), c.ipVersion, c.local, remote, payload, uint16(c.engine.stack.fragmentIdentification.Add(1))) + platformIO := c.engine.platformIO + mtu := platformIO.mtu() + if packet.Len() <= mtu { + return goIgnoreDropped(platformIO.writeDatagram(packet.Bytes(), ForwardFrameMeta{})) + } + return goWriteFragmented(platformIO, &c.engine.stack.fragmentIdentification, packet.Bytes(), mtu) +} + +func goEncodeUDPPacket(packet []byte, ipVersion uint8, source netip.AddrPort, destination netip.AddrPort, payload []byte, ident uint16) { + udpLength := uint16(header.UDPMinimumSize + len(payload)) + pseudoSum := goEncodeNetworkHeader(packet, ipVersion, header.UDPProtocolNumber, source.Addr(), destination.Addr(), int(udpLength), ident) + udpHdr := header.UDP(packet[goNetworkHeaderLength(ipVersion):]) + udpHdr.Encode(&header.UDPFields{ + SrcPort: source.Port(), + DstPort: destination.Port(), + Length: udpLength, + }) + copy(udpHdr[header.UDPMinimumSize:], payload) + sum := ^checksum.Checksum(payload, udpHdr.CalculateChecksum(pseudoSum)) + if sum == 0 { + sum = 0xffff + } + udpHdr.SetChecksum(sum) +} + +func (c *GoUDPConn) markClosed(err error) bool { + c.access.Lock() + if c.closed { + c.access.Unlock() + return false + } + c.closed = true + if c.openErr == nil { + c.openErr = err + } + c.access.Unlock() + close(c.closeSignal) + return true +} + +func (c *GoUDPConn) closeError() error { + c.access.Lock() + defer c.access.Unlock() + if c.openErr != nil { + return c.openErr + } + return net.ErrClosed +} + +func (c *GoUDPConn) drain() { + c.access.Lock() + for _, datagram := range c.queue[c.queueHead:] { + datagram.buffer.Release() + } + clear(c.queue) + c.queue = c.queue[:0] + c.queueHead = 0 + c.queuedBytes = 0 + c.access.Unlock() +} + +func (c *GoUDPConn) Close() error { + if c.markClosed(net.ErrClosed) { + c.engine.postMessage(&c.closeMessage) + } + select { + case <-c.closeDone: + case <-c.engine.exitSignal: + } + return nil +} + +func (c *GoUDPConn) LocalAddr() net.Addr { + return net.UDPAddrFromAddrPort(c.local) +} + +func (c *GoUDPConn) RemoteAddr() net.Addr { + if !c.connected { + return nil + } + return net.UDPAddrFromAddrPort(c.remote) +} + +func (c *GoUDPConn) SetDeadline(t time.Time) error { + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil +} + +func (c *GoUDPConn) SetReadDeadline(t time.Time) error { + c.readDeadline.Set(t) + return nil +} + +func (c *GoUDPConn) SetReadBuffer(size int) error { + c.access.Lock() + c.receiveCapacity = min(max(size, goUDPReceiveMinimum), goReceiveCapacityMax) + c.access.Unlock() + return nil +} + +func (c *GoUDPConn) SetWriteBuffer(size int) error { + return nil +} + +func (c *GoUDPConn) SetWriteDeadline(t time.Time) error { + c.writeDeadline.Set(t) + return nil +} + +var ( + _ N.NetPacketConn = (*GoUDPConn)(nil) + _ N.PacketReadWaiter = (*GoUDPConn)(nil) +) diff --git a/stack_go_udp_test.go b/stack_go_udp_test.go new file mode 100644 index 00000000..7f03fbc3 --- /dev/null +++ b/stack_go_udp_test.go @@ -0,0 +1,183 @@ +package tun + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/netip" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/sagernet/sing-tun/gtcpip/header" + "github.com/sagernet/sing/common/logger" + M "github.com/sagernet/sing/common/metadata" + N "github.com/sagernet/sing/common/network" +) + +type goUDPImmediateCloseHandler struct { + Handler + finished chan struct{} + tracker *goUDPCloseTracker +} + +func (h *goUDPImmediateCloseHandler) JudgeFlow(uint8, netip.AddrPort, netip.AddrPort, []byte) FlowVerdict { + return FlowVerdict{Action: ActionFlow, Port: goUDPUnavailablePort{}, NewTracker: func() FlowTracker { + select { + case <-h.finished: + case <-time.After(50 * time.Millisecond): + } + return h.tracker + }} +} + +func (h *goUDPImmediateCloseHandler) NewPacketConnectionEx(_ context.Context, conn N.PacketConn, _, _ M.Socksaddr, onClose N.CloseHandlerFunc) { + conn.Close() + onClose(nil) + close(h.finished) +} + +type goUDPUnavailablePort struct{ Port } + +func (goUDPUnavailablePort) PortAddresses() (netip.Addr, netip.Addr) { + return netip.Addr{}, netip.Addr{} +} + +type goUDPCloseTracker struct { + attached atomic.Bool + closed chan bool +} + +func (t *goUDPCloseTracker) AttachFlow(FlowHandle) { t.attached.Store(true) } +func (*goUDPCloseTracker) CountForward(int) {} +func (*goUDPCloseTracker) CountReverse(int) {} +func (*goUDPCloseTracker) FlowEstablished() {} +func (t *goUDPCloseTracker) CloseFlow(FlowCloseReason) { + t.closed <- t.attached.Load() +} + +func TestGoUDPSessionImmediateClose(t *testing.T) { + tracker := &goUDPCloseTracker{closed: make(chan bool, 1)} + handler := &goUDPImmediateCloseHandler{finished: make(chan struct{}), tracker: tracker} + device := NewMemoryTun(MemoryTunOptions{MTU: 1500}) + t.Cleanup(func() { device.Close() }) + stack, err := NewGo(StackOptions{ + Context: context.Background(), Tun: device, TunOptions: Options{MTU: 1500}, + Handler: handler, Logger: logger.NOP(), UDPTimeout: time.Minute, + }) + if err != nil { + t.Fatal(err) + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(packet, 4, netip.MustParseAddrPort("198.18.0.2:12345"), netip.MustParseAddrPort("203.0.113.1:443"), []byte{1}, 1) + _, err = device.Write(packet) + if err != nil { + t.Fatal(err) + } + select { + case attached := <-tracker.closed: + if !attached { + t.Fatal("packet session closed before its tracker was attached") + } + case <-time.After(time.Second): + t.Fatal("packet session finished without closing its tracker") + } + select { + case <-handler.finished: + case <-time.After(time.Second): + t.Fatal("packet handler did not finish") + } +} + +func TestGoUDPReceiveBudget(t *testing.T) { + for _, payloadSize := range []int{0, 1} { + t.Run(fmt.Sprintf("payload=%d", payloadSize), func(t *testing.T) { + device := NewMemoryTun(MemoryTunOptions{MTU: 1500}) + t.Cleanup(func() { device.Close() }) + stack, err := NewGo(StackOptions{Context: context.Background(), Tun: device, TunOptions: Options{MTU: 1500}, Logger: logger.NOP()}) + if err != nil { + t.Fatal(err) + } + err = stack.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stack.Close() }) + local := netip.MustParseAddrPort("198.18.0.2:12345") + remote := netip.MustParseAddrPort("203.0.113.1:443") + socket, err := stack.ListenUDP(local) + if err != nil { + t.Fatal(err) + } + defer socket.Close() + socket.SetReadBuffer(1) + barrier, err := stack.ListenUDP(netip.AddrPortFrom(local.Addr(), local.Port()+1)) + if err != nil { + t.Fatal(err) + } + defer barrier.Close() + packet := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+payloadSize) + goEncodeUDPPacket(packet, 4, remote, local, make([]byte, payloadSize), 1) + const sent = 200 + packets := make([][]byte, sent+1) + for index := range sent { + packets[index] = packet + } + barrierPacket := make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+1) + goEncodeUDPPacket(barrierPacket, 4, remote, netip.AddrPortFrom(local.Addr(), local.Port()+1), []byte{1}, 1) + packets[sent] = barrierPacket + accepted, err := device.WritePackets(packets) + if err != nil || accepted != len(packets) { + t.Fatalf("inject receive burst: accepted %d/%d: %v", accepted, len(packets), err) + } + barrier.SetReadDeadline(time.Now().Add(time.Second)) + _, err = barrier.Read(make([]byte, 1)) + if err != nil { + t.Fatal("receive burst did not finish:", err) + } + socket.SetReadDeadline(time.Now().Add(20 * time.Millisecond)) + received := 0 + for { + buffer, source, readErr := socket.WaitReadPacket() + if errors.Is(readErr, os.ErrDeadlineExceeded) { + break + } + if readErr != nil { + t.Fatal(readErr) + } + valid := buffer.Len() == payloadSize && source.AddrPort() == remote + buffer.Release() + if !valid { + t.Fatal("receive burst payload or source changed") + } + received++ + } + if received == 0 || received >= sent { + t.Fatalf("receive buffer retained %d/%d datagrams despite its small budget", received, sent) + } + socket.SetReadDeadline(time.Now().Add(time.Second)) + payload := []byte("capacity restored after reading") + packet = make([]byte, header.IPv4MinimumSize+header.UDPMinimumSize+len(payload)) + goEncodeUDPPacket(packet, 4, remote, local, payload, 2) + _, err = device.Write(packet) + if err != nil { + t.Fatal(err) + } + buffer, source, err := socket.WaitReadPacket() + if err != nil { + t.Fatal("receive after draining:", err) + } + defer buffer.Release() + if !bytes.Equal(buffer.Bytes(), payload) || source.AddrPort() != remote { + t.Fatal("receive after draining changed the datagram") + } + }) + } +} diff --git a/stack_go_wheel.go b/stack_go_wheel.go new file mode 100644 index 00000000..6bdcb130 --- /dev/null +++ b/stack_go_wheel.go @@ -0,0 +1,192 @@ +package tun + +import ( + "math" + "time" +) + +const ( + goWheelTick = int64(time.Millisecond) + goWheelLevel0Slots = 128 + goWheelLevel1Slots = 64 + goWheelLevel1Ticks = goWheelLevel0Slots + goWheelRotationTicks = goWheelLevel1Slots * goWheelLevel1Ticks +) + +const ( + goWheelLevel0 uint8 = iota + goWheelLevel1 + goWheelOverflow +) + +type goWheelNode struct { + prev *goWheelNode + next *goWheelNode + slot **goWheelNode + deadline int64 + level uint8 + expire func(now int64) +} + +type goWheel struct { + level0 [goWheelLevel0Slots]*goWheelNode + level1 [goWheelLevel1Slots]*goWheelNode + overflow *goWheelNode + currentTick int64 + overflowMin int64 + scheduled int + level0Count int + level1Count int +} + +func (w *goWheel) schedule(node *goWheelNode, deadline int64) { + if node.slot != nil { + w.unlink(node) + } else { + w.scheduled++ + } + node.deadline = deadline + w.file(node) +} + +func (w *goWheel) cancel(node *goWheelNode) { + if node.slot == nil { + return + } + w.unlink(node) + w.scheduled-- +} + +func (w *goWheel) file(node *goWheelNode) { + tick := max((node.deadline+goWheelTick-1)/goWheelTick, w.currentTick+1) + var slot **goWheelNode + if tick-w.currentTick < goWheelLevel0Slots { + slot = &w.level0[tick%goWheelLevel0Slots] + node.level = goWheelLevel0 + w.level0Count++ + } else if tick/goWheelLevel1Ticks-w.currentTick/goWheelLevel1Ticks < goWheelLevel1Slots { + slot = &w.level1[(tick/goWheelLevel1Ticks)%goWheelLevel1Slots] + node.level = goWheelLevel1 + w.level1Count++ + } else { + slot = &w.overflow + node.level = goWheelOverflow + if w.overflow == nil || tick < w.overflowMin { + w.overflowMin = tick + } + } + node.prev = nil + node.next = *slot + if node.next != nil { + node.next.prev = node + } + *slot = node + node.slot = slot +} + +func (w *goWheel) unlink(node *goWheelNode) { + if node.prev != nil { + node.prev.next = node.next + } else { + *node.slot = node.next + } + if node.next != nil { + node.next.prev = node.prev + } + node.prev = nil + node.next = nil + node.slot = nil + switch node.level { + case goWheelLevel0: + w.level0Count-- + case goWheelLevel1: + w.level1Count-- + } +} + +func (w *goWheel) advance(now int64) { + targetTick := now / goWheelTick + for w.currentTick < targetTick { + if w.scheduled == 0 { + w.currentTick = targetTick + return + } + if w.level0Count == 0 { + boundary := (w.currentTick/goWheelLevel1Ticks + 1) * goWheelLevel1Ticks + if w.level1Count == 0 { + boundary = (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks + } + if boundary > targetTick { + w.currentTick = targetTick + return + } + w.currentTick = boundary + } else { + w.currentTick++ + } + if w.currentTick%goWheelLevel1Ticks == 0 { + if w.currentTick%goWheelRotationTicks == 0 { + w.refile(&w.overflow) + } + w.refile(&w.level1[(w.currentTick/goWheelLevel1Ticks)%goWheelLevel1Slots]) + } + w.fireSlot(&w.level0[w.currentTick%goWheelLevel0Slots], now) + } +} + +func (w *goWheel) refile(slot **goWheelNode) { + chain := *slot + *slot = nil + for chain != nil { + node := chain + chain = chain.next + node.prev = nil + node.next = nil + node.slot = nil + if node.level == goWheelLevel1 { + w.level1Count-- + } + w.file(node) + } +} + +func (w *goWheel) fireSlot(slot **goWheelNode, now int64) { + for *slot != nil { + node := *slot + w.unlink(node) + w.scheduled-- + node.expire(now) + } +} + +func (w *goWheel) nextDeadline() (int64, bool) { + if w.scheduled == 0 { + return 0, false + } + deadline := int64(math.MaxInt64) + if w.level0Count > 0 { + for offset := int64(1); offset < goWheelLevel0Slots; offset++ { + tick := w.currentTick + offset + if w.level0[tick%goWheelLevel0Slots] != nil { + deadline = tick * goWheelTick + break + } + } + } + if w.level1Count > 0 { + level1Index := w.currentTick / goWheelLevel1Ticks + for offset := int64(1); offset < goWheelLevel1Slots; offset++ { + index := level1Index + offset + if w.level1[index%goWheelLevel1Slots] != nil { + deadline = min(deadline, index*goWheelLevel1Ticks*goWheelTick) + break + } + } + } + if w.overflow != nil { + rotation := (w.currentTick/goWheelRotationTicks + 1) * goWheelRotationTicks + landing := (w.overflowMin / goWheelRotationTicks) * goWheelRotationTicks + deadline = min(deadline, max(rotation, landing)*goWheelTick) + } + return deadline, true +} diff --git a/stack_gvisor.go b/stack_gvisor.go index c226d05d..5d4f3ac3 100644 --- a/stack_gvisor.go +++ b/stack_gvisor.go @@ -81,12 +81,11 @@ func NewGVisor( inet6LoopbackAddress: options.TunOptions.Inet6LoopbackAddress, icmpTimeout: options.ICMPTimeout, udpNATOptions: UDPNatOptions{ - Timeout: options.UDPTimeout, - Mapping: options.UDPMapping, - Filtering: options.UDPFiltering, - MaxSize: options.UDPNATMax, - InterfaceFinder: options.InterfaceFinder, - ExcludeInterface: []string{options.TunOptions.Name}, + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, }, broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), handler: options.Handler, diff --git a/stack_gvisor_filter.go b/stack_gvisor_filter.go index 6f889ed0..3cf9e014 100644 --- a/stack_gvisor_filter.go +++ b/stack_gvisor_filter.go @@ -10,8 +10,6 @@ import ( "github.com/sagernet/gvisor/pkg/tcpip/stack" ) -var _ stack.LinkEndpoint = (*LinkEndpointFilter)(nil) - type LinkEndpointFilter struct { stack.LinkEndpoint BroadcastAddress netip.Addr @@ -32,7 +30,7 @@ func (w *LinkEndpointFilter) Attach(dispatcher stack.NetworkDispatcher) { NetworkDispatcher: dispatcher, broadcastAddress: w.BroadcastAddress, writer: w.Writer, - dispatcher: w.Dispatcher, + dispatchStage: w.Dispatcher.NewStage(nil), inet4Address: w.Inet4Address, inet6Address: w.Inet6Address, inet4LoopbackAddress: w.Inet4LoopbackAddress, @@ -40,13 +38,11 @@ func (w *LinkEndpointFilter) Attach(dispatcher stack.NetworkDispatcher) { }) } -var _ stack.NetworkDispatcher = (*networkDispatcherFilter)(nil) - type networkDispatcherFilter struct { stack.NetworkDispatcher broadcastAddress netip.Addr writer GVisorTun - dispatcher *ForwardDispatcher + dispatchStage *ForwardStage inet4Address netip.Addr inet6Address netip.Addr inet4LoopbackAddress []netip.Addr @@ -73,10 +69,10 @@ func (w *networkDispatcherFilter) DeliverNetworkPacket(protocol tcpip.NetworkPro w.writer.WritePacket(pkt) return } - if w.dispatcher != nil && pkt.GSOOptions.Type == stack.GSONone && !pkt.GSOOptions.NeedsCsum { + if w.dispatchStage != nil && pkt.GSOOptions.Type == stack.GSONone && !pkt.GSOOptions.NeedsCsum { if view, loaded := pkt.Data().PullUp(pkt.Data().Size()); loaded { consumed := w.dispatch(protocol, destination, view) - w.dispatcher.Flush() + w.dispatchStage.Flush() if consumed { return } @@ -113,5 +109,5 @@ func (w *networkDispatcherFilter) dispatch(protocol tcpip.NetworkProtocolNumber, } } } - return w.dispatcher.Dispatch(view) + return w.dispatchStage.Dispatch(view) } diff --git a/stack_gvisor_icmp.go b/stack_gvisor_icmp.go index 360f600b..2f0646b5 100644 --- a/stack_gvisor_icmp.go +++ b/stack_gvisor_icmp.go @@ -289,7 +289,9 @@ func (f *ICMPForwarder) installFlow(key icmpFlowKey, verdict FlowVerdict, pkt *s if flow.tracker != nil { flow.tracker.AttachFlow(flow) } - f.writeToPort(flow, pkt) + if !flow.closed.Load() { + f.writeToPort(flow, pkt) + } return true } diff --git a/stack_gvisor_icmp_test.go b/stack_gvisor_icmp_test.go new file mode 100644 index 00000000..19f40751 --- /dev/null +++ b/stack_gvisor_icmp_test.go @@ -0,0 +1,27 @@ +//go:build with_gvisor + +package tun + +import ( + "net/netip" + "testing" + + "github.com/sagernet/gvisor/pkg/buffer" + "github.com/sagernet/gvisor/pkg/tcpip/stack" + "github.com/sagernet/sing/common/logger" +) + +func TestICMPForwarderClosedOnAttach(t *testing.T) { + port := &recordingPort{address: netip.IPv4Unspecified()} + forwarder := NewICMPForwarder(nil, nil, logger.NOP()) + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData([]byte{1})}) + defer pkt.DecRef() + key := icmpFlowKey{source: netip.MustParseAddr("198.18.0.2"), destination: netip.MustParseAddr("203.0.113.1"), identifier: 1} + verdict := FlowVerdict{Action: ActionFlow, Port: port, NewTracker: func() FlowTracker { return closeOnAttachTracker{} }} + if !forwarder.installFlow(key, verdict, pkt) { + t.Fatal("flow was not installed") + } + if port.written != 0 { + t.Fatal("packet forwarded on a flow closed by its tracker") + } +} diff --git a/stack_gvisor_tcp_conn.go b/stack_gvisor_tcp_conn.go index ad48d42f..ff3db0b4 100644 --- a/stack_gvisor_tcp_conn.go +++ b/stack_gvisor_tcp_conn.go @@ -18,10 +18,7 @@ import ( N "github.com/sagernet/sing/common/network" ) -var ( - _ net.Conn = (*gTCPConn)(nil) - _ N.ReadWaiter = (*gTCPConn)(nil) -) +var _ N.ReadWaiter = (*gTCPConn)(nil) type gTCPConn struct { gTCPDeadline diff --git a/stack_mixed.go b/stack_mixed.go index cafc9e7c..ba1b9b7b 100644 --- a/stack_mixed.go +++ b/stack_mixed.go @@ -127,7 +127,7 @@ func (m *Mixed) tunLoop() { m.logger.Trace(E.Cause(err, "write packet")) } } - m.dispatcher.Flush() + m.dispatchStage.Flush() } } @@ -147,7 +147,7 @@ func (m *Mixed) wintunLoop(winTun WinTun) { m.logger.Trace(E.Cause(err, "write packet")) } } - m.dispatcher.Flush() + m.dispatchStage.Flush() release() } } @@ -195,7 +195,7 @@ func (m *Mixed) batchLoopLinux(linuxTUN LinuxTUN, batchSize int) { } writeBuffers = writeBuffers[:0] } - m.dispatcher.Flush() + m.dispatchStage.Flush() } } @@ -204,7 +204,7 @@ func (m *Mixed) batchLoopDarwin(darwinTUN DarwinTUN) { var releaseBuffers []*buf.Buffer var readRetry ReadRetry for { - buffers, err := darwinTUN.BatchRead() + buffers, err := darwinTUN.BatchRead(0, 0) if err != nil { if !IsRecoverableReadError(err) { if !E.IsClosed(err) && !errors.Is(err, syscall.EBADF) { @@ -241,7 +241,7 @@ func (m *Mixed) batchLoopDarwin(darwinTUN DarwinTUN) { } buf.ReleaseMulti(writeBuffers) } - m.dispatcher.Flush() + m.dispatchStage.Flush() buf.ReleaseMulti(releaseBuffers) } } diff --git a/stack_system.go b/stack_system.go index 3ff1c4ff..5b34d0af 100644 --- a/stack_system.go +++ b/stack_system.go @@ -52,6 +52,7 @@ type System struct { udpNat *UDPNat udpNATOptions UDPNatOptions dispatcher *ForwardDispatcher + dispatchStage *ForwardStage bindInterface bool interfaceFinder control.InterfaceFinder frontHeadroom int @@ -83,12 +84,11 @@ func NewSystem(options StackOptions) (Stack, error) { inet6Prefixes: options.TunOptions.Inet6Address, broadcastAddr: BroadcastAddr(options.TunOptions.Inet4Address), udpNATOptions: UDPNatOptions{ - Timeout: options.UDPTimeout, - Mapping: options.UDPMapping, - Filtering: options.UDPFiltering, - MaxSize: options.UDPNATMax, - InterfaceFinder: options.InterfaceFinder, - ExcludeInterface: []string{options.TunOptions.Name}, + Timeout: options.UDPTimeout, + Mapping: options.UDPMapping, + Filtering: options.UDPFiltering, + MaxSize: options.UDPNATMax, + InterfaceFinder: options.InterfaceFinder, }, bindInterface: options.ForwarderBindInterface, interfaceFinder: options.InterfaceFinder, @@ -201,6 +201,7 @@ func (s *System) start() error { } if s.handler != nil { s.dispatcher = NewForwardDispatcher(s.handler, newSystemWriteback(s.tun, s.frontHeadroom), s.logger, s.udpTimeout, s.icmpTimeout) + s.dispatchStage = s.dispatcher.NewStage(nil) } return nil } @@ -248,7 +249,7 @@ func (s *System) tunLoop() { s.logger.Trace(E.Cause(err, "write packet")) } } - s.dispatcher.Flush() + s.dispatchStage.Flush() } } @@ -268,7 +269,7 @@ func (s *System) wintunLoop(winTun WinTun) { s.logger.Trace(E.Cause(err, "write packet")) } } - s.dispatcher.Flush() + s.dispatchStage.Flush() release() } } @@ -316,7 +317,7 @@ func (s *System) batchLoopLinux(linuxTUN LinuxTUN, batchSize int) { } writeBuffers = writeBuffers[:0] } - s.dispatcher.Flush() + s.dispatchStage.Flush() } } @@ -325,7 +326,7 @@ func (s *System) batchLoopDarwin(darwinTUN DarwinTUN) { var releaseBuffers []*buf.Buffer var readRetry ReadRetry for { - buffers, err := darwinTUN.BatchRead() + buffers, err := darwinTUN.BatchRead(0, 0) if err != nil { if !IsRecoverableReadError(err) { if !E.IsClosed(err) && !errors.Is(err, syscall.EBADF) { @@ -362,7 +363,7 @@ func (s *System) batchLoopDarwin(darwinTUN DarwinTUN) { } buf.ReleaseMulti(writeBuffers) } - s.dispatcher.Flush() + s.dispatchStage.Flush() buf.ReleaseMulti(releaseBuffers) } } @@ -420,7 +421,7 @@ func (s *System) dispatchIPv4(ipHdr header.IPv4, destination netip.Addr) bool { return false } } - return s.dispatcher.Dispatch(ipHdr) + return s.dispatchStage.Dispatch(ipHdr) } func (s *System) dispatchIPv6(ipHdr header.IPv6, destination netip.Addr) bool { @@ -439,7 +440,7 @@ func (s *System) dispatchIPv6(ipHdr header.IPv6, destination netip.Addr) bool { return false } } - return s.dispatcher.Dispatch(ipHdr) + return s.dispatchStage.Dispatch(ipHdr) } func (s *System) processIPv4(ipHdr header.IPv4) (writeBack bool, err error) { @@ -729,8 +730,19 @@ func (s *System) processIPv4ICMP(ipHdr header.IPv4, icmpHdr header.ICMPv4) (bool if len(icmpHdr) < header.ICMPv4MinimumSize { return false, E.New("ipv4: icmp: invalid packet") } + return rewriteEchoReplyIPv4(ipHdr, icmpHdr), nil +} + +func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool, error) { + if len(icmpHdr) < header.ICMPv6MinimumSize { + return false, E.New("ipv6: icmp: invalid packet") + } + return rewriteEchoReplyIPv6(ipHdr, icmpHdr), nil +} + +func rewriteEchoReplyIPv4(ipHdr header.IPv4, icmpHdr header.ICMPv4) bool { if icmpHdr.Type() != header.ICMPv4Echo || icmpHdr.Code() != 0 { - return false, nil + return false } icmpHdr.SetType(header.ICMPv4EchoReply) sourceAddress := ipHdr.SourceAddr() @@ -738,15 +750,12 @@ func (s *System) processIPv4ICMP(ipHdr header.IPv4, icmpHdr header.ICMPv4) (bool ipHdr.SetDestinationAddr(sourceAddress) icmpHdr.SetChecksum(header.ICMPv4Checksum(icmpHdr, 0)) ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - return true, nil + return true } -func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool, error) { - if len(icmpHdr) < header.ICMPv6MinimumSize { - return false, E.New("ipv6: icmp: invalid packet") - } +func rewriteEchoReplyIPv6(ipHdr header.IPv6, icmpHdr header.ICMPv6) bool { if icmpHdr.Type() != header.ICMPv6EchoRequest || icmpHdr.Code() != 0 { - return false, nil + return false } icmpHdr.SetType(header.ICMPv6EchoReply) sourceAddress := ipHdr.SourceAddr() @@ -757,7 +766,7 @@ func (s *System) processIPv6ICMP(ipHdr header.IPv6, icmpHdr header.ICMPv6) (bool Src: ipHdr.SourceAddressSlice(), Dst: ipHdr.DestinationAddressSlice(), })) - return true, nil + return true } type systemUDPPacketWriter4 struct { @@ -772,9 +781,8 @@ func (w *systemUDPPacketWriter4) FrontHeadroom() int { return w.frontHeadroom + len(w.header) } -func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { +func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) { payloadLen := buffer.Len() - buffer = (N.ReadWaitOptions{FrontHeadroom: w.FrontHeadroom()}).Copy(buffer) copy(buffer.ExtendHeader(len(w.header)), w.header) ipHdr := header.IPv4(buffer.Bytes()) ipHdr.SetTotalLength(uint16(buffer.Len())) @@ -796,22 +804,20 @@ func (w *systemUDPPacketWriter4) preparePacket(buffer *buf.Buffer, destination M udpHdr.SetChecksum(0) } ipHdr.SetChecksum(^ipHdr.CalculateChecksum()) - return buffer } -func (w *systemUDPPacketWriter4) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { - buffer = w.preparePacket(buffer, destination) +func (w *systemUDPPacketWriter4) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) { + w.preparePacket(buffer, destination) if PacketOffset > 0 { PacketFillHeader(buffer.ExtendHeader(PacketOffset), header.IPv4Version) } if remainingHeadroom := w.frontHeadroom - PacketOffset; remainingHeadroom > 0 { buffer.Advance(-remainingHeadroom) } - return buffer } func (w *systemUDPPacketWriter4) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - buffer = w.prepareWritePacket(buffer, destination) + w.prepareWritePacket(buffer, destination) defer buffer.Release() return common.Error(w.tun.Write(buffer.Bytes())) } @@ -837,15 +843,14 @@ func (w *systemUDPPacketWriter4) WritePacketBatch(buffers []*buf.Buffer, destina case LinuxTUN: packets := make([][]byte, len(buffers)) for index, buffer := range buffers { - buffer = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) buffer.Advance(-w.frontHeadroom) - buffers[index] = buffer packets[index] = buffer.Bytes() } return common.Error(tunInterface.BatchWrite(packets, w.frontHeadroom)) case DarwinTUN: for index, buffer := range buffers { - buffers[index] = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) } return tunInterface.BatchWrite(buffers) default: @@ -865,9 +870,8 @@ func (w *systemUDPPacketWriter6) FrontHeadroom() int { return w.frontHeadroom + len(w.header) } -func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { +func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M.Socksaddr) { payloadLen := buffer.Len() - buffer = (N.ReadWaitOptions{FrontHeadroom: w.FrontHeadroom()}).Copy(buffer) copy(buffer.ExtendHeader(len(w.header)), w.header) ipHdr := header.IPv6(buffer.Bytes()) udpLen := uint16(header.UDPMinimumSize + payloadLen) @@ -889,22 +893,20 @@ func (w *systemUDPPacketWriter6) preparePacket(buffer *buf.Buffer, destination M } else { udpHdr.SetChecksum(0) } - return buffer } -func (w *systemUDPPacketWriter6) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) *buf.Buffer { - buffer = w.preparePacket(buffer, destination) +func (w *systemUDPPacketWriter6) prepareWritePacket(buffer *buf.Buffer, destination M.Socksaddr) { + w.preparePacket(buffer, destination) if PacketOffset > 0 { PacketFillHeader(buffer.ExtendHeader(PacketOffset), header.IPv6Version) } if remainingHeadroom := w.frontHeadroom - PacketOffset; remainingHeadroom > 0 { buffer.Advance(-remainingHeadroom) } - return buffer } func (w *systemUDPPacketWriter6) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { - buffer = w.prepareWritePacket(buffer, destination) + w.prepareWritePacket(buffer, destination) defer buffer.Release() return common.Error(w.tun.Write(buffer.Bytes())) } @@ -930,15 +932,14 @@ func (w *systemUDPPacketWriter6) WritePacketBatch(buffers []*buf.Buffer, destina case LinuxTUN: packets := make([][]byte, len(buffers)) for index, buffer := range buffers { - buffer = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) buffer.Advance(-w.frontHeadroom) - buffers[index] = buffer packets[index] = buffer.Bytes() } return common.Error(tunInterface.BatchWrite(packets, w.frontHeadroom)) case DarwinTUN: for index, buffer := range buffers { - buffers[index] = w.preparePacket(buffer, destinations[index]) + w.preparePacket(buffer, destinations[index]) } return tunInterface.BatchWrite(buffers) default: diff --git a/tun.go b/tun.go index e7701222..e7013e37 100644 --- a/tun.go +++ b/tun.go @@ -50,7 +50,7 @@ type LinuxTUN interface { type DarwinTUN interface { Tun - BatchRead() ([]*buf.Buffer, error) + BatchRead(frontHeadroom int, rearHeadroom int) ([]*buf.Buffer, error) BatchWrite(buffers []*buf.Buffer) error } @@ -73,6 +73,7 @@ type Options struct { Inet6Address []netip.Prefix MTU uint32 GSO bool + MultiQueue bool AutoRoute bool InterfaceScope bool Inet4Gateway netip.Addr @@ -86,6 +87,7 @@ type Options struct { AutoRedirectInputMark uint32 AutoRedirectOutputMark uint32 AutoRedirectResetMark uint32 + AutoRedirectTProxyMark uint32 AutoRedirectNFQueue uint16 ExcludeMPTCP bool Inet4LoopbackAddress []netip.Addr @@ -97,6 +99,7 @@ type Options struct { Inet6RouteExcludeAddress []netip.Prefix IncludeInterface []string ExcludeInterface []string + BridgeInterface []string IncludeUID []ranges.Range[uint32] ExcludeUID []ranges.Range[uint32] IncludeAndroidUser []int @@ -116,11 +119,15 @@ type Options struct { EXP_DisableDNSHijack bool EXP_ExternalConfiguration bool - // For gvisor stack, it should be enabled when MTU is less than 32768; otherwise it should be less than or equal to 8192. - // The above condition is just an estimate and not exact, calculated on M4 pro. + // Safe at every MTU: the darwin pending packet limit is derived from the utun control socket + // receive buffer. EXP_MultiPendingPackets bool - // Will cause the darwin network to die, do not use. + // Do not enable. In netif mode (every utun handed out by Network Extension) utun_pkt_input + // returns ENOSPC without freeing the mbuf once 512 packets are queued + // (net.utun.max_pending_input), leaking one mbuf per rejected write until reboot; sendmsg_x + // pushes the write rate past the netif drain rate (about 500k to 700k packets per second on an + // M4 Pro) and reproduces the device-wide network loss, writev stays below it by its own cost. EXP_SendMsgX bool } diff --git a/tun_darwin.go b/tun_darwin.go index 4b00ac59..5ede7565 100644 --- a/tun_darwin.go +++ b/tun_darwin.go @@ -59,15 +59,21 @@ func newIovecBuffer(mtu int) iovecBuffer { } } -func (b *iovecBuffer) nextIovecs() []unix.Iovec { +func (b *iovecBuffer) nextIovecs(frontHeadroom int, rearHeadroom int) []unix.Iovec { if b.iovecs[0].Len == 0 { headBuffer := make([]byte, PacketOffset) b.iovecs[0].Base = &headBuffer[0] b.iovecs[0].SetLen(PacketOffset) } + bufferSize := frontHeadroom + b.mtu + rearHeadroom + if b.buffer != nil && (b.buffer.Start() != frontHeadroom || b.buffer.Cap() != bufferSize) { + b.buffer.Release() + b.buffer = nil + } if b.buffer == nil { - b.buffer = buf.NewSize(b.mtu) - b.iovecs[1] = b.buffer.Iovec(b.buffer.Cap()) + b.buffer = buf.NewSize(bufferSize) + b.buffer.Resize(frontHeadroom, 0) + b.iovecs[1] = b.buffer.Iovec(b.mtu) } return b.iovecs } @@ -111,14 +117,14 @@ func New(options Options) (Tun, error) { unix.Close(tunFd) return nil, err } - err = configure(tunFd, options.EXP_MultiPendingPackets, batchSize) + err = configure(tunFd, options.EXP_MultiPendingPackets, int(options.MTU)) if err != nil { unix.Close(tunFd) return nil, err } } else { tunFd = options.FileDescriptor - err := configure(tunFd, options.EXP_MultiPendingPackets, batchSize) + err := configure(tunFd, options.EXP_MultiPendingPackets, int(options.MTU)) if err != nil { return nil, err } @@ -341,24 +347,79 @@ func create(tunFd int, ifIndex int, name string, options Options) error { return nil } -func configure(tunFd int, multiPendingPackets bool, batchSize int) error { +const ( + utunReceiveBufferTarget = 8 << 20 + utunReceiveBufferMinimum = 1 << 20 + utunReceiveBufferDefault = 512 << 10 + utunMaxPendingPackets = 64 +) + +func configure(tunFd int, multiPendingPackets bool, mtu int) error { err := unix.SetNonblock(tunFd, true) if err != nil { return os.NewSyscallError("SetNonblock", err) } - if multiPendingPackets { - const UTUN_OPT_MAX_PENDING_PACKETS = 16 - err = unix.SetsockoptInt(tunFd, 2, UTUN_OPT_MAX_PENDING_PACKETS, batchSize) - if err != nil { - return os.NewSyscallError("SetsockoptInt UTUN_OPT_MAX_PENDING_PACKETS", err) - } + if !multiPendingPackets { + return nil + } + // The utun control socket drops outbound packets with ENOBUFS once the queued bytes reach + // SO_RCVBUF (kern_control.c ctl_rcvbspace; sbspace counts bytes only for SB_KCTL, default + // 512 KB), whereas reaching UTUN_OPT_MAX_PENDING_PACKETS pauses the interface until the + // socket is read (if_utun.c utun_start / utun_ctl_rcvd). SO_RCVBUF is clamped to + // kern.ipc.maxsockbuf by sbreserve. + receiveBuffer := raiseReceiveBuffer(tunFd) + pending := receiveBuffer / 8 * 7 / (mtu + PacketOffset) + pending = max(min(pending, utunMaxPendingPackets), 1) + const UTUN_OPT_MAX_PENDING_PACKETS = 16 + err = unix.SetsockoptInt(tunFd, 2, UTUN_OPT_MAX_PENDING_PACKETS, pending) + if err != nil { + return os.NewSyscallError("SetsockoptInt UTUN_OPT_MAX_PENDING_PACKETS", err) } return nil } -func (t *NativeTun) BatchRead() ([]*buf.Buffer, error) { +func raiseReceiveBuffer(tunFd int) int { + for size := utunReceiveBufferTarget; size >= utunReceiveBufferMinimum; size /= 2 { + err := unix.SetsockoptInt(tunFd, unix.SOL_SOCKET, unix.SO_RCVBUF, size) + if err == nil { + break + } + } + current, err := unix.GetsockoptInt(tunFd, unix.SOL_SOCKET, unix.SO_RCVBUF) + if err != nil || current <= 0 { + return utunReceiveBufferDefault + } + return current +} + +func (t *NativeTun) rawFileDescriptor() int { + return t.tunFd +} + +// os.NewFile registers a non-blocking descriptor with the runtime poller, which then wakes an +// idle thread for every packet the engine loop is already waiting for on its own kqueue. +func (t *NativeTun) detachRuntimePoller() error { + duplicated, err := unix.FcntlInt(uintptr(t.tunFd), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return err + } + previous := t.tunFile + t.tunFd = duplicated + t.tunFile = newUnpolledFile(duplicated, "utun") + return previous.Close() +} + +func (t *NativeTun) transmitAccess() *sync.Mutex { + return &t.writeAccess +} + +func (t *NativeTun) enableMaxPendingPackets() error { + return configure(t.tunFd, true, int(t.options.MTU)) +} + +func (t *NativeTun) BatchRead(frontHeadroom int, rearHeadroom int) ([]*buf.Buffer, error) { for i := 0; i < t.batchSize; i++ { - iovecs := t.iovecs[i].nextIovecs() + iovecs := t.iovecs[i].nextIovecs(frontHeadroom, rearHeadroom) // Cannot clear only the length field. Older versions of the darwin kernel will check whether other data is empty. // https://github.com/Darm64/XNU/blob/xnu-2782.40.9/bsd/kern/uipc_syscalls.c#L2026-L2048 t.msgHdrs[i] = rawfile.MsgHdrX{} diff --git a/tun_linux.go b/tun_linux.go index 15515ea9..aaaedc8e 100644 --- a/tun_linux.go +++ b/tun_linux.go @@ -3,12 +3,13 @@ package tun import ( "errors" "fmt" - "math/rand" "net" "net/netip" "os" "os/exec" "runtime" + "strconv" + "strings" "sync" "syscall" "unsafe" @@ -38,6 +39,7 @@ type NativeTun struct { readAccess sync.Mutex writeAccess sync.Mutex vnetHdr bool + multiQueue bool writeBuffer []byte readRawConn syscall.RawConn pendingBuffer []byte @@ -53,7 +55,7 @@ type NativeTun struct { func New(options Options) (Tun, error) { if options.FileDescriptor == 0 { return execInNetworkNamespace(options.NetNs, func() (Tun, error) { - tunFd, err := open(options.Name, options.GSO) + tunFd, multiQueue, err := open(options.Name, options.GSO, options.MultiQueue) if err != nil { return nil, E.Cause(err, "open tun") } @@ -62,9 +64,10 @@ func New(options Options) (Tun, error) { return nil, E.Errors(err, unix.Close(tunFd)) } nativeTun := &NativeTun{ - tunFd: tunFd, - tunFile: os.NewFile(uintptr(tunFd), "tun"), - options: options, + tunFd: tunFd, + tunFile: os.NewFile(uintptr(tunFd), "tun"), + options: options, + multiQueue: multiQueue, } err = nativeTun.configure(tunLink) if err != nil { @@ -101,7 +104,30 @@ func init() { } } -func open(name string, vnetHdr bool) (int, error) { +// A persistent device created without IFF_MULTI_QUEUE refuses the flag with EINVAL +// (tun_set_iff), so the plain flags are retried and such a device stays single-queue. +func open(name string, vnetHdr bool, multiQueue bool) (int, bool, error) { + flags := unix.IFF_TUN | unix.IFF_NO_PI + if vnetHdr { + flags |= unix.IFF_VNET_HDR + } + if multiQueue { + fd, err := openTun(name, flags|unix.IFF_MULTI_QUEUE) + if err == nil { + return fd, true, nil + } + if !errors.Is(err, unix.EINVAL) { + return -1, false, err + } + } + fd, err := openTun(name, flags) + if err != nil { + return -1, false, err + } + return fd, false, nil +} + +func openTun(name string, flags int) (int, error) { fd, err := unix.Open(controlPath, unix.O_RDWR, 0) if err != nil { return -1, E.Cause(err, "open ", controlPath) @@ -109,22 +135,38 @@ func open(name string, vnetHdr bool) (int, error) { ifr, err := unix.NewIfreq(name) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "create ifreq") - } - flags := unix.IFF_TUN | unix.IFF_NO_PI - if vnetHdr { - flags |= unix.IFF_VNET_HDR + return -1, E.Cause(err, "create ifreq") } ifr.SetUint16(uint16(flags)) err = unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "TUNSETIFF") + return -1, E.Cause(err, "TUNSETIFF") } err = unix.SetNonblock(fd, true) if err != nil { unix.Close(fd) - return 0, E.Cause(err, "set nonblock") + return -1, E.Cause(err, "set nonblock") + } + return fd, nil +} + +func (t *NativeTun) openQueue() (int, error) { + if !t.multiQueue { + return -1, E.New("tun device is not multi-queue") + } + flags := unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE + if t.options.GSO { + flags |= unix.IFF_VNET_HDR + } + var fd int + err := runInNetworkNamespace(t.options.NetNs, func() error { + var openErr error + fd, openErr = openTun(t.options.Name, flags) + return openErr + }) + if err != nil { + return -1, err } return fd, nil } @@ -199,8 +241,6 @@ func (t *NativeTun) enableGSO() error { return E.Cause(err, "enable offload: IFF_VNET_HDR not enabled") } t.vnetHdr = true - t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) - t.pendingBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) t.tcpGROTable = newTCPGROTable() t.udpGROTable = newUDPGROTable() err = setTCPOffload(t.tunFd) @@ -329,16 +369,31 @@ func (t *NativeTun) start() error { return nil } - _ = os.WriteFile("/proc/sys/net/ipv4/conf/"+t.options.Name+"/rp_filter", []byte("2"), 0o644) + // The kernel uses max(all, interface) as the effective rp_filter value, so + // writing loose mode here relaxes an inherited strict mode but tightens a + // disabled one, where loose mode still drops replies whose source has no + // route on any interface. + rpFilter := 0 + for _, procPath := range []string{ + "/proc/sys/net/ipv4/conf/all/rp_filter", + "/proc/sys/net/ipv4/conf/" + t.options.Name + "/rp_filter", + } { + content, readErr := os.ReadFile(procPath) + if readErr != nil { + continue + } + value, parseErr := strconv.Atoi(strings.TrimSpace(string(content))) + if parseErr != nil { + continue + } + rpFilter = max(rpFilter, value) + } + if rpFilter == 1 { + _ = os.WriteFile("/proc/sys/net/ipv4/conf/"+t.options.Name+"/rp_filter", []byte("2"), 0o644) + } if t.options.IPRoute2TableIndex == 0 { - for { - t.options.IPRoute2TableIndex = int(rand.Uint32()) - routeList, fErr := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: t.options.IPRoute2TableIndex}, netlink.RT_FILTER_TABLE) - if len(routeList) == 0 || fErr != nil { - break - } - } + t.options.IPRoute2TableIndex = chooseRouteTableIndex() } err = t.setRoute(tunLink) @@ -388,6 +443,9 @@ func (t *NativeTun) Close() error { func (t *NativeTun) Read(p []byte) (n int, err error) { if t.vnetHdr { + if t.writeBuffer == nil { + t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) + } n, err = t.tunFile.Read(t.writeBuffer) if err != nil { if errors.Is(err, syscall.EBADFD) { @@ -510,6 +568,9 @@ func (t *NativeTun) BatchRead(buffers [][]byte, offset int, readN []int) (int, e used = count } for used < len(buffers) { + if t.writeBuffer == nil { + t.writeBuffer = make([]byte, virtioNetHdrLen+gsoMaxSize) + } var ( readLength int err error @@ -613,6 +674,28 @@ func (t *NativeTun) TXChecksumOffload() bool { return t.txChecksumOffload } +func (t *NativeTun) rawFileDescriptor() int { + return t.tunFd +} + +// os.NewFile registers a non-blocking descriptor with the runtime poller, which then wakes an +// idle thread for every packet the engine loop is already waiting for on its own epoll. +func (t *NativeTun) detachRuntimePoller() error { + duplicated, err := unix.FcntlInt(uintptr(t.tunFd), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + return err + } + previous := t.tunFile + t.tunFd = duplicated + t.tunFile = newUnpolledFile(duplicated, "tun") + t.readRawConn, err = t.tunFile.SyscallConn() + return E.Errors(err, previous.Close()) +} + +func (t *NativeTun) vnetHeaderEnabled() (bool, error) { + return checkVNETHDREnabled(t.tunFd, t.options.Name) +} + func prefixToIPNet(prefix netip.Prefix) *net.IPNet { return &net.IPNet{ IP: prefix.Addr().AsSlice(), @@ -720,11 +803,17 @@ func (t *NativeTun) rules() []*netlink.Rule { priority6 := priority if t.options.AutoRedirectMarkMode { + inputMark := effectiveMark(t.options.AutoRedirectInputMark, DefaultAutoRedirectInputMark, DefaultAutoRedirectInputMarkAndroid) + outputMark := effectiveMark(t.options.AutoRedirectOutputMark, DefaultAutoRedirectOutputMark, DefaultAutoRedirectOutputMarkAndroid) + resetMark := effectiveMark(t.options.AutoRedirectResetMark, DefaultAutoRedirectResetMark, DefaultAutoRedirectResetMarkAndroid) + tproxyMark := effectiveMark(t.options.AutoRedirectTProxyMark, DefaultAutoRedirectTProxyMark, DefaultAutoRedirectTProxyMarkAndroid) + markMask := int(inputMark | outputMark | resetMark | tproxyMark) if p4 { it = netlink.NewRule() it.Priority = priority - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Goto = priority + 2 it.Family = unix.AF_INET rules = append(rules, it) @@ -732,8 +821,9 @@ func (t *NativeTun) rules() []*netlink.Rule { it = netlink.NewRule() it.Priority = priority - it.Mark = t.options.AutoRedirectInputMark + it.Mark = inputMark it.MarkSet = true + it.Mask = markMask it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET rules = append(rules, it) @@ -747,8 +837,9 @@ func (t *NativeTun) rules() []*netlink.Rule { if p6 { it = netlink.NewRule() it.Priority = priority6 - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Goto = priority6 + 2 it.Family = unix.AF_INET6 rules = append(rules, it) @@ -756,8 +847,9 @@ func (t *NativeTun) rules() []*netlink.Rule { it = netlink.NewRule() it.Priority = priority6 - it.Mark = t.options.AutoRedirectInputMark + it.Mark = inputMark it.MarkSet = true + it.Mask = markMask it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET6 rules = append(rules, it) @@ -768,13 +860,12 @@ func (t *NativeTun) rules() []*netlink.Rule { it.Family = unix.AF_INET6 rules = append(rules, it) } - // Fallback rules after system default rules (32766: main, 32767: default) - // Only reached when main and default tables have no route if p4 { it = netlink.NewRule() it.Priority = t.options.IPRoute2AutoRedirectFallbackRuleIndex - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Invert = true it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET @@ -783,8 +874,9 @@ func (t *NativeTun) rules() []*netlink.Rule { if p6 { it = netlink.NewRule() it.Priority = t.options.IPRoute2AutoRedirectFallbackRuleIndex - it.Mark = t.options.AutoRedirectOutputMark + it.Mark = outputMark it.MarkSet = true + it.Mask = markMask it.Invert = true it.Table = t.options.IPRoute2TableIndex it.Family = unix.AF_INET6 @@ -1039,6 +1131,14 @@ func (t *NativeTun) rules() []*netlink.Rule { // priority++ } if p6 { + it = netlink.NewRule() + it.Priority = priority6 + it.IifName = t.options.Name + it.Goto = nopPriority + it.Family = unix.AF_INET6 + rules = append(rules, it) + priority6++ + for _, address := range t.options.Inet6Address { it = netlink.NewRule() it.Priority = priority6 @@ -1050,13 +1150,6 @@ func (t *NativeTun) rules() []*netlink.Rule { } priority6++ - it = netlink.NewRule() - it.Priority = priority6 - it.IifName = t.options.Name - it.Goto = nopPriority - it.Family = unix.AF_INET6 - rules = append(rules, it) - it = netlink.NewRule() it.Priority = priority6 it.IifName = "lo" diff --git a/tun_linux_gvisor.go b/tun_linux_gvisor.go index 48e142a9..43b58bb1 100644 --- a/tun_linux_gvisor.go +++ b/tun_linux_gvisor.go @@ -166,5 +166,3 @@ func (e *linuxTUNEndpoint) dispatchLoop() { } } } - -var _ stack.LinkEndpoint = (*linuxTUNEndpoint)(nil) diff --git a/tun_memory.go b/tun_memory.go new file mode 100644 index 00000000..aff3a1fa --- /dev/null +++ b/tun_memory.go @@ -0,0 +1,178 @@ +package tun + +import ( + "os" + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common/buf" +) + +const ( + memoryTunDefaultBatchSize = 128 + memoryTunMaxPacketSize = 65535 +) + +type MemoryTunOptions struct { + MTU int + Headroom int + RearSpace int + BatchSize int + Outbound func(packets []*buf.Buffer) + Route func(packet []byte) *OutboundQueue +} + +type MemoryTun struct { + mtu atomic.Int32 + headroom int + rearSpace int + batchSize int + route func(packet []byte) *OutboundQueue + attached atomic.Bool + closed atomic.Bool + inboundAccess sync.Mutex + inbound goPacketRing + inboundHeadroom atomic.Int32 + inboundRearSpace atomic.Int32 + outbound *OutboundQueue + writableAccess sync.Mutex + writable []*OutboundQueue + writablePending atomic.Bool + engineWake goSignal + closeSignal chan struct{} +} + +func NewMemoryTun(options MemoryTunOptions) *MemoryTun { + batchSize := options.BatchSize + if batchSize <= 0 { + batchSize = memoryTunDefaultBatchSize + } + memoryTun := &MemoryTun{ + headroom: options.Headroom, + rearSpace: options.RearSpace, + batchSize: batchSize, + route: options.Route, + inbound: newGoPacketRing(4 * batchSize), + engineWake: make(goSignal, 1), + closeSignal: make(chan struct{}), + } + outboundHandler := options.Outbound + if outboundHandler == nil { + outboundHandler = buf.ReleaseMulti + } + memoryTun.outbound = memoryTun.NewOutboundQueue(outboundHandler) + memoryTun.mtu.Store(int32(options.MTU)) + return memoryTun +} + +func (t *MemoryTun) Name() (string, error) { + return "memory", nil +} + +func (t *MemoryTun) Start() error { + return nil +} + +func (t *MemoryTun) UpdateRouteOptions(tunOptions Options) error { + return nil +} + +func (t *MemoryTun) MTU() int { + return int(t.mtu.Load()) +} + +func (t *MemoryTun) UpdateMTU(mtu int) { + t.mtu.Store(int32(mtu)) +} + +func (t *MemoryTun) Read(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (t *MemoryTun) Write(p []byte) (int, error) { + packets := [1][]byte{p} + _, err := t.WritePackets(packets[:]) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (t *MemoryTun) WritePackets(packets [][]byte) (int, error) { + t.inboundAccess.Lock() + if t.closed.Load() { + t.inboundAccess.Unlock() + return 0, os.ErrClosed + } + if !t.attached.Load() { + t.inboundAccess.Unlock() + return 0, nil + } + headroom := int(t.inboundHeadroom.Load()) + rearSpace := int(t.inboundRearSpace.Load()) + wasEmpty := t.inbound.empty() + accepted := 0 + for _, packet := range packets { + if len(packet) == 0 || len(packet) > memoryTunMaxPacketSize { + continue + } + if t.inbound.full() { + continue + } + buffer := buf.NewSize(headroom + len(packet) + rearSpace) + buffer.Resize(headroom, 0) + buffer.Write(packet) + t.inbound.push(buffer, nil, 0) + accepted++ + } + t.inboundAccess.Unlock() + if wasEmpty && accepted > 0 { + t.engineWake.notify() + } + return accepted, nil +} + +func (t *MemoryTun) Close() error { + if t.closed.Swap(true) { + return nil + } + t.inboundAccess.Lock() + t.inbound.releaseAll() + t.inboundAccess.Unlock() + t.outbound.Close() + close(t.closeSignal) + return nil +} + +func (t *MemoryTun) queueWritable(queue *OutboundQueue) { + t.writableAccess.Lock() + if queue.writableQueued { + t.writableAccess.Unlock() + return + } + queue.writableQueued = true + t.writable = append(t.writable, queue) + t.writablePending.Store(true) + t.writableAccess.Unlock() + t.engineWake.notify() +} + +func (t *MemoryTun) takeWritable(spare []*OutboundQueue) []*OutboundQueue { + if !t.writablePending.Swap(false) { + return spare + } + t.writableAccess.Lock() + queues := t.writable + t.writable = spare + for _, queue := range queues { + queue.writableQueued = false + } + t.writableAccess.Unlock() + return queues +} + +func (t *MemoryTun) newOutboundPacket(length int) *buf.Buffer { + packet := buf.NewSize(t.headroom + length + t.rearSpace) + packet.Resize(t.headroom, 0) + return packet +} diff --git a/tun_memory_queue.go b/tun_memory_queue.go new file mode 100644 index 00000000..023982ea --- /dev/null +++ b/tun_memory_queue.go @@ -0,0 +1,248 @@ +package tun + +import ( + "sync" + "sync/atomic" + + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/buf" +) + +type OutboundQueue struct { + tun *MemoryTun + handler func(packets []*buf.Buffer) + access sync.Mutex + control goPacketRing + datagram goPacketRing + data goPacketRing + dataTurn bool + dataQueued atomic.Int32 + transmitInterest bool + writableQueued bool + closed atomic.Bool + readerWake goSignal + closeSignal chan struct{} +} + +func (t *MemoryTun) NewOutboundQueue(handler func(packets []*buf.Buffer)) *OutboundQueue { + queue := &OutboundQueue{ + tun: t, + handler: handler, + control: newGoPacketRing(2 * t.batchSize), + datagram: newGoPacketRing(2 * t.batchSize), + data: newGoPacketRing(2 * t.batchSize), + readerWake: make(goSignal, 1), + closeSignal: make(chan struct{}), + } + go queue.loopHandle() + return queue +} + +func (q *OutboundQueue) WriteBuffers(buffers []*buf.Buffer) { + q.access.Lock() + if q.closed.Load() { + q.access.Unlock() + buf.ReleaseMulti(buffers) + return + } + wasEmpty := q.emptyLocked() + for _, buffer := range buffers { + if buffer.IsEmpty() || q.datagram.full() { + buffer.Release() + continue + } + q.pushLocked(&q.datagram, buffer, nil, 0) + } + notify := wasEmpty && !q.emptyLocked() + q.access.Unlock() + if notify { + q.readerWake.notify() + } +} + +func (q *OutboundQueue) Close() error { + q.access.Lock() + if q.closed.Swap(true) { + q.access.Unlock() + return nil + } + q.control.releaseAll() + q.datagram.releaseAll() + q.data.releaseAll() + q.dataQueued.Store(0) + q.transmitInterest = false + q.access.Unlock() + close(q.closeSignal) + q.tun.queueWritable(q) + return nil +} + +func (q *OutboundQueue) loopHandle() { + packets := make([]*buf.Buffer, q.tun.batchSize) + for { + count := q.drain(packets) + if count > 0 { + q.handler(packets[:count]) + clear(packets[:count]) + continue + } + select { + case <-q.readerWake: + case <-q.closeSignal: + return + } + } +} + +func (q *OutboundQueue) drain(buffers []*buf.Buffer) int { + q.access.Lock() + count := 0 + for count < len(buffers) { + buffer, owner, segmentEnd := q.popLocked() + if buffer == nil { + break + } + if owner != nil { + owner.frameDequeued(buffer.Len(), segmentEnd) + } + buffers[count] = buffer + count++ + } + writable := q.transmitInterest && !q.data.full() + if writable { + q.transmitInterest = false + } + q.access.Unlock() + if writable { + q.tun.queueWritable(q) + } + return count +} + +func (q *OutboundQueue) emptyLocked() bool { + return q.control.empty() && q.datagram.empty() && q.data.empty() +} + +func (q *OutboundQueue) popLocked() (*buf.Buffer, *GoConn, uint64) { + if !q.control.empty() { + return q.control.pop() + } + if !q.datagram.empty() && (q.data.empty() || !q.dataTurn) { + q.dataTurn = true + return q.datagram.pop() + } + if !q.data.empty() { + q.dataTurn = false + q.dataQueued.Add(-1) + return q.data.pop() + } + return nil, nil, 0 +} + +func (q *OutboundQueue) enqueue(packet *buf.Buffer, fragments [][]byte, class goOutboundClass, owner *GoConn, segmentEnd uint64) error { + q.access.Lock() + ring := &q.control + switch class { + case goOutboundDatagram: + ring = &q.datagram + case goOutboundData: + ring = &q.data + } + var err error + switch { + case q.closed.Load(): + err = errGoFrameDropped + case ring.full() && class == goOutboundData: + err = errGoTransmitBlocked + case ring.capacity-ring.count < max(len(fragments), 1): + err = errGoFrameDropped + } + if err != nil { + q.access.Unlock() + packet.Release() + return err + } + wasEmpty := q.emptyLocked() + if fragments == nil { + q.pushLocked(ring, packet, owner, segmentEnd) + } else { + for _, fragment := range fragments { + fragmentPacket := q.tun.newOutboundPacket(len(fragment)) + common.Must1(fragmentPacket.Write(fragment)) + q.pushLocked(ring, fragmentPacket, owner, segmentEnd) + } + packet.Release() + } + q.access.Unlock() + if wasEmpty { + q.readerWake.notify() + } + return nil +} + +func (q *OutboundQueue) pushLocked(ring *goPacketRing, packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + if owner != nil { + owner.frameEnqueued(packet.Len()) + } + if ring == &q.data { + q.dataQueued.Add(1) + } + ring.push(packet, owner, segmentEnd) +} + +const goPacketRingInitialSize = 16 + +type goPacketRing struct { + entries []goPacketEntry + capacity int + head int + count int +} + +type goPacketEntry struct { + packet *buf.Buffer + owner *GoConn + segmentEnd uint64 +} + +func newGoPacketRing(capacity int) goPacketRing { + return goPacketRing{capacity: capacity} +} + +func (r *goPacketRing) empty() bool { + return r.count == 0 +} + +func (r *goPacketRing) full() bool { + return r.count == r.capacity +} + +func (r *goPacketRing) push(packet *buf.Buffer, owner *GoConn, segmentEnd uint64) { + if r.count == len(r.entries) { + entries := make([]goPacketEntry, min(max(2*len(r.entries), goPacketRingInitialSize), r.capacity)) + copied := copy(entries, r.entries[r.head:]) + copy(entries[copied:], r.entries[:r.head]) + r.entries = entries + r.head = 0 + } + r.entries[(r.head+r.count)%len(r.entries)] = goPacketEntry{packet: packet, owner: owner, segmentEnd: segmentEnd} + r.count++ +} + +func (r *goPacketRing) pop() (*buf.Buffer, *GoConn, uint64) { + entry := r.entries[r.head] + r.entries[r.head] = goPacketEntry{} + r.head = (r.head + 1) % len(r.entries) + r.count-- + return entry.packet, entry.owner, entry.segmentEnd +} + +func (r *goPacketRing) releaseAll() { + for !r.empty() { + packet, owner, segmentEnd := r.pop() + if owner != nil { + owner.frameDequeued(packet.Len(), segmentEnd) + } + packet.Release() + } +} diff --git a/tun_windows.go b/tun_windows.go index 6dfce2f2..c612bd16 100644 --- a/tun_windows.go +++ b/tun_windows.go @@ -22,7 +22,10 @@ import ( "golang.org/x/sys/windows" ) -var TunnelType = "sing-tun" +var ( + TunnelType = "sing-tun" + ipv6AllRoutersMulticast = winsys.FWP_BYTE_ARRAY16{ByteArray16: [16]uint8{0xff, 0x02, 15: 0x02}} +) type NativeTun struct { adapter *wintun.Adapter @@ -273,6 +276,47 @@ func (t *NativeTun) Start() error { }*/ if len(t.options.Inet6Address) == 0 { + ndpCondition := make([]winsys.FWPM_FILTER_CONDITION0, 4) + ndpCondition[0].FieldKey = winsys.FWPM_CONDITION_IP_PROTOCOL + ndpCondition[0].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[0].ConditionValue.Type = winsys.FWP_UINT8 + ndpCondition[0].ConditionValue.Value = uintptr(winsys.IPPROTO_ICMPV6) + ndpCondition[1].FieldKey = winsys.FWPM_CONDITION_ICMP_TYPE + ndpCondition[1].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[1].ConditionValue.Type = winsys.FWP_UINT16 + ndpCondition[2].FieldKey = winsys.FWPM_CONDITION_ICMP_CODE + ndpCondition[2].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[2].ConditionValue.Type = winsys.FWP_UINT16 + ndpCondition[2].ConditionValue.Value = 0 + ndpCondition[3].FieldKey = winsys.FWPM_CONDITION_IP_REMOTE_ADDRESS + ndpCondition[3].MatchType = winsys.FWP_MATCH_EQUAL + ndpCondition[3].ConditionValue.Type = winsys.FWP_BYTE_ARRAY16_TYPE + ndpCondition[3].ConditionValue.Value = uintptr(unsafe.Pointer(&ipv6AllRoutersMulticast)) + for _, ndpMessage := range []struct { + name string + icmpType uint16 + numConditions uint32 + }{ + {"router solicitation", 133, 4}, + {"neighbor solicitation", 135, 3}, + {"neighbor advertisement", 136, 3}, + } { + ndpCondition[1].ConditionValue.Value = uintptr(ndpMessage.icmpType) + ndpFilter := winsys.FWPM_FILTER0{} + ndpFilter.FilterCondition = &ndpCondition[0] + ndpFilter.NumFilterConditions = ndpMessage.numConditions + ndpFilter.DisplayData = winsys.CreateDisplayData(TunnelType, "allow ipv6 "+ndpMessage.name) + ndpFilter.SubLayerKey = subLayerKey + ndpFilter.LayerKey = winsys.FWPM_LAYER_ALE_AUTH_CONNECT_V6 + ndpFilter.Action.Type = winsys.FWP_ACTION_PERMIT + ndpFilter.Weight.Type = winsys.FWP_UINT8 + ndpFilter.Weight.Value = uintptr(13) + err = winsys.FwpmFilterAdd0(engine, &ndpFilter, 0, &filterId) + if err != nil { + return os.NewSyscallError("FwpmFilterAdd0", err) + } + } + blockFilter := winsys.FWPM_FILTER0{} blockFilter.DisplayData = winsys.CreateDisplayData(TunnelType, "block ipv6") blockFilter.SubLayerKey = subLayerKey @@ -381,10 +425,9 @@ retry: if t.close.Load() == 1 { return 0, os.ErrClosed } - var packet []byte - packet, err = t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: n = copy(p, packet) t.session.ReleaseReceivePacket(packet) t.rate.update(uint64(n)) @@ -401,7 +444,7 @@ retry: case windows.ERROR_INVALID_DATA: return 0, errors.New("send ring corrupt") } - return 0, fmt.Errorf("read failed: %w", err) + return 0, fmt.Errorf("read failed: %w", errno) } } @@ -434,9 +477,9 @@ retry: t.running.Done() return nil, nil, os.ErrClosed } - packet, err := t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: packetSize := len(packet) t.rate.update(uint64(packetSize)) return packet, func() { @@ -458,7 +501,7 @@ retry: return nil, nil, errors.New("send ring corrupt") } t.running.Done() - return nil, nil, fmt.Errorf("read failed: %w", err) + return nil, nil, fmt.Errorf("read failed: %w", errno) } } @@ -475,9 +518,9 @@ retry: if t.close.Load() == 1 { return os.ErrClosed } - packet, err := t.session.ReceivePacket() - switch err { - case nil: + packet, errno := t.session.ReceivePacket() + switch errno { + case 0: packetSize := len(packet) block(packet) t.session.ReleaseReceivePacket(packet) @@ -495,7 +538,7 @@ retry: case windows.ERROR_INVALID_DATA: return errors.New("send ring corrupt") } - return fmt.Errorf("read failed: %w", err) + return fmt.Errorf("read failed: %w", errno) } } @@ -506,19 +549,19 @@ func (t *NativeTun) Write(p []byte) (n int, err error) { return 0, os.ErrClosed } t.rate.update(uint64(len(p))) - packet, err := t.session.AllocateSendPacket(len(p)) + packet, errno := t.session.AllocateSendPacket(len(p)) copy(packet, p) - if err == nil { + if errno == 0 { t.session.SendPacket(packet) return len(p), nil } - switch err { + switch errno { case windows.ERROR_HANDLE_EOF: return 0, os.ErrClosed case windows.ERROR_BUFFER_OVERFLOW: return 0, nil // Dropping when ring is full. } - return 0, fmt.Errorf("write failed: %w", err) + return 0, fmt.Errorf("write failed: %w", errno) } func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { @@ -532,8 +575,8 @@ func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { packetSize += len(packetElement) } t.rate.update(uint64(packetSize)) - packet, err := t.session.AllocateSendPacket(packetSize) - if err == nil { + packet, errno := t.session.AllocateSendPacket(packetSize) + if errno == 0 { var index int for _, packetElement := range packetElementList { index += copy(packet[index:], packetElement) @@ -541,13 +584,71 @@ func (t *NativeTun) write(packetElementList [][]byte) (n int, err error) { t.session.SendPacket(packet) return } - switch err { + switch errno { case windows.ERROR_HANDLE_EOF: return 0, os.ErrClosed case windows.ERROR_BUFFER_OVERFLOW: return 0, nil // Dropping when ring is full. } - return 0, fmt.Errorf("write failed: %w", err) + return 0, fmt.Errorf("write failed: %w", errno) +} + +func (t *NativeTun) readWaitHandle() windows.Handle { + return t.readWait +} + +func (t *NativeTun) receiveInto(buffer []byte) (int, error) { + t.running.Add(1) + defer t.running.Done() + for { + if t.close.Load() == 1 { + return 0, os.ErrClosed + } + packet, errno := t.session.ReceivePacket() + if errno != 0 { + switch errno { + case windows.ERROR_NO_MORE_ITEMS: + return 0, nil + case windows.ERROR_HANDLE_EOF: + return 0, os.ErrClosed + case windows.ERROR_INVALID_DATA: + return 0, E.New("wintun: receive ring corrupt") + } + return 0, E.Cause(errno, "wintun: receive packet") + } + if len(packet) > len(buffer) { + t.session.ReleaseReceivePacket(packet) + continue + } + n := copy(buffer, packet) + t.session.ReleaseReceivePacket(packet) + return n, nil + } +} + +func (t *NativeTun) transmitGather(segments [][]byte) error { + t.running.Add(1) + defer t.running.Done() + if t.close.Load() == 1 { + return os.ErrClosed + } + var packetSize int + for _, segment := range segments { + packetSize += len(segment) + } + packet, errno := t.session.AllocateSendPacket(packetSize) + if errno != 0 { + if errno == windows.ERROR_HANDLE_EOF { + return os.ErrClosed + } + return errno + } + var index int + for _, segment := range segments { + index += copy(packet[index:], segment) + } + t.session.SendPacket(packet) + return nil } func (t *NativeTun) Close() error { @@ -556,7 +657,9 @@ func (t *NativeTun) Close() error { t.close.Store(1) windows.SetEvent(t.readWait) t.running.Wait() - t.session.End() + if t.session != (wintun.Session{}) { + t.session.End() + } t.adapter.Close() if t.fwpmSession != 0 { winsys.FwpmEngineClose0(t.fwpmSession) diff --git a/tun_windows_gvisor.go b/tun_windows_gvisor.go index 9e68b444..cf9683d2 100644 --- a/tun_windows_gvisor.go +++ b/tun_windows_gvisor.go @@ -21,8 +21,6 @@ func (t *NativeTun) NewEndpoint() (stack.LinkEndpoint, stack.NICOptions, error) return &WintunEndpoint{tun: t}, stack.NICOptions{}, nil } -var _ stack.LinkEndpoint = (*WintunEndpoint)(nil) - type WintunEndpoint struct { tun *NativeTun access sync.RWMutex diff --git a/udp_egress.go b/udp_egress.go index 0ace53e8..df4e46f6 100644 --- a/udp_egress.go +++ b/udp_egress.go @@ -24,7 +24,6 @@ type UDPEgressPoolOptions struct { Control control.Func InterfaceFinder control.InterfaceFinder InterfaceMonitor DefaultInterfaceMonitor - ExcludeInterface string IsExempt func() bool } @@ -34,7 +33,6 @@ type UDPEgressPool struct { control control.Func interfaceFinder control.InterfaceFinder interfaceMonitor DefaultInterfaceMonitor - excludeInterface string isExempt func() bool access sync.Mutex port uint16 @@ -70,7 +68,6 @@ func NewUDPEgressPool(options UDPEgressPoolOptions) *UDPEgressPool { control: options.Control, interfaceFinder: options.InterfaceFinder, interfaceMonitor: options.InterfaceMonitor, - excludeInterface: options.ExcludeInterface, isExempt: options.IsExempt, anchorInterfaceIndex: -1, members: make(map[udpEgressSpec]*udpEgressMember), @@ -182,8 +179,7 @@ func (p *UDPEgressPool) rebuildLocked() { networkInterface.Flags&net.FlagLoopback != 0 || networkInterface.Flags&net.FlagPointToPoint != 0 || networkInterface.Flags&net.FlagBroadcast == 0 || - networkInterface.Index == p.anchorInterfaceIndex || - networkInterface.Name == p.excludeInterface { + networkInterface.Index == p.anchorInterfaceIndex { continue } for _, prefix := range networkInterface.Addresses { diff --git a/udp_nat.go b/udp_nat.go index 6d5dd945..3f19b38d 100644 --- a/udp_nat.go +++ b/udp_nat.go @@ -14,9 +14,9 @@ import ( "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/buf" + "github.com/sagernet/sing/common/bufio" "github.com/sagernet/sing/common/canceler" "github.com/sagernet/sing/common/control" - "github.com/sagernet/sing/common/memory" M "github.com/sagernet/sing/common/metadata" N "github.com/sagernet/sing/common/network" "github.com/sagernet/sing/common/pipe" @@ -52,8 +52,7 @@ type UDPNatOptions struct { Filtering NATFiltering MaxSize uint32 - InterfaceFinder control.InterfaceFinder - ExcludeInterface []string + InterfaceFinder control.InterfaceFinder } type udpNatSessionKey struct { @@ -92,15 +91,14 @@ type UDPNat struct { timeout time.Duration mapping NATMapping filtering NATFiltering - cache *freelru.Cache[udpNatSessionKey, *udpNatConn] - filterCache *freelru.Cache[udpNatFilterKey, *udpNatConn] + cache *freelru.Cache[udpNatSessionKey, *UDPNatConn] + filterCache *freelru.Cache[udpNatFilterKey, *UDPNatConn] nextFilterSessionID atomic.Uint64 interfaceFinder control.InterfaceFinder - excludeInterface []string interfaceElement *list.Element[control.InterfaceUpdateCallback] egress atomic.Pointer[udpNatEgressTable] classAccess sync.Mutex - classConns map[uint32]map[*udpNatConn]struct{} + classConns map[uint32]map[*UDPNatConn]struct{} cleanup *udpNatCleanupQueue state atomic.Uint32 lifecycleAccess sync.Mutex @@ -117,35 +115,32 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { if maxSize == 0 { if runtime.GOOS == "ios" { maxSize = 4096 - } else if totalMemory := memory.Total(); totalMemory == 0 { - maxSize = 16384 } else { - maxSize = uint32(min(max(totalMemory/16384, 4096), 16384)) + maxSize = 16384 } } hasher := maphash.NewHasher[udpNatSessionKey]() - cache := common.Must1(freelru.New[udpNatSessionKey, *udpNatConn](maxSize, hasher.Hash32, options.Shared)) - var filterCache *freelru.Cache[udpNatFilterKey, *udpNatConn] + cache := common.Must1(freelru.New[udpNatSessionKey, *UDPNatConn](maxSize, hasher.Hash32, options.Shared)) + var filterCache *freelru.Cache[udpNatFilterKey, *UDPNatConn] if NATMapping(options.Filtering) > options.Mapping { filterHasher := maphash.NewHasher[udpNatFilterKey]() - filterCache = common.Must1(freelru.New[udpNatFilterKey, *udpNatConn](maxSize, filterHasher.Hash32, options.Shared)) + filterCache = common.Must1(freelru.New[udpNatFilterKey, *UDPNatConn](maxSize, filterHasher.Hash32, options.Shared)) } service := &UDPNat{ - handler: options.Handler, - prepare: options.Prepare, - timeout: options.Timeout, - mapping: options.Mapping, - filtering: options.Filtering, - cache: cache, - filterCache: filterCache, - interfaceFinder: options.InterfaceFinder, - excludeInterface: options.ExcludeInterface, - classConns: make(map[uint32]map[*udpNatConn]struct{}), - cleanupDone: make(chan struct{}), + handler: options.Handler, + prepare: options.Prepare, + timeout: options.Timeout, + mapping: options.Mapping, + filtering: options.Filtering, + cache: cache, + filterCache: filterCache, + interfaceFinder: options.InterfaceFinder, + classConns: make(map[uint32]map[*UDPNatConn]struct{}), + cleanupDone: make(chan struct{}), } service.cleanup = newUDPNatCleanupQueue(service) cache.SetLifetime(options.Timeout) - cache.SetHealthCheck(func(_ udpNatSessionKey, conn *udpNatConn) bool { + cache.SetHealthCheck(func(_ udpNatSessionKey, conn *UDPNatConn) bool { select { case <-conn.doneChan: return false @@ -153,11 +148,11 @@ func NewUDPNat(options UDPNatOptions) *UDPNat { return true } }) - cache.SetOnEvict(func(_ udpNatSessionKey, conn *udpNatConn) { + cache.SetOnEvict(func(_ udpNatSessionKey, conn *UDPNatConn) { conn.closeFromCache() }) if filterCache != nil { - filterCache.SetOnEvict(func(key udpNatFilterKey, conn *udpNatConn) { + filterCache.SetOnEvict(func(key udpNatFilterKey, conn *UDPNatConn) { conn.removeFilterPeer(key.peer) }) } @@ -201,9 +196,6 @@ func (s *UDPNat) updateInterfaces(interfaces []control.Interface) { networkInterface.Flags&net.FlagBroadcast == 0 { continue } - if slices.Contains(s.excludeInterface, networkInterface.Name) { - continue - } for _, prefix := range networkInterface.Addresses { if !prefix.Addr().IsGlobalUnicast() { continue @@ -212,7 +204,7 @@ func (s *UDPNat) updateInterfaces(interfaces []control.Interface) { } } s.egress.Store(newUDPNatEgressTable(entries)) - var closeConns []*udpNatConn + var closeConns []*UDPNatConn s.classAccess.Lock() for interfaceIndex, conns := range s.classConns { if !slices.ContainsFunc(entries, func(entry udpNatEgressEntry) bool { @@ -295,18 +287,18 @@ func (t *udpNatEgressTable) lookup(address netip.Addr) uint32 { return 0 } -func (s *UDPNat) registerClass(conn *udpNatConn) { +func (s *UDPNat) registerClass(conn *UDPNatConn) { s.classAccess.Lock() conns := s.classConns[conn.interfaceIndex] if conns == nil { - conns = make(map[*udpNatConn]struct{}) + conns = make(map[*UDPNatConn]struct{}) s.classConns[conn.interfaceIndex] = conns } conns[conn] = struct{}{} s.classAccess.Unlock() } -func (s *UDPNat) unregisterClass(conn *udpNatConn) { +func (s *UDPNat) unregisterClass(conn *UDPNatConn) { s.classAccess.Lock() conns := s.classConns[conn.interfaceIndex] if conns != nil { @@ -319,7 +311,7 @@ func (s *UDPNat) unregisterClass(conn *udpNatConn) { } func (s *UDPNat) NewPacket(bufferSlices [][]byte, source M.Socksaddr, destination M.Socksaddr, userData any) { - conn, ok := s.getOrCreateConn(source, destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(source, destination), source, destination, userData, nil) if !ok { return } @@ -336,10 +328,7 @@ func (s *UDPNat) NewPacket(bufferSlices [][]byte, source M.Socksaddr, destinatio conn.enqueue(buffer, destination) } -func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, userData any) (*udpNatConn, bool) { - if s.state.Load() != udpNatStateStarted { - return nil, false - } +func (s *UDPNat) sessionKey(source M.Socksaddr, destination M.Socksaddr) udpNatSessionKey { key := udpNatSessionKey{ sourceAddr: source.Addr.Unmap(), sourcePort: source.Port, @@ -353,23 +342,31 @@ func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, us key.destinationAddr = destination.Addr.Unmap() key.destinationPort = destination.Port } + return key +} + +func (s *UDPNat) getOrCreate(key udpNatSessionKey, source M.Socksaddr, destination M.Socksaddr, userData any, initialize func(*UDPNatConn)) (*UDPNatConn, bool) { + if s.state.Load() != udpNatStateStarted { + return nil, false + } var ( newContext context.Context newOnClose N.CloseHandlerFunc ) - conn, loaded, ok := s.cache.GetAndRefreshOrAdd(key, func() (*udpNatConn, bool) { + conn, loaded, ok := s.cache.GetAndRefreshOrAdd(key, func() (*UDPNatConn, bool) { ok, ctx, writer, onClose := s.prepare(source, destination, userData) if !ok { return nil, false } - newConn := &udpNatConn{ - service: s, - key: key, - writer: writer, - localAddr: source, - packetChan: make(chan *N.PacketBuffer, 64), - doneChan: make(chan struct{}), - readDeadline: pipe.MakeDeadline(), + newConn := &UDPNatConn{ + service: s, + key: key, + writer: writer, + localAddr: source, + packetChan: make(chan *N.PacketBuffer, 64), + doneChan: make(chan struct{}), + readDeadline: pipe.MakeDeadline(), + writeDeadline: pipe.MakeDeadline(), } newConn.cleanupEntry = &udpNatCleanupEntry{ conn: newConn, @@ -410,7 +407,13 @@ func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, us } if !loaded { s.cleanup.addOrUpdate(conn.cleanupEntry, time.Now().Add(s.timeout)) + if initialize != nil { + initialize(conn) + } if conn.isClosed() { + if newOnClose != nil { + newOnClose(io.ErrClosedPipe) + } return nil, false } go s.handler.NewPacketConnectionEx(newContext, conn, source, destination, newOnClose) @@ -419,7 +422,7 @@ func (s *UDPNat) getOrCreateConn(source M.Socksaddr, destination M.Socksaddr, us return conn, true } -func (c *udpNatConn) enqueue(buffer *buf.Buffer, destination M.Socksaddr) { +func (c *UDPNatConn) enqueue(buffer *buf.Buffer, destination M.Socksaddr) { c.packetAccess.RLock() select { case <-c.doneChan: @@ -448,7 +451,7 @@ func (s *UDPNat) NewPacketBatch(buffers []*buf.Buffer, sources []M.Socksaddr, de return } for index, buffer := range buffers { - conn, ok := s.getOrCreateConn(sources[index], destination, userData) + conn, ok := s.getOrCreate(s.sessionKey(sources[index], destination), sources[index], destination, userData, nil) if !ok { buffer.Release() continue @@ -477,13 +480,12 @@ func (s *UDPNat) PurgeExpired() { } var ( - _ N.PacketConn = (*udpNatConn)(nil) - _ canceler.PacketConn = (*udpNatConn)(nil) - _ N.PacketBatchReadWaitCreator = (*udpNatConn)(nil) - _ N.PacketBatchWriteCreator = (*udpNatConn)(nil) + _ canceler.PacketConn = (*UDPNatConn)(nil) + _ N.PacketBatchReadWaitCreator = (*UDPNatConn)(nil) + _ N.PacketBatchWriteCreator = (*UDPNatConn)(nil) ) -type udpNatConn struct { +type UDPNatConn struct { service *UDPNat key udpNatSessionKey interfaceIndex uint32 @@ -494,6 +496,7 @@ type udpNatConn struct { closeOnce sync.Once doneChan chan struct{} readDeadline pipe.Deadline + writeDeadline pipe.Deadline readWaitOptions atomic.Pointer[N.ReadWaitOptions] readBatch *udpNatReadBatch cleanupEntry *udpNatCleanupEntry @@ -509,7 +512,7 @@ type udpNatReadBatch struct { destinations []M.Socksaddr } -func (c *udpNatConn) loadReadWaitOptions() N.ReadWaitOptions { +func (c *UDPNatConn) loadReadWaitOptions() N.ReadWaitOptions { options := c.readWaitOptions.Load() if options == nil { return N.ReadWaitOptions{} @@ -517,7 +520,7 @@ func (c *udpNatConn) loadReadWaitOptions() N.ReadWaitOptions { return *options } -func (c *udpNatConn) addFilterPeer(destination M.Socksaddr) { +func (c *UDPNatConn) addFilterPeer(destination M.Socksaddr) { if c.filterSessionID == 0 || !destination.IsIP() { return } @@ -553,13 +556,13 @@ func (c *udpNatConn) addFilterPeer(destination M.Socksaddr) { } } -func (c *udpNatConn) removeFilterPeer(peer netip.AddrPort) { +func (c *UDPNatConn) removeFilterPeer(peer netip.AddrPort) { c.filterAccess.Lock() delete(c.filterPeers, peer) c.filterAccess.Unlock() } -func (c *udpNatConn) clearFilterPeers() { +func (c *UDPNatConn) clearFilterPeers() { if c.filterSessionID == 0 { return } @@ -575,7 +578,7 @@ func (c *udpNatConn) clearFilterPeers() { } } -func (c *udpNatConn) allowPeer(destination M.Socksaddr) bool { +func (c *UDPNatConn) allowPeer(destination M.Socksaddr) bool { if c.service.filtering == NATFilteringEndpointIndependent || !destination.IsIP() { return true } @@ -593,10 +596,19 @@ func (c *udpNatConn) allowPeer(destination M.Socksaddr) bool { return loaded && filterConn == c } -func (c *udpNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error) { +func (c *UDPNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error) { + select { + case <-c.readDeadline.Wait(): + return M.Socksaddr{}, os.ErrDeadlineExceeded + default: + } select { case p := <-c.packetChan: - _, err = buffer.ReadOnceFrom(p.Buffer) + if buffer.IsFull() { + err = io.ErrShortBuffer + } else { + _, err = buffer.Write(p.Buffer.Bytes()) + } destination := p.Destination p.Buffer.Release() N.PutPacketBuffer(p) @@ -608,7 +620,12 @@ func (c *udpNatConn) ReadPacket(buffer *buf.Buffer) (addr M.Socksaddr, err error } } -func (c *udpNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { +func (c *UDPNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) error { + err := c.writeError() + if err != nil { + buffer.Release() + return err + } if !c.allowPeer(destination) { buffer.Release() return nil @@ -616,29 +633,62 @@ func (c *udpNatConn) WritePacket(buffer *buf.Buffer, destination M.Socksaddr) er return c.writer.WritePacket(buffer, destination) } -func (c *udpNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { +func (c *UDPNatConn) CreatePacketBatchWriter() (N.PacketBatchWriter, bool) { if c.service.filtering != NATFilteringEndpointIndependent { return nil, false } - if creator, isCreator := c.writer.(N.PacketBatchWriteCreator); isCreator { - return creator.CreatePacketBatchWriter() + writer, created := bufio.CreatePacketBatchWriter(c.writer) + if !created { + return nil, false } - if writer, isWriter := c.writer.(N.PacketBatchWriter); isWriter { - return writer, true + return &udpNatPacketBatchWriter{conn: c, writer: writer}, true +} + +type udpNatPacketBatchWriter struct { + conn *UDPNatConn + writer N.PacketBatchWriter +} + +func (w *udpNatPacketBatchWriter) WritePacketBatch(buffers []*buf.Buffer, destinations []M.Socksaddr) error { + err := w.conn.writeError() + if err != nil { + buf.ReleaseMulti(buffers) + return err } - return nil, false + return w.writer.WritePacketBatch(buffers, destinations) +} + +func (w *udpNatPacketBatchWriter) Upstream() any { + return w.conn } -func (c *udpNatConn) InitializeReadWaiter(options N.ReadWaitOptions) (needCopy bool) { +func (c *UDPNatConn) writeError() error { + if c.isClosed() { + return os.ErrClosed + } + select { + case <-c.writeDeadline.Wait(): + return os.ErrDeadlineExceeded + default: + return nil + } +} + +func (c *UDPNatConn) InitializeReadWaiter(options N.ReadWaitOptions) (needCopy bool) { c.readWaitOptions.Store(&options) return false } -func (c *udpNatConn) WaitReadPacket() (buffer *buf.Buffer, destination M.Socksaddr, err error) { +func (c *UDPNatConn) WaitReadPacket() (buffer *buf.Buffer, destination M.Socksaddr, err error) { return c.waitReadPacket(c.loadReadWaitOptions()) } -func (c *udpNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buffer, destination M.Socksaddr, err error) { +func (c *UDPNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buffer, destination M.Socksaddr, err error) { + select { + case <-c.readDeadline.Wait(): + return nil, M.Socksaddr{}, os.ErrDeadlineExceeded + default: + } select { case packet := <-c.packetChan: buffer = options.Copy(packet.Buffer) @@ -652,11 +702,16 @@ func (c *udpNatConn) waitReadPacket(options N.ReadWaitOptions) (buffer *buf.Buff } } -func (c *udpNatConn) CreatePacketBatchReadWaiter() (N.PacketBatchReadWaiter, bool) { +func (c *UDPNatConn) CreatePacketBatchReadWaiter() (N.PacketBatchReadWaiter, bool) { return c, true } -func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M.Socksaddr, err error) { +func (c *UDPNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M.Socksaddr, err error) { + batch := c.readBatch + if batch != nil { + clear(batch.buffers) + clear(batch.destinations) + } options := c.loadReadWaitOptions() buffer, destination, err := c.waitReadPacket(options) if err != nil { @@ -666,13 +721,9 @@ func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M. if batchSize <= 0 { batchSize = 1 } - batch := c.readBatch if batch == nil { batch = new(udpNatReadBatch) c.readBatch = batch - } else { - clear(batch.buffers) - clear(batch.destinations) } buffers = batch.buffers[:0] destinations = batch.destinations[:0] @@ -695,7 +746,7 @@ func (c *udpNatConn) WaitReadPackets() (buffers []*buf.Buffer, destinations []M. return } -func (c *udpNatConn) Timeout() time.Duration { +func (c *UDPNatConn) Timeout() time.Duration { rawConn, lifetime, loaded := c.service.cache.PeekWithLifetime(c.key) if !loaded || rawConn != c { return 0 @@ -706,7 +757,7 @@ func (c *udpNatConn) Timeout() time.Duration { return time.Until(lifetime) } -func (c *udpNatConn) SetTimeout(timeout time.Duration) bool { +func (c *UDPNatConn) SetTimeout(timeout time.Duration) bool { updated := c.service.cache.UpdateLifetime(c.key, c, timeout) if !updated { return false @@ -719,7 +770,7 @@ func (c *udpNatConn) SetTimeout(timeout time.Duration) bool { return true } -func (c *udpNatConn) Close() error { +func (c *UDPNatConn) Close() error { c.close() if c.service.state.Load() == udpNatStateStarted { c.service.cleanup.addOrUpdate(c.cleanupEntry, time.Now()) @@ -727,7 +778,7 @@ func (c *udpNatConn) Close() error { return nil } -func (c *udpNatConn) close() { +func (c *UDPNatConn) close() { c.closeOnce.Do(func() { c.packetAccess.Lock() close(c.doneChan) @@ -746,15 +797,16 @@ func (c *udpNatConn) close() { if c.interfaceIndex != 0 { c.service.unregisterClass(c) } + common.Close(c.writer) }) } -func (c *udpNatConn) closeFromCache() { +func (c *UDPNatConn) closeFromCache() { c.close() c.service.cleanup.remove(c.cleanupEntry) } -func (c *udpNatConn) isClosed() bool { +func (c *UDPNatConn) isClosed() bool { select { case <-c.doneChan: return true @@ -763,27 +815,39 @@ func (c *udpNatConn) isClosed() bool { } } -func (c *udpNatConn) LocalAddr() net.Addr { +func (c *UDPNatConn) LocalAddr() net.Addr { return c.localAddr } -func (c *udpNatConn) RemoteAddr() net.Addr { +func (c *UDPNatConn) RemoteAddr() net.Addr { return M.Socksaddr{} } -func (c *udpNatConn) SetDeadline(t time.Time) error { - return os.ErrInvalid +func (c *UDPNatConn) SetDeadline(t time.Time) error { + if c.isClosed() { + return io.ErrClosedPipe + } + c.readDeadline.Set(t) + c.writeDeadline.Set(t) + return nil } -func (c *udpNatConn) SetReadDeadline(t time.Time) error { +func (c *UDPNatConn) SetReadDeadline(t time.Time) error { + if c.isClosed() { + return io.ErrClosedPipe + } c.readDeadline.Set(t) return nil } -func (c *udpNatConn) SetWriteDeadline(t time.Time) error { - return os.ErrInvalid +func (c *UDPNatConn) SetWriteDeadline(t time.Time) error { + if c.isClosed() { + return io.ErrClosedPipe + } + c.writeDeadline.Set(t) + return nil } -func (c *udpNatConn) Upstream() any { +func (c *UDPNatConn) Upstream() any { return c.writer } diff --git a/udp_nat_cleanup.go b/udp_nat_cleanup.go index c63b8af1..a2d00908 100644 --- a/udp_nat_cleanup.go +++ b/udp_nat_cleanup.go @@ -34,7 +34,7 @@ func (s *UDPNat) Start() error { } type udpNatCleanupEntry struct { - conn *udpNatConn + conn *UDPNatConn deadline time.Time index int }