wintun: ring management moved to wintun.dll

Signed-off-by: Simon Rozman <simon@rozman.si>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
This commit is contained in:
Simon Rozman 2020-10-24 22:40:46 +02:00 committed by Jason A. Donenfeld
parent 65e03a9182
commit a3b231b31e
4 changed files with 147 additions and 214 deletions

View file

@ -9,10 +9,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
"unsafe" _ "unsafe"
"golang.org/x/sys/windows" "golang.org/x/sys/windows"
@ -40,8 +39,8 @@ type NativeTun struct {
errors chan error errors chan error
forcedMTU int forcedMTU int
rate rateJuggler rate rateJuggler
rings *wintun.RingDescriptor session wintun.Session
writeLock sync.Mutex readWait windows.Handle
} }
var WintunPool *wintun.Pool var WintunPool *wintun.Pool
@ -103,17 +102,13 @@ func CreateTUNWithRequestedGUID(ifname string, requestedGUID *windows.GUID, mtu
forcedMTU: forcedMTU, forcedMTU: forcedMTU,
} }
tun.rings, err = wintun.NewRingDescriptor() tun.session, err = wt.StartSession(0x800000) // Ring capacity, 8 MiB
if err != nil { if err != nil {
tun.Close() _, err = tun.wt.Delete(false)
return nil, fmt.Errorf("Error creating events: %v", err) close(tun.events)
} return nil, fmt.Errorf("Error starting session: %v", err)
tun.handle, err = tun.wt.Register(tun.rings)
if err != nil {
tun.Close()
return nil, fmt.Errorf("Error registering rings: %v", err)
} }
tun.readWait = tun.session.ReadWaitEvent()
return tun, nil return tun, nil
} }
@ -131,13 +126,7 @@ func (tun *NativeTun) Events() chan Event {
func (tun *NativeTun) Close() error { func (tun *NativeTun) Close() error {
tun.close = true tun.close = true
if tun.rings.Send.TailMoved != 0 { tun.session.End()
windows.SetEvent(tun.rings.Send.TailMoved) // wake the reader if it's sleeping
}
if tun.handle != windows.InvalidHandle {
windows.CloseHandle(tun.handle)
}
tun.rings.Close()
var err error var err error
if tun.wt != nil { if tun.wt != nil {
_, err = tun.wt.Delete(false) _, err = tun.wt.Delete(false)
@ -164,56 +153,34 @@ retry:
return 0, err return 0, err
default: default:
} }
if tun.close {
return 0, os.ErrClosed
}
buffHead := atomic.LoadUint32(&tun.rings.Send.Ring.Head)
if buffHead >= wintun.PacketCapacity {
return 0, os.ErrClosed
}
start := nanotime() start := nanotime()
shouldSpin := atomic.LoadUint64(&tun.rate.current) >= spinloopRateThreshold && uint64(start-atomic.LoadInt64(&tun.rate.nextStartTime)) <= rateMeasurementGranularity*2 shouldSpin := atomic.LoadUint64(&tun.rate.current) >= spinloopRateThreshold && uint64(start-atomic.LoadInt64(&tun.rate.nextStartTime)) <= rateMeasurementGranularity*2
var buffTail uint32
for { for {
buffTail = atomic.LoadUint32(&tun.rings.Send.Ring.Tail)
if buffHead != buffTail {
break
}
if tun.close { if tun.close {
return 0, os.ErrClosed return 0, os.ErrClosed
} }
if !shouldSpin || uint64(nanotime()-start) >= spinloopDuration { packet, err := tun.session.ReceivePacket()
windows.WaitForSingleObject(tun.rings.Send.TailMoved, windows.INFINITE) switch err {
goto retry case nil:
packetSize := len(packet)
copy(buff[offset:], packet)
tun.session.ReleaseReceivePacket(packet)
tun.rate.update(uint64(packetSize))
return packetSize, nil
case windows.ERROR_NO_MORE_ITEMS:
if !shouldSpin || uint64(nanotime()-start) >= spinloopDuration {
windows.WaitForSingleObject(tun.readWait, windows.INFINITE)
goto retry
}
procyield(1)
continue
case windows.ERROR_HANDLE_EOF:
return 0, os.ErrClosed
case windows.ERROR_INVALID_DATA:
return 0, errors.New("Send ring corrupt")
} }
procyield(1) return 0, fmt.Errorf("Read failed: %v", err)
} }
if buffTail >= wintun.PacketCapacity {
return 0, os.ErrClosed
}
buffContent := tun.rings.Send.Ring.Wrap(buffTail - buffHead)
if buffContent < uint32(unsafe.Sizeof(wintun.PacketHeader{})) {
return 0, errors.New("incomplete packet header in send ring")
}
packet := (*wintun.Packet)(unsafe.Pointer(&tun.rings.Send.Ring.Data[buffHead]))
if packet.Size > wintun.PacketSizeMax {
return 0, errors.New("packet too big in send ring")
}
alignedPacketSize := wintun.PacketAlign(uint32(unsafe.Sizeof(wintun.PacketHeader{})) + packet.Size)
if alignedPacketSize > buffContent {
return 0, errors.New("incomplete packet in send ring")
}
copy(buff[offset:], packet.Data[:packet.Size])
buffHead = tun.rings.Send.Ring.Wrap(buffHead + alignedPacketSize)
atomic.StoreUint32(&tun.rings.Send.Ring.Head, buffHead)
tun.rate.update(uint64(packet.Size))
return int(packet.Size), nil
} }
func (tun *NativeTun) Flush() error { func (tun *NativeTun) Flush() error {
@ -225,36 +192,22 @@ func (tun *NativeTun) Write(buff []byte, offset int) (int, error) {
return 0, os.ErrClosed return 0, os.ErrClosed
} }
packetSize := uint32(len(buff) - offset) packetSize := len(buff) - offset
tun.rate.update(uint64(packetSize)) tun.rate.update(uint64(packetSize))
alignedPacketSize := wintun.PacketAlign(uint32(unsafe.Sizeof(wintun.PacketHeader{})) + packetSize)
tun.writeLock.Lock() packet, err := tun.session.AllocateSendPacket(packetSize)
defer tun.writeLock.Unlock() if err == nil {
copy(packet, buff[offset:])
buffHead := atomic.LoadUint32(&tun.rings.Receive.Ring.Head) tun.session.SendPacket(packet)
if buffHead >= wintun.PacketCapacity { return packetSize, nil
return 0, os.ErrClosed
} }
switch err {
buffTail := atomic.LoadUint32(&tun.rings.Receive.Ring.Tail) case windows.ERROR_HANDLE_EOF:
if buffTail >= wintun.PacketCapacity {
return 0, os.ErrClosed return 0, os.ErrClosed
} case windows.ERROR_BUFFER_OVERFLOW:
buffSpace := tun.rings.Receive.Ring.Wrap(buffHead - buffTail - wintun.PacketAlignment)
if alignedPacketSize > buffSpace {
return 0, nil // Dropping when ring is full. return 0, nil // Dropping when ring is full.
} }
return 0, fmt.Errorf("Write failed: %v", err)
packet := (*wintun.Packet)(unsafe.Pointer(&tun.rings.Receive.Ring.Data[buffTail]))
packet.Size = packetSize
copy(packet.Data[:packetSize], buff[offset:])
atomic.StoreUint32(&tun.rings.Receive.Ring.Tail, tun.rings.Receive.Ring.Wrap(buffTail+alignedPacketSize))
if atomic.LoadInt32(&tun.rings.Receive.Ring.Alertable) != 0 {
windows.SetEvent(tun.rings.Receive.TailMoved)
}
return int(packetSize), nil
} }
// LUID returns Windows interface instance ID. // LUID returns Windows interface instance ID.

View file

@ -1,117 +0,0 @@
/* SPDX-License-Identifier: MIT
*
* Copyright (C) 2020 WireGuard LLC. All Rights Reserved.
*/
package wintun
import (
"runtime"
"unsafe"
"golang.org/x/sys/windows"
)
const (
PacketAlignment = 4 // Number of bytes packets are aligned to in rings
PacketSizeMax = 0xffff // Maximum packet size
PacketCapacity = 0x800000 // Ring capacity, 8MiB
PacketTrailingSize = uint32(unsafe.Sizeof(PacketHeader{})) + ((PacketSizeMax + (PacketAlignment - 1)) &^ (PacketAlignment - 1)) - PacketAlignment
ioctlRegisterRings = (51820 << 16) | (0x970 << 2) | 0 /*METHOD_BUFFERED*/ | (0x3 /*FILE_READ_DATA | FILE_WRITE_DATA*/ << 14)
)
type PacketHeader struct {
Size uint32
}
type Packet struct {
PacketHeader
Data [PacketSizeMax]byte
}
type Ring struct {
Head uint32
Tail uint32
Alertable int32
Data [PacketCapacity + PacketTrailingSize]byte
}
type RingDescriptor struct {
Send, Receive struct {
Size uint32
Ring *Ring
TailMoved windows.Handle
}
}
// Wrap returns value modulo ring capacity
func (rb *Ring) Wrap(value uint32) uint32 {
return value & (PacketCapacity - 1)
}
// Aligns a packet size to PacketAlignment
func PacketAlign(size uint32) uint32 {
return (size + (PacketAlignment - 1)) &^ (PacketAlignment - 1)
}
func NewRingDescriptor() (descriptor *RingDescriptor, err error) {
descriptor = new(RingDescriptor)
allocatedRegion, err := windows.VirtualAlloc(0, unsafe.Sizeof(Ring{})*2, windows.MEM_COMMIT|windows.MEM_RESERVE, windows.PAGE_READWRITE)
if err != nil {
return
}
defer func() {
if err != nil {
descriptor.free()
descriptor = nil
}
}()
descriptor.Send.Size = uint32(unsafe.Sizeof(Ring{}))
descriptor.Send.Ring = (*Ring)(unsafe.Pointer(allocatedRegion))
descriptor.Send.TailMoved, err = windows.CreateEvent(nil, 0, 0, nil)
if err != nil {
return
}
descriptor.Receive.Size = uint32(unsafe.Sizeof(Ring{}))
descriptor.Receive.Ring = (*Ring)(unsafe.Pointer(allocatedRegion + unsafe.Sizeof(Ring{})))
descriptor.Receive.TailMoved, err = windows.CreateEvent(nil, 0, 0, nil)
if err != nil {
windows.CloseHandle(descriptor.Send.TailMoved)
return
}
runtime.SetFinalizer(descriptor, func(d *RingDescriptor) { d.free() })
return
}
func (descriptor *RingDescriptor) free() {
if descriptor.Send.Ring != nil {
windows.VirtualFree(uintptr(unsafe.Pointer(descriptor.Send.Ring)), 0, windows.MEM_RELEASE)
descriptor.Send.Ring = nil
descriptor.Receive.Ring = nil
}
}
func (descriptor *RingDescriptor) Close() {
if descriptor.Send.TailMoved != 0 {
windows.CloseHandle(descriptor.Send.TailMoved)
descriptor.Send.TailMoved = 0
}
if descriptor.Send.TailMoved != 0 {
windows.CloseHandle(descriptor.Receive.TailMoved)
descriptor.Receive.TailMoved = 0
}
}
func (wintun *Adapter) Register(descriptor *RingDescriptor) (windows.Handle, error) {
handle, err := wintun.OpenAdapterDeviceObject()
if err != nil {
return 0, err
}
var bytesReturned uint32
err = windows.DeviceIoControl(handle, ioctlRegisterRings, (*byte)(unsafe.Pointer(descriptor)), uint32(unsafe.Sizeof(*descriptor)), nil, 0, &bytesReturned, nil)
if err != nil {
return 0, err
}
return handle, nil
}

View file

@ -0,0 +1,108 @@
/* SPDX-License-Identifier: MIT
*
* Copyright (C) 2020 WireGuard LLC. All Rights Reserved.
*/
package wintun
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
type Session struct {
handle uintptr
}
const (
PacketSizeMax = 0xffff // Maximum packet size
RingCapacityMin = 0x20000 // Minimum ring capacity (128 kiB)
RingCapacityMax = 0x4000000 // Maximum ring capacity (64 MiB)
)
// Packet with data
type Packet struct {
Next *Packet // Pointer to next packet in queue
Size uint32 // Size of packet (max WINTUN_MAX_IP_PACKET_SIZE)
Data *[PacketSizeMax]byte // Pointer to layer 3 IPv4 or IPv6 packet
}
var (
procWintunAllocateSendPacket = modwintun.NewProc("WintunAllocateSendPacket").Addr()
procWintunEndSession = modwintun.NewProc("WintunEndSession")
procWintunGetReadWaitEvent = modwintun.NewProc("WintunGetReadWaitEvent")
procWintunReceivePacket = modwintun.NewProc("WintunReceivePacket").Addr()
procWintunReleaseReceivePacket = modwintun.NewProc("WintunReleaseReceivePacket").Addr()
procWintunSendPacket = modwintun.NewProc("WintunSendPacket").Addr()
procWintunStartSession = modwintun.NewProc("WintunStartSession")
)
func (wintun *Adapter) StartSession(capacity uint32) (session Session, err error) {
r0, _, e1 := syscall.Syscall(procWintunStartSession.Addr(), 2, uintptr(wintun.handle), uintptr(capacity), 0)
if r0 == 0 {
err = e1
} else {
session = Session{r0}
}
return
}
func (session Session) End() {
syscall.Syscall(procWintunEndSession.Addr(), 1, session.handle, 0, 0)
session.handle = 0
}
func (session Session) ReadWaitEvent() (handle windows.Handle) {
r0, _, _ := syscall.Syscall(procWintunGetReadWaitEvent.Addr(), 1, session.handle, 0, 0)
handle = windows.Handle(r0)
return
}
func (session Session) ReceivePacket() (packet []byte, err error) {
var packetSize uint32
r0, _, e1 := syscall.Syscall(procWintunReceivePacket, 2, session.handle, uintptr(unsafe.Pointer(&packetSize)), 0)
if r0 == 0 {
err = e1
return
}
unsafeSlice(unsafe.Pointer(&packet), unsafe.Pointer(r0), int(packetSize))
return
}
func (session Session) ReleaseReceivePacket(packet []byte) {
syscall.Syscall(procWintunReleaseReceivePacket, 2, session.handle, uintptr(unsafe.Pointer(&packet[0])), 0)
}
func (session Session) AllocateSendPacket(packetSize int) (packet []byte, err error) {
r0, _, e1 := syscall.Syscall(procWintunAllocateSendPacket, 2, session.handle, uintptr(packetSize), 0)
if r0 == 0 {
err = e1
return
}
unsafeSlice(unsafe.Pointer(&packet), unsafe.Pointer(r0), int(packetSize))
return
}
func (session Session) SendPacket(packet []byte) {
syscall.Syscall(procWintunSendPacket, 2, session.handle, uintptr(unsafe.Pointer(&packet[0])), 0)
}
// unsafeSlice updates the slice slicePtr to be a slice
// referencing the provided data with its length & capacity set to
// lenCap.
//
// TODO: when Go 1.16 or Go 1.17 is the minimum supported version,
// update callers to use unsafe.Slice instead of this.
func unsafeSlice(slicePtr, data unsafe.Pointer, lenCap int) {
type sliceHeader struct {
Data unsafe.Pointer
Len int
Cap int
}
h := (*sliceHeader)(slicePtr)
h.Data = data
h.Len = lenCap
h.Cap = lenCap
}

View file

@ -45,7 +45,6 @@ var (
procWintunGetAdapterLUID = modwintun.NewProc("WintunGetAdapterLUID") procWintunGetAdapterLUID = modwintun.NewProc("WintunGetAdapterLUID")
procWintunGetAdapterName = modwintun.NewProc("WintunGetAdapterName") procWintunGetAdapterName = modwintun.NewProc("WintunGetAdapterName")
procWintunGetRunningDriverVersion = modwintun.NewProc("WintunGetRunningDriverVersion") procWintunGetRunningDriverVersion = modwintun.NewProc("WintunGetRunningDriverVersion")
procWintunOpenAdapterDeviceObject = modwintun.NewProc("WintunOpenAdapterDeviceObject")
procWintunSetAdapterName = modwintun.NewProc("WintunSetAdapterName") procWintunSetAdapterName = modwintun.NewProc("WintunSetAdapterName")
procWintunSetLogger = modwintun.NewProc("WintunSetLogger") procWintunSetLogger = modwintun.NewProc("WintunSetLogger")
) )
@ -210,16 +209,6 @@ func RunningVersion() (version uint32, err error) {
return return
} }
// handle returns a handle to the adapter device object. Release handle with windows.CloseHandle
func (wintun *Adapter) OpenAdapterDeviceObject() (handle windows.Handle, err error) {
r0, _, e1 := syscall.Syscall(procWintunOpenAdapterDeviceObject.Addr(), 1, uintptr(wintun.handle), 0, 0)
handle = windows.Handle(r0)
if handle == windows.InvalidHandle {
err = e1
}
return
}
// LUID returns the LUID of the adapter. // LUID returns the LUID of the adapter.
func (wintun *Adapter) LUID() (luid uint64) { func (wintun *Adapter) LUID() (luid uint64) {
syscall.Syscall(procWintunGetAdapterLUID.Addr(), 2, uintptr(wintun.handle), uintptr(unsafe.Pointer(&luid)), 0) syscall.Syscall(procWintunGetAdapterLUID.Addr(), 2, uintptr(wintun.handle), uintptr(unsafe.Pointer(&luid)), 0)