diff --git a/device/device.go b/device/device.go
index 4b131a2..3b8770b 100644
--- a/device/device.go
+++ b/device/device.go
@@ -55,9 +55,13 @@ type Device struct {
 		publicKey  NoisePublicKey
 	}
 
+	rate struct {
+		underLoadUntil int64
+		limiter        ratelimiter.Ratelimiter
+	}
+
 	peers struct {
-		empty        AtomicBool // empty reports whether len(keyMap) == 0
-		sync.RWMutex            // protects keyMap
+		sync.RWMutex // protects keyMap
 		keyMap       map[NoisePublicKey]*Peer
 	}
 
@@ -65,11 +69,6 @@ type Device struct {
 	indexTable    IndexTable
 	cookieChecker CookieChecker
 
-	rate struct {
-		underLoadUntil int64
-		limiter        ratelimiter.Ratelimiter
-	}
-
 	pool struct {
 		messageBuffers   *WaitPool
 		inboundElements  *WaitPool
@@ -135,7 +134,6 @@ func removePeerLocked(device *Device, peer *Peer, key NoisePublicKey) {
 
 	// remove from peer map
 	delete(device.peers.keyMap, key)
-	device.peers.empty.Set(len(device.peers.keyMap) == 0)
 }
 
 // changeState attempts to change the device state to match want.
diff --git a/device/peer.go b/device/peer.go
index 332f7bd..a063b91 100644
--- a/device/peer.go
+++ b/device/peer.go
@@ -111,7 +111,6 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 
 	// add
 	device.peers.keyMap[pk] = peer
-	device.peers.empty.Set(false)
 
 	// start peer
 	peer.timersInit()
diff --git a/device/timers.go b/device/timers.go
index fa44874..ee191e5 100644
--- a/device/timers.go
+++ b/device/timers.go
@@ -71,7 +71,7 @@ func (timer *Timer) IsPending() bool {
 }
 
 func (peer *Peer) timersActive() bool {
-	return peer.isRunning.Get() && peer.device != nil && peer.device.isUp() && !peer.device.peers.empty.Get()
+	return peer.isRunning.Get() && peer.device != nil && peer.device.isUp()
 }
 
 func expiredRetransmitHandshake(peer *Peer) {