Move the tunnel receive loop into its own thread

Author: oskirby

src/platforms/macos/daemon/wgsessionmacos.cpp

@@ -73,12 +73,14 @@ WgSessionMacos::~WgSessionMacos() {
   if (m_netSocket >= 0) {
     close(m_netSocket);
   }
+  if (m_encryptWorker) {
+    m_encryptWorker->shutdown();
+    m_encryptWorker->wait();
+  }
 
   // Shut down the worker pools.
   m_decryptPool.clear();
-  m_encryptPool.clear();
   m_decryptPool.waitForDone();
-  m_encryptPool.waitForDone();
 }
 
 void WgSessionMacos::processResult(int op, const QByteArray& buf) const {
@@ -364,9 +366,11 @@ void WgSessionMacos::setTunSocket(qintptr sd) {
   fcntl(sd, F_SETFL, flags | O_NONBLOCK);
 
   m_tunSocket = sd;
-  auto notifier = new QSocketNotifier(sd, QSocketNotifier::Read, this);
-  connect(notifier, &QSocketNotifier::activated, this,
-          &WgSessionMacos::tunReadyRead);
+
+  // Start the encryption worker.
+  m_encryptWorker = new WgEncryptWorker(this, sd);
+  m_encryptWorker->setMtu(m_tunmtu);
+  m_encryptWorker->start();
 }
 
 void WgSessionMacos::setMtu(int mtu) {
@@ -413,47 +417,6 @@ void WgSessionMacos::netReadyRead() {
   }
 }
 
-void WgSessionMacos::tunReadyRead() {
-  // The tunnel socket is ready for reading.
-  quint32 header = 0;
-  QByteArray rxbuf;
-  rxbuf.resize(m_tunmtu + 16);
-
-  struct iovec iov[2];
-  iov[0].iov_base = &header;
-  iov[0].iov_len = sizeof(header);
-  iov[1].iov_base = (void*)rxbuf.data();
-  iov[1].iov_len = m_tunmtu;
-
-  while (true) {
-    // Try to read a packet from the tunnel.
-    int len = readv(m_tunSocket, iov, sizeof(iov) / sizeof(struct iovec));
-    if (len < 0) {
-      if (errno == EAGAIN) return;
-      logger.debug() << "Tunnel error:" << strerror(errno);
-      return;
-    }
-    int pktlen = len - sizeof(header);
-    if ((pktlen < 0) || (pktlen > m_tunmtu)) {
-      continue;
-    }
-
-    // I think there is a small bug in boringtun to do with message padding.
-    // The wireguard protocol states that the encapsulated packet must first be
-    // padded out to a multiple of 16 bytes in length, but boringtun does no
-    // such padding during encryption. So let's do it manually ourself.
-    int tail = pktlen % 16;
-    if (tail) {
-      int padsz = 16 - tail;
-      memset(rxbuf.data() + pktlen, 0, padsz);
-      pktlen += padsz;
-    }
-
-    auto worker = new WgEncryptWorker(this, rxbuf.first(pktlen));
-    m_encryptPool.start(worker);
-  }
-}
-
 void WgSessionMacos::tunWrite(qintptr fd, const QByteArray& packet, const QByteArray& append) const {
   //logger.debug() << name() << "decrypt:" << QString(packet.toBase64());
   quint32 family = ((packet.at(0) >> 4) == 4) ? AF_INET : AF_INET6;

src/platforms/macos/daemon/wgsessionmacos.h

@@ -18,6 +18,9 @@ struct wireguard_tunnel;
 struct ipv4header;
 struct sockaddr;
 
+class WgEncryptWorker;
+class WgDecryptWorker;
+
 class WgSessionMacos final : public QObject {
   Q_OBJECT
 
@@ -41,7 +44,6 @@ class WgSessionMacos final : public QObject {
 
  protected slots:
   void netReadyRead();
-  void tunReadyRead();
 
  private:
   void timeout();
@@ -92,7 +94,7 @@ class WgSessionMacos final : public QObject {
 
  protected:
   struct wireguard_tunnel* m_tunnel = nullptr;
-  QThreadPool m_encryptPool;
+  WgEncryptWorker* m_encryptWorker = nullptr;
   QThreadPool m_decryptPool;
 
   friend class WgEncryptWorker;

src/platforms/macos/daemon/wgsessionworker.cpp

@@ -4,21 +4,95 @@
 
 #include "wgsessionworker.h"
 
+#include <fcntl.h>
+#include <netinet/ip6.h>
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "leakdetector.h"
+#include "logger.h"
 #include "wgsessionmacos.h"
 
 extern "C" {
 #include "wireguard_ffi.h"
 }
 
+namespace {
+Logger logger("WgSessionWorker");
+};  // namespace
+
+WgEncryptWorker::WgEncryptWorker(WgSessionMacos* session, qintptr socket)
+    : QThread(session), m_session(session) {
+  MZ_COUNT_CTOR(WgSessionMacos);
+  m_socket = dup(socket);
+  m_mtu = IPV6_MMTU;
+
+  int flags = fcntl(m_socket, F_GETFL, 0);
+  fcntl(m_socket, F_SETFL, flags & ~O_NONBLOCK);
+}
+
+WgEncryptWorker::~WgEncryptWorker() {
+  MZ_COUNT_DTOR(WgSessionMacos);
+  if (m_socket >= 0) {
+    close(m_socket);
+  }
+}
+
 void WgEncryptWorker::run() {
-  QByteArray output;
-  output.resize(m_packet.size() + WgSessionMacos::WG_PACKET_OVERHEAD);
+  quint32 header = 0;
+  QByteArray packet;
+  QByteArray encrypt;
 
-  const uint8_t* ptr = reinterpret_cast<const uint8_t*>(m_packet.constData());
-  uint8_t* outptr = reinterpret_cast<uint8_t*>(output.data());
-  auto result = wireguard_write(m_session->m_tunnel, ptr, m_packet.size(),
-                                outptr, output.size());
-  m_session->processResult(result.op, output.first(result.size));
+  while (!isInterruptionRequested()) {
+    // Resize in case the MTU changed.
+    int mtu = m_mtu.loadAcquire();
+    packet.resize(mtu + 16);
+    encrypt.resize(mtu + WgSessionMacos::WG_PACKET_OVERHEAD);
+
+    struct iovec iov[2];
+    iov[0].iov_base = &header;
+    iov[0].iov_len = sizeof(header);
+    iov[1].iov_len = mtu;
+    iov[1].iov_base = (void*)packet.data();
+
+    // Try to read a packet from the tunnel.
+    int len = readv(m_socket, iov, sizeof(iov) / sizeof(struct iovec));
+    if (len < 0) {
+      if (errno == EAGAIN) continue;
+      if (errno == EINTR) continue;
+      logger.debug() << "Tunnel error:" << strerror(errno);
+      return;
+    }
+    int pktlen = len - sizeof(header);
+    if ((pktlen < 0) || (pktlen > mtu)) {
+      continue;
+    }
+
+    // I think there is a small bug in boringtun to do with message padding.
+    // The wireguard protocol states that the encapsulated packet must first be
+    // padded out to a multiple of 16 bytes in length, but boringtun does no
+    // such padding during encryption. So let's do it manually ourself.
+    int tail = pktlen % 16;
+    if (tail) {
+      int padsz = 16 - tail;
+      memset(packet.data() + pktlen, 0, padsz);
+      pktlen += padsz;
+    }
+
+    const uint8_t* ptr = reinterpret_cast<const uint8_t*>(packet.constData());
+    uint8_t* enc = reinterpret_cast<uint8_t*>(encrypt.data());
+    auto res = wireguard_write(m_session->m_tunnel, ptr, pktlen, enc,
+                                  encrypt.size());
+
+    m_session->processResult(res.op, encrypt.first(res.size));
+  }
+}
+
+void WgEncryptWorker::shutdown() {
+  requestInterruption();
+  ::shutdown(m_socket, SHUT_RD);
 }
 
 void WgDecryptWorker::run() {

src/platforms/macos/daemon/wgsessionworker.h

@@ -7,19 +7,24 @@
 
 #include <QByteArray>
 #include <QRunnable>
+#include <QThread>
 
 class WgSessionMacos;
 
-class WgEncryptWorker final : public QRunnable {
+class WgEncryptWorker final : public QThread {
  public:
-  WgEncryptWorker(WgSessionMacos* session, const QByteArray& packet)
-    : QRunnable(), m_session(session), m_packet(packet) {}
+  WgEncryptWorker(WgSessionMacos* session, qintptr socket);
+  ~WgEncryptWorker();
+
+  void setMtu(int mtu) { m_mtu = mtu; }
+  void shutdown();
 
  protected:
   void run() override;
 
   WgSessionMacos* m_session;
-  QByteArray m_packet;
+  QAtomicInt m_mtu;
+  int m_socket;
 };
 
 class WgDecryptWorker final : public QRunnable {