From c7b00f5cb78ef9928c5902b074939abf32bdd6e8 Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Fri, 24 Oct 2025 12:38:22 -0400 Subject: [PATCH 1/7] Configure queue id for XDP Device (socket) --- Pcap++/header/XdpDevice.h | 20 +++++++++++++++++++- Pcap++/src/XdpDevice.cpp | 12 ++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index effb4b07a7..7b86501e03 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -11,6 +11,8 @@ /// @ namespace pcpp { +#define XDP_MAX_RXTX_QUEUES 16 + /// @class XdpDevice /// A class wrapping the main functionality of using AF_XDP (XSK) sockets /// which are optimized for high performance packet processing. @@ -76,6 +78,10 @@ namespace pcpp /// The max number of packets to be received or sent in one batch uint16_t rxTxBatchSize; + /// The queue identifier for the underlying socket. This value should be less than the number + /// of hardware queues supported by the device + uint16_t queueId; + /// A c'tor for this struct. Each parameter has a default value described below. /// @param[in] attachMode AF_XDP operation mode. The default value is auto mode /// @param[in] umemNumFrames Number of UMEM frames to allocate. The default value is 4096 @@ -87,10 +93,11 @@ namespace pcpp /// @param[in] txSize The size of the TX ring used by the AF_XDP socket. The default value is 2048 /// @param[in] rxTxBatchSize The max number of packets to be received or sent in one batch. The default /// value is 64 + /// @param[in] queueId The hardware queue id of the underlying socket. The default value is 0 explicit XdpDeviceConfiguration(AttachMode attachMode = AutoMode, uint16_t umemNumFrames = 0, uint16_t umemFrameSize = 0, uint32_t fillRingSize = 0, uint32_t completionRingSize = 0, uint32_t rxSize = 0, uint32_t txSize = 0, - uint16_t rxTxBatchSize = 0) + uint16_t rxTxBatchSize = 0, uint32_t queueId = 0) { this->attachMode = attachMode; this->umemNumFrames = umemNumFrames; @@ -100,6 +107,7 @@ namespace pcpp this->rxSize = rxSize; this->txSize = txSize; this->rxTxBatchSize = rxTxBatchSize; + this->queueId = queueId; } }; @@ -238,6 +246,14 @@ namespace pcpp /// @return Current device statistics XdpDeviceStats getStatistics(); + /// @return Return queue identifier for underlying socket + uint32_t getQueueId() + { + if(m_Config) return m_Config->queueId; + + return 0; + } + private: class XdpUmem { @@ -312,5 +328,7 @@ namespace pcpp bool initUmem(); bool populateConfigDefaults(XdpDeviceConfiguration& config) const; bool getSocketStats(); + + uint32_t getNumQueues(const std::string& iface) const; }; } // namespace pcpp diff --git a/Pcap++/src/XdpDevice.cpp b/Pcap++/src/XdpDevice.cpp index 1f17cc4eea..64eabb3d45 100644 --- a/Pcap++/src/XdpDevice.cpp +++ b/Pcap++/src/XdpDevice.cpp @@ -419,7 +419,7 @@ namespace pcpp xskConfig.xdp_flags = XDP_FLAGS_DRV_MODE; } - int ret = xsk_socket__create(&socketInfo->xsk, m_InterfaceName.c_str(), 0, umemInfo->umem, &socketInfo->rx, + int ret = xsk_socket__create(&socketInfo->xsk, m_InterfaceName.c_str(), m_Config->queueId, umemInfo->umem, &socketInfo->rx, &socketInfo->tx, &xskConfig); if (ret) { @@ -429,6 +429,7 @@ namespace pcpp } m_SocketInfo = socketInfo; + return true; } @@ -449,6 +450,7 @@ namespace pcpp uint32_t rxSize = config.rxSize ? config.rxSize : XSK_RING_CONS__DEFAULT_NUM_DESCS; uint32_t txSize = config.txSize ? config.txSize : XSK_RING_PROD__DEFAULT_NUM_DESCS; uint32_t batchSize = config.rxTxBatchSize ? config.rxTxBatchSize : DEFAULT_BATCH_SIZE; + uint32_t qId = config.queueId; // default is zero if (frameSize != getpagesize()) { @@ -499,6 +501,12 @@ namespace pcpp return false; } + if (qId >= XDP_MAX_RXTX_QUEUES) + { + PCPP_LOG_ERROR("Queue Id " << qId << " must be lower than the maximum number of hardware queues"); + return(false); + } + config.umemNumFrames = numFrames; config.umemFrameSize = frameSize; config.fillRingSize = fillRingSize; @@ -506,6 +514,7 @@ namespace pcpp config.rxSize = rxSize; config.txSize = txSize; config.rxTxBatchSize = batchSize; + config.queueId = qId; return true; } @@ -635,5 +644,4 @@ namespace pcpp return m_Stats; } - } // namespace pcpp From fe735fe070ec39d7c6a6fa044fe0edd5be3922f9 Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 27 Oct 2025 15:43:09 -0400 Subject: [PATCH 2/7] Utility to find HW queues for device --- Pcap++/header/XdpDevice.h | 13 +++++++++--- Pcap++/src/XdpDevice.cpp | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index 7b86501e03..cfa8c96ced 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -249,11 +249,20 @@ namespace pcpp /// @return Return queue identifier for underlying socket uint32_t getQueueId() { - if(m_Config) return m_Config->queueId; + if(m_Config) + { + return m_Config->queueId; + + } return 0; } + /// Get number of RX or TX hardware queues for device + /// @param[in] tx If true, return TX queues, otherwise RX. Default is false + /// @return The number of hardware queues associated with the device. + static uint32_t getNumQueues(const std::string& iface, bool tx = false); + private: class XdpUmem { @@ -328,7 +337,5 @@ namespace pcpp bool initUmem(); bool populateConfigDefaults(XdpDeviceConfiguration& config) const; bool getSocketStats(); - - uint32_t getNumQueues(const std::string& iface) const; }; } // namespace pcpp diff --git a/Pcap++/src/XdpDevice.cpp b/Pcap++/src/XdpDevice.cpp index 64eabb3d45..48287b72ac 100644 --- a/Pcap++/src/XdpDevice.cpp +++ b/Pcap++/src/XdpDevice.cpp @@ -10,9 +10,11 @@ #include #include #include +#include #include #include #include +#include #include namespace pcpp @@ -507,6 +509,13 @@ namespace pcpp return(false); } + unsigned int nhwqueues = getNumQueues(m_InterfaceName); + if (qId >= nhwqueues) + { + PCPP_LOG_ERROR("Queue Id (" << qId << ") must be less than the number hardware queues (" << nhwqueues << ") of device"); + return false; + } + config.umemNumFrames = numFrames; config.umemFrameSize = frameSize; config.fillRingSize = fillRingSize; @@ -644,4 +653,37 @@ namespace pcpp return m_Stats; } + + uint32_t XdpDevice::getNumQueues(const std::string& iface, bool tx) + { + // returns number of hardware queues associated with the device + uint32_t rxtxqueues = 0; + std::string prefix = tx ? "tx-" : "rx-"; + std::string path = "/sys/class/net/" + iface + "/queues/"; + DIR *dir = opendir(path.c_str()); + + if(dir) + { + std::regex rxtx_regex("^" + prefix + "[0-9]+$"); + + struct dirent* entry; + while((entry = readdir(dir)) != nullptr) + { + if(std::regex_match(entry->d_name, rxtx_regex)) + { + rxtxqueues++; + } + } + + closedir(dir); + } + + else + { + PCPP_LOG_ERROR("Error getting number of hardware queues from " << iface); + } + + return rxtxqueues; + } + } // namespace pcpp From db23ec391459b3cd5cc63e1db3f6462ad36cae43 Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 27 Oct 2025 15:53:51 -0400 Subject: [PATCH 3/7] Remove old check for queueid --- Pcap++/src/XdpDevice.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Pcap++/src/XdpDevice.cpp b/Pcap++/src/XdpDevice.cpp index 48287b72ac..18e9ab6219 100644 --- a/Pcap++/src/XdpDevice.cpp +++ b/Pcap++/src/XdpDevice.cpp @@ -503,12 +503,6 @@ namespace pcpp return false; } - if (qId >= XDP_MAX_RXTX_QUEUES) - { - PCPP_LOG_ERROR("Queue Id " << qId << " must be lower than the maximum number of hardware queues"); - return(false); - } - unsigned int nhwqueues = getNumQueues(m_InterfaceName); if (qId >= nhwqueues) { From a0906cf8c1c52fc4d18d31907adeac57668b228e Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 27 Oct 2025 16:08:46 -0400 Subject: [PATCH 4/7] Fix doxygen error --- Pcap++/header/XdpDevice.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index cfa8c96ced..b9539ee027 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -259,6 +259,7 @@ namespace pcpp } /// Get number of RX or TX hardware queues for device + /// @param[in] interfaceName The interface name to use to detect hardware queues /// @param[in] tx If true, return TX queues, otherwise RX. Default is false /// @return The number of hardware queues associated with the device. static uint32_t getNumQueues(const std::string& iface, bool tx = false); From e6a1e9b190688a3584a8a317eae46e4968f28bc1 Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 27 Oct 2025 16:11:16 -0400 Subject: [PATCH 5/7] Fix doxygen error --- Pcap++/header/XdpDevice.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index b9539ee027..c7dafaf511 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -262,7 +262,7 @@ namespace pcpp /// @param[in] interfaceName The interface name to use to detect hardware queues /// @param[in] tx If true, return TX queues, otherwise RX. Default is false /// @return The number of hardware queues associated with the device. - static uint32_t getNumQueues(const std::string& iface, bool tx = false); + static uint32_t getNumQueues(const std::string& interfaceName, bool tx = false); private: class XdpUmem From a8d97bfcaf408114261b8e351f11a930caaa98aa Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 10 Nov 2025 11:39:05 -0500 Subject: [PATCH 6/7] Formatting and bug --- Pcap++/header/XdpDevice.h | 4 ++-- Pcap++/src/XdpDevice.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index c7dafaf511..338139fe2c 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -249,7 +249,7 @@ namespace pcpp /// @return Return queue identifier for underlying socket uint32_t getQueueId() { - if(m_Config) + if(m_Config) { return m_Config->queueId; @@ -261,7 +261,7 @@ namespace pcpp /// Get number of RX or TX hardware queues for device /// @param[in] interfaceName The interface name to use to detect hardware queues /// @param[in] tx If true, return TX queues, otherwise RX. Default is false - /// @return The number of hardware queues associated with the device. + /// @return The number of hardware queues associated with the device. static uint32_t getNumQueues(const std::string& interfaceName, bool tx = false); private: diff --git a/Pcap++/src/XdpDevice.cpp b/Pcap++/src/XdpDevice.cpp index 18e9ab6219..42428c3b84 100644 --- a/Pcap++/src/XdpDevice.cpp +++ b/Pcap++/src/XdpDevice.cpp @@ -405,8 +405,8 @@ namespace pcpp auto umemInfo = static_cast(m_Umem->getInfo()); struct xsk_socket_config xskConfig; - xskConfig.rx_size = m_Config->txSize; - xskConfig.tx_size = m_Config->rxSize; + xskConfig.rx_size = m_Config->rxSize; + xskConfig.tx_size = m_Config->txSize; xskConfig.libbpf_flags = 0; xskConfig.xdp_flags = 0; xskConfig.bind_flags = 0; @@ -431,7 +431,7 @@ namespace pcpp } m_SocketInfo = socketInfo; - + return true; } From cb5e47edaaa3adf7206e7740db0f9b25b877b1af Mon Sep 17 00:00:00 2001 From: "james.metzger" Date: Mon, 10 Nov 2025 11:46:19 -0500 Subject: [PATCH 7/7] Minor adjustments recommended --- Pcap++/header/XdpDevice.h | 4 ++-- Pcap++/src/XdpDevice.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Pcap++/header/XdpDevice.h b/Pcap++/header/XdpDevice.h index 338139fe2c..866aa93255 100644 --- a/Pcap++/header/XdpDevice.h +++ b/Pcap++/header/XdpDevice.h @@ -247,7 +247,7 @@ namespace pcpp XdpDeviceStats getStatistics(); /// @return Return queue identifier for underlying socket - uint32_t getQueueId() + uint32_t getQueueId() const { if(m_Config) { @@ -262,7 +262,7 @@ namespace pcpp /// @param[in] interfaceName The interface name to use to detect hardware queues /// @param[in] tx If true, return TX queues, otherwise RX. Default is false /// @return The number of hardware queues associated with the device. - static uint32_t getNumQueues(const std::string& interfaceName, bool tx = false); + static uint32_t numQueues(const std::string& interfaceName, bool tx = false); private: class XdpUmem diff --git a/Pcap++/src/XdpDevice.cpp b/Pcap++/src/XdpDevice.cpp index 42428c3b84..2ec047b7fa 100644 --- a/Pcap++/src/XdpDevice.cpp +++ b/Pcap++/src/XdpDevice.cpp @@ -503,7 +503,7 @@ namespace pcpp return false; } - unsigned int nhwqueues = getNumQueues(m_InterfaceName); + unsigned int nhwqueues = numQueues(m_InterfaceName); if (qId >= nhwqueues) { PCPP_LOG_ERROR("Queue Id (" << qId << ") must be less than the number hardware queues (" << nhwqueues << ") of device"); @@ -648,7 +648,7 @@ namespace pcpp return m_Stats; } - uint32_t XdpDevice::getNumQueues(const std::string& iface, bool tx) + uint32_t XdpDevice::numQueues(const std::string& iface, bool tx) { // returns number of hardware queues associated with the device uint32_t rxtxqueues = 0;