Skip to content

Commit 62cf608

Browse files
laanwjlaanwj
committed
Merge #14336: net: implement poll
4927bf2 Increase maxconnections limit when using poll. (Patrick Strateman) 11cc491 Implement poll() on systems which support it properly. (Patrick Strateman) 28211a4 Move SocketEvents logic to private method. (Patrick Strateman) 7e403c0 Move GenerateSelectSet logic to private method. (Patrick Strateman) 1e6afd0 Introduce and use constant SELECT_TIMEOUT_MILLISECONDS. (Patrick Strateman) Pull request description: Implement poll() on systems which support it properly. This eliminates the restriction on maximum socket descriptor number. Tree-SHA512: b945cd9294afdafcce96d547f67679d5cdd684cf257904a239cd1248de3b5e093b8d6d28d8d1b7cc923dc0b2b5723faef9bc9bf118a9ce1bdcf357c2323f5573
2 parents 3ec4cc0 + 4927bf2 commit 62cf608

File tree

5 files changed

+172
-43
lines changed

5 files changed

+172
-43
lines changed

‎src/compat.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,15 @@ typedef void* sockopt_arg_type;
9292
typedef char* sockopt_arg_type;
9393
#endif
9494

95+
// Note these both should work with the current usage of poll, but best to be safe
96+
// WIN32 poll is broken https://daniel.haxx.se/blog/2012/10/10/wsapoll-is-broken/
97+
// __APPLE__ poll is broke https://github.com/bitcoin/bitcoin/pull/14336#issuecomment-437384408
98+
#if defined(__linux__)
99+
#define USE_POLL
100+
#endif
101+
95102
bool static inline IsSelectableSocket(const SOCKET& s) {
96-
#ifdef WIN32
103+
#if defined(USE_POLL) || defined(WIN32)
97104
return true;
98105
#else
99106
return (s < FD_SETSIZE);

‎src/init.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -953,8 +953,13 @@ bool AppInitParameterInteraction()
953953

954954
// Trim requested connection counts, to fit into system limitations
955955
// <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695
956-
nMaxConnections = std::max(std::min<int>(nMaxConnections, FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0);
957956
nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS);
957+
#ifdef USE_POLL
958+
int fd_max = nFD;
959+
#else
960+
int fd_max = FD_SETSIZE;
961+
#endif
962+
nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0);
958963
if (nFD < MIN_CORE_FILEDESCRIPTORS)
959964
return InitError(_("Not enough file descriptors available."));
960965
nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections);

‎src/net.cpp

Lines changed: 136 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,18 @@
2626
#include <fcntl.h>
2727
#endif
2828

29+
#ifdef USE_POLL
30+
#include <poll.h>
31+
#endif
32+
2933
#ifdef USE_UPNP
3034
#include <miniupnpc/miniupnpc.h>
3135
#include <miniupnpc/miniwget.h>
3236
#include <miniupnpc/upnpcommands.h>
3337
#include <miniupnpc/upnperrors.h>
3438
#endif
3539

40+
#include <unordered_map>
3641

3742
#include <math.h>
3843

@@ -71,6 +76,10 @@ enum BindFlags {
7176
BF_WHITELIST = (1U << 2),
7277
};
7378

79+
// The set of sockets cannot be modified while waiting
80+
// The sleep time needs to be small to avoid new sockets stalling
81+
static const uint64_t SELECT_TIMEOUT_MILLISECONDS = 50;
82+
7483
const static std::string NET_MESSAGE_COMMAND_OTHER = "*other*";
7584

7685
static const uint64_t RANDOMIZER_ID_NETGROUP = 0x6c0edd8036ef4036ULL; // SHA256("netgroup")[0:8]
@@ -1258,28 +1267,10 @@ void CConnman::InactivityCheck(CNode *pnode)
12581267
}
12591268
}
12601269

1261-
void CConnman::SocketHandler()
1270+
bool CConnman::GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set)
12621271
{
1263-
//
1264-
// Find which sockets have data to receive
1265-
//
1266-
struct timeval timeout;
1267-
timeout.tv_sec = 0;
1268-
timeout.tv_usec = 50000; // frequency to poll pnode->vSend
1269-
1270-
fd_set fdsetRecv;
1271-
fd_set fdsetSend;
1272-
fd_set fdsetError;
1273-
FD_ZERO(&fdsetRecv);
1274-
FD_ZERO(&fdsetSend);
1275-
FD_ZERO(&fdsetError);
1276-
SOCKET hSocketMax = 0;
1277-
bool have_fds = false;
1278-
12791272
for (const ListenSocket& hListenSocket : vhListenSocket) {
1280-
FD_SET(hListenSocket.socket, &fdsetRecv);
1281-
hSocketMax = std::max(hSocketMax, hListenSocket.socket);
1282-
have_fds = true;
1273+
recv_set.insert(hListenSocket.socket);
12831274
}
12841275

12851276
{
@@ -1308,46 +1299,151 @@ void CConnman::SocketHandler()
13081299
if (pnode->hSocket == INVALID_SOCKET)
13091300
continue;
13101301

1311-
FD_SET(pnode->hSocket, &fdsetError);
1312-
hSocketMax = std::max(hSocketMax, pnode->hSocket);
1313-
have_fds = true;
1314-
1302+
error_set.insert(pnode->hSocket);
13151303
if (select_send) {
1316-
FD_SET(pnode->hSocket, &fdsetSend);
1304+
send_set.insert(pnode->hSocket);
13171305
continue;
13181306
}
13191307
if (select_recv) {
1320-
FD_SET(pnode->hSocket, &fdsetRecv);
1308+
recv_set.insert(pnode->hSocket);
13211309
}
13221310
}
13231311
}
13241312

1325-
int nSelect = select(have_fds ? hSocketMax + 1 : 0,
1326-
&fdsetRecv, &fdsetSend, &fdsetError, &timeout);
1313+
return !recv_set.empty() || !send_set.empty() || !error_set.empty();
1314+
}
1315+
1316+
#ifdef USE_POLL
1317+
void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set)
1318+
{
1319+
std::set<SOCKET> recv_select_set, send_select_set, error_select_set;
1320+
if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) {
1321+
interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS));
1322+
return;
1323+
}
1324+
1325+
std::unordered_map<SOCKET, struct pollfd> pollfds;
1326+
for (SOCKET socket_id : recv_select_set) {
1327+
pollfds[socket_id].fd = socket_id;
1328+
pollfds[socket_id].events |= POLLIN;
1329+
}
1330+
1331+
for (SOCKET socket_id : send_select_set) {
1332+
pollfds[socket_id].fd = socket_id;
1333+
pollfds[socket_id].events |= POLLOUT;
1334+
}
1335+
1336+
for (SOCKET socket_id : error_select_set) {
1337+
pollfds[socket_id].fd = socket_id;
1338+
// These flags are ignored, but we set them for clarity
1339+
pollfds[socket_id].events |= POLLERR|POLLHUP;
1340+
}
1341+
1342+
std::vector<struct pollfd> vpollfds;
1343+
vpollfds.reserve(pollfds.size());
1344+
for (auto it : pollfds) {
1345+
vpollfds.push_back(std::move(it.second));
1346+
}
1347+
1348+
if (poll(vpollfds.data(), vpollfds.size(), SELECT_TIMEOUT_MILLISECONDS) < 0) return;
1349+
1350+
if (interruptNet) return;
1351+
1352+
for (struct pollfd pollfd_entry : vpollfds) {
1353+
if (pollfd_entry.revents & POLLIN) recv_set.insert(pollfd_entry.fd);
1354+
if (pollfd_entry.revents & POLLOUT) send_set.insert(pollfd_entry.fd);
1355+
if (pollfd_entry.revents & (POLLERR|POLLHUP)) error_set.insert(pollfd_entry.fd);
1356+
}
1357+
}
1358+
#else
1359+
void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set)
1360+
{
1361+
std::set<SOCKET> recv_select_set, send_select_set, error_select_set;
1362+
if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) {
1363+
interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS));
1364+
return;
1365+
}
1366+
1367+
//
1368+
// Find which sockets have data to receive
1369+
//
1370+
struct timeval timeout;
1371+
timeout.tv_sec = 0;
1372+
timeout.tv_usec = SELECT_TIMEOUT_MILLISECONDS * 1000; // frequency to poll pnode->vSend
1373+
1374+
fd_set fdsetRecv;
1375+
fd_set fdsetSend;
1376+
fd_set fdsetError;
1377+
FD_ZERO(&fdsetRecv);
1378+
FD_ZERO(&fdsetSend);
1379+
FD_ZERO(&fdsetError);
1380+
SOCKET hSocketMax = 0;
1381+
1382+
for (SOCKET hSocket : recv_select_set) {
1383+
FD_SET(hSocket, &fdsetRecv);
1384+
hSocketMax = std::max(hSocketMax, hSocket);
1385+
}
1386+
1387+
for (SOCKET hSocket : send_select_set) {
1388+
FD_SET(hSocket, &fdsetSend);
1389+
hSocketMax = std::max(hSocketMax, hSocket);
1390+
}
1391+
1392+
for (SOCKET hSocket : error_select_set) {
1393+
FD_SET(hSocket, &fdsetError);
1394+
hSocketMax = std::max(hSocketMax, hSocket);
1395+
}
1396+
1397+
int nSelect = select(hSocketMax + 1, &fdsetRecv, &fdsetSend, &fdsetError, &timeout);
1398+
13271399
if (interruptNet)
13281400
return;
13291401

13301402
if (nSelect == SOCKET_ERROR)
13311403
{
1332-
if (have_fds)
1333-
{
1334-
int nErr = WSAGetLastError();
1335-
LogPrintf("socket select error %s\n", NetworkErrorString(nErr));
1336-
for (unsigned int i = 0; i <= hSocketMax; i++)
1337-
FD_SET(i, &fdsetRecv);
1338-
}
1404+
int nErr = WSAGetLastError();
1405+
LogPrintf("socket select error %s\n", NetworkErrorString(nErr));
1406+
for (unsigned int i = 0; i <= hSocketMax; i++)
1407+
FD_SET(i, &fdsetRecv);
13391408
FD_ZERO(&fdsetSend);
13401409
FD_ZERO(&fdsetError);
1341-
if (!interruptNet.sleep_for(std::chrono::milliseconds(timeout.tv_usec/1000)))
1410+
if (!interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS)))
13421411
return;
13431412
}
13441413

1414+
for (SOCKET hSocket : recv_select_set) {
1415+
if (FD_ISSET(hSocket, &fdsetRecv)) {
1416+
recv_set.insert(hSocket);
1417+
}
1418+
}
1419+
1420+
for (SOCKET hSocket : send_select_set) {
1421+
if (FD_ISSET(hSocket, &fdsetSend)) {
1422+
send_set.insert(hSocket);
1423+
}
1424+
}
1425+
1426+
for (SOCKET hSocket : error_select_set) {
1427+
if (FD_ISSET(hSocket, &fdsetError)) {
1428+
error_set.insert(hSocket);
1429+
}
1430+
}
1431+
}
1432+
#endif
1433+
1434+
void CConnman::SocketHandler()
1435+
{
1436+
std::set<SOCKET> recv_set, send_set, error_set;
1437+
SocketEvents(recv_set, send_set, error_set);
1438+
1439+
if (interruptNet) return;
1440+
13451441
//
13461442
// Accept new connections
13471443
//
13481444
for (const ListenSocket& hListenSocket : vhListenSocket)
13491445
{
1350-
if (hListenSocket.socket != INVALID_SOCKET && FD_ISSET(hListenSocket.socket, &fdsetRecv))
1446+
if (hListenSocket.socket != INVALID_SOCKET && recv_set.count(hListenSocket.socket) > 0)
13511447
{
13521448
AcceptConnection(hListenSocket);
13531449
}
@@ -1378,9 +1474,9 @@ void CConnman::SocketHandler()
13781474
LOCK(pnode->cs_hSocket);
13791475
if (pnode->hSocket == INVALID_SOCKET)
13801476
continue;
1381-
recvSet = FD_ISSET(pnode->hSocket, &fdsetRecv);
1382-
sendSet = FD_ISSET(pnode->hSocket, &fdsetSend);
1383-
errorSet = FD_ISSET(pnode->hSocket, &fdsetError);
1477+
recvSet = recv_set.count(pnode->hSocket) > 0;
1478+
sendSet = send_set.count(pnode->hSocket) > 0;
1479+
errorSet = error_set.count(pnode->hSocket) > 0;
13841480
}
13851481
if (recvSet || errorSet)
13861482
{

‎src/net.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ class CConnman
346346
void DisconnectNodes();
347347
void NotifyNumConnectionsChanged();
348348
void InactivityCheck(CNode *pnode);
349+
bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set);
350+
void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set);
349351
void SocketHandler();
350352
void ThreadSocketHandler();
351353
void ThreadDNSAddressSeed();

‎src/netbase.cpp

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
#include <codecvt>
2222
#endif
2323

24+
#ifdef USE_POLL
25+
#include <poll.h>
26+
#endif
27+
2428
#if !defined(MSG_NOSIGNAL)
2529
#define MSG_NOSIGNAL 0
2630
#endif
@@ -264,11 +268,19 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c
264268
if (!IsSelectableSocket(hSocket)) {
265269
return IntrRecvError::NetworkError;
266270
}
267-
struct timeval tval = MillisToTimeval(std::min(endTime - curTime, maxWait));
271+
int timeout_ms = std::min(endTime - curTime, maxWait);
272+
#ifdef USE_POLL
273+
struct pollfd pollfd = {};
274+
pollfd.fd = hSocket;
275+
pollfd.events = POLLIN | POLLOUT;
276+
int nRet = poll(&pollfd, 1, timeout_ms);
277+
#else
278+
struct timeval tval = MillisToTimeval(timeout_ms);
268279
fd_set fdset;
269280
FD_ZERO(&fdset);
270281
FD_SET(hSocket, &fdset);
271282
int nRet = select(hSocket + 1, &fdset, nullptr, nullptr, &tval);
283+
#endif
272284
if (nRet == SOCKET_ERROR) {
273285
return IntrRecvError::NetworkError;
274286
}
@@ -499,11 +511,18 @@ bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocket, i
499511
// WSAEINVAL is here because some legacy version of winsock uses it
500512
if (nErr == WSAEINPROGRESS || nErr == WSAEWOULDBLOCK || nErr == WSAEINVAL)
501513
{
514+
#ifdef USE_POLL
515+
struct pollfd pollfd = {};
516+
pollfd.fd = hSocket;
517+
pollfd.events = POLLIN | POLLOUT;
518+
int nRet = poll(&pollfd, 1, nTimeout);
519+
#else
502520
struct timeval timeout = MillisToTimeval(nTimeout);
503521
fd_set fdset;
504522
FD_ZERO(&fdset);
505523
FD_SET(hSocket, &fdset);
506524
int nRet = select(hSocket + 1, nullptr, &fdset, nullptr, &timeout);
525+
#endif
507526
if (nRet == 0)
508527
{
509528
LogPrint(BCLog::NET, "connection to %s timeout\n", addrConnect.ToString());

0 commit comments

Comments
 (0)