Strange IOCP behavior when connecting/disconnecting more than 2500 sockets

Hi,
I've been looking for an answer to my problem for 2 weeks but I seem to
get stuck into it so I decided to post it here, maybe someone could help.

Here's what I did:

- I created a dummy TCP/IP IOCP based server with 1 thread for the user
GUI, 1 for the listener and 1 for the worker.

- I also created a dummy console test program that connects a specific
amount of sockets you can type at the console input.

- Everything seems to work fine when I connect 2500 sockets and then
disconnect them one by one (in a for{} loop). If i try to do it with more
than 2500, they get disconnected but the last N-2500 sockets won't .



Example:



a.. With 3000 connected sockets I only get 2499 disconnected;
b.. With 5000 sockets I get 1106 in a CLOSE_WAIT cycle (seen with
netstat) until i close the server.
I think the problem is that the worker thread does not receive the
completion notification for the rest of them and I'm wondering why??



Here's what I did:



Listening thread...



//Declaring our per-handle data

typedef struct user_info
{


SOCKET usersocket;
OVERLAPPED overlapped;

} UserInfo, *LPUserInfo;

DWORD connections; //global variable for the active connections

LPUserInfo users; //this will be the our per-handle data array

DWORD WINAPI ListeningThread(LPVOID user)
{

//. initializing the listening socket

sockaddr_in* temp = new sockaddr_in;

DWORD bytesrecv, flags = 0;

WSABUF buf;


listensocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP,
NULL, NULL, WSA_FLAG_OVERLAPPED);

temp->sin_addr = default_ip;
temp->sin_family = AF_INET;
temp->sin_port = htons(atoi("30000"));

bind(listensocket, (sockaddr*)temp, sizeof(sockaddr_in));

delete temp;

if(listen(listensocket, 10) == SOCKET_ERROR) return -1;

users = new UserInfo[10000]; //initiating...

buf.len = 512;
buf.buf = new char[buf.len];

// zero-fill our array because of the OVERLAPPED member that needs to be
so..

SecureZeroMemory(users, sizeof(UserInfo)*10000);



// Entering the accept-loop...

SOCKET temp;
connections = 0;

while(true)

{

temp = accept(listensocket, NULL, NULL); // no other info, we need
only the socket

if(temp == SOCKET_ERROR) return WSAGetLastError();

//now that we got a valid socket, let's start to initialize our
IOCP...

users[connections].socket = temp;

//adding our socket to the IOCP...

CreateIoCompletionPort((HANDLE)temp, DefaultCompletionPort,
(ULONG_PTR)&users[connections], NULL);


//last but not least... initiating the IO_PENDING...

WSARecv(temp, &buf, 1, &bytesrecv, &flags,
&users[connections++].overlapped, NULL);

//let the user know
DisplayConnectionsOnStatusBar(connections);

};

return 0;

}

/************************************************/





and the Worker thread



DWORD Winapi WorkerThread (LPVOID userdata)
{

LPOVERLAPPED lpovp;
OVERLAPPED ovp;
DWORD bytes;
ULONG_PTR completionKey;

LPUserInfo userinfo;

WSABUF buf;

//again, another buffers

buf.len = 512;
buf.buf = new char[buf.len];

lpovp = &ovp;

//now waiting for queued actions...

while (true)
{

GetQueuedCompletionStatus(CompletionPort, &bytes, &completionKey,
&lpovp, INFINITE);

userinfo = (LPUserInfo)completionKey;

if (bytes == 0) // if bytes = 0 then we deal with a disconnect;
{
closesocket(userinfo->socket);
SecureZeroMemory(userinfo, sizeof(UserInfo));

DisplayConnectionsOnStatusBar(--connections);

};

};

return 0;

}

/**************************************************/



That's the whole thing. It works fine with up to 2500 connections. I
wonder why
I tried everything possible, like setsockopt with SO_LINGER on, off, then
I tried to deactivate SO_SNDBUF and SO_RCVBUF. I really don't understand
where I missed something.

I'd really appreciate your help.

Thanks a lot!



Dragos

I see very little error handling in your code. Are you sure you are not
getting an error that would explain what is happening?
Vladimir.

"Dragos Dutu" wrote in message
news:#

Hi,
I've been looking for an answer to my problem for 2 weeks but I seem to
get stuck into it so I decided to post it here, maybe someone could help.

Here's what I did:

- I created a dummy TCP/IP IOCP based server with 1 thread for the user
GUI, 1 for the listener and 1 for the worker.

- I also created a dummy console test program that connects a specific
amount of sockets you can type at the console input.

- Everything seems to work fine when I connect 2500 sockets and then
disconnect them one by one (in a for{} loop). If i try to do it with more
than 2500, they get disconnected but the last N-2500 sockets won't .



Example:



a.. With 3000 connected sockets I only get 2499 disconnected;
b.. With 5000 sockets I get 1106 in a CLOSE_WAIT cycle (seen with
netstat) until i close the server.
I think the problem is that the worker thread does not receive the
completion notification for the rest of them and I'm wondering why??



Here's what I did:



Listening thread...



//Declaring our per-handle data

typedef struct user_info
{


SOCKET usersocket;
OVERLAPPED overlapped;

} UserInfo, *LPUserInfo;

DWORD connections; //global variable for the active connections

LPUserInfo users; //this will be the our per-handle data array

DWORD WINAPI ListeningThread(LPVOID user)
{

//. initializing the listening socket

sockaddr_in* temp = new sockaddr_in;

DWORD bytesrecv, flags = 0;

WSABUF buf;


listensocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP,
NULL, NULL, WSA_FLAG_OVERLAPPED);

temp->sin_addr = default_ip;
temp->sin_family = AF_INET;
temp->sin_port = htons(atoi("30000"));

bind(listensocket, (sockaddr*)temp, sizeof(sockaddr_in));

delete temp;

if(listen(listensocket, 10) == SOCKET_ERROR) return -1;

users = new UserInfo[10000]; //initiating...

buf.len = 512;
buf.buf = new char[buf.len];

// zero-fill our array because of the OVERLAPPED member that needs to be
so..

SecureZeroMemory(users, sizeof(UserInfo)*10000);



// Entering the accept-loop...

SOCKET temp;
connections = 0;

while(true)

{

temp = accept(listensocket, NULL, NULL); // no other info, we need
only the socket

if(temp == SOCKET_ERROR) return WSAGetLastError();

//now that we got a valid socket, let's start to initialize our
IOCP...

users[connections].socket = temp;

//adding our socket to the IOCP...

CreateIoCompletionPort((HANDLE)temp, DefaultCompletionPort,
(ULONG_PTR)&users[connections], NULL);


//last but not least... initiating the IO_PENDING...

WSARecv(temp, &buf, 1, &bytesrecv, &flags,
&users[connections++].overlapped, NULL);

//let the user know
DisplayConnectionsOnStatusBar(connections);

};

return 0;

}

/************************************************/





and the Worker thread



DWORD Winapi WorkerThread (LPVOID userdata)
{

LPOVERLAPPED lpovp;
OVERLAPPED ovp;
DWORD bytes;
ULONG_PTR completionKey;

LPUserInfo userinfo;

WSABUF buf;

//again, another buffers

buf.len = 512;
buf.buf = new char[buf.len];

lpovp = &ovp;

//now waiting for queued actions...

while (true)
{

GetQueuedCompletionStatus(CompletionPort, &bytes, &completionKey,
&lpovp, INFINITE);

userinfo = (LPUserInfo)completionKey;

if (bytes == 0) // if bytes = 0 then we deal with a disconnect;
{
closesocket(userinfo->socket);
SecureZeroMemory(userinfo, sizeof(UserInfo));

DisplayConnectionsOnStatusBar(--connections);

};

};

return 0;

}

/**************************************************/



That's the whole thing. It works fine with up to 2500 connections. I
wonder why
I tried everything possible, like setsockopt with SO_LINGER on, off, then
I tried to deactivate SO_SNDBUF and SO_RCVBUF. I really don't understand
where I missed something.

I'd really appreciate your help.

Thanks a lot!



Dragos

It's just a snippet of code poor in error handlings. I wrote it with error
handlings too, but it's twice bigger than this.
The problem is not in the code, or at least I didn't notice any so far and
that's why I simplified it.
Interesting is the fact that it works well with up to 2500 connected
sockets. Then IOCP won't send any notification and I don't understand
why...

"Vladimir Petter [MSFT]" wrote in message
news:

I see very little error handling in your code. Are you sure you are not
getting an error that would explain what is happening?
Vladimir.

"Dragos Dutu" wrote in message
news:#

Hi,
I've been looking for an answer to my problem for 2 weeks but I seem to
get stuck into it so I decided to post it here, maybe someone could
help.

Here's what I did:

- I created a dummy TCP/IP IOCP based server with 1 thread for the user
GUI, 1 for the listener and 1 for the worker.

- I also created a dummy console test program that connects a specific
amount of sockets you can type at the console input.

- Everything seems to work fine when I connect 2500 sockets and then
disconnect them one by one (in a for{} loop). If i try to do it with
more than 2500, they get disconnected but the last N-2500 sockets won't
.



Example:



a.. With 3000 connected sockets I only get 2499 disconnected;
b.. With 5000 sockets I get 1106 in a CLOSE_WAIT cycle (seen with
netstat) until i close the server.
I think the problem is that the worker thread does not receive the
completion notification for the rest of them and I'm wondering why??



Here's what I did:



Listening thread...



//Declaring our per-handle data

typedef struct user_info
{


SOCKET usersocket;
OVERLAPPED overlapped;

} UserInfo, *LPUserInfo;

DWORD connections; //global variable for the active connections

LPUserInfo users; //this will be the our per-handle data array

DWORD WINAPI ListeningThread(LPVOID user)
{

//. initializing the listening socket

sockaddr_in* temp = new sockaddr_in;

DWORD bytesrecv, flags = 0;

WSABUF buf;


listensocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP,
NULL, NULL, WSA_FLAG_OVERLAPPED);

temp->sin_addr = default_ip;
temp->sin_family = AF_INET;
temp->sin_port = htons(atoi("30000"));

bind(listensocket, (sockaddr*)temp, sizeof(sockaddr_in));

delete temp;

if(listen(listensocket, 10) == SOCKET_ERROR) return -1;

users = new UserInfo[10000]; //initiating...

buf.len = 512;
buf.buf = new char[buf.len];

// zero-fill our array because of the OVERLAPPED member that needs to
be so..

SecureZeroMemory(users, sizeof(UserInfo)*10000);



// Entering the accept-loop...

SOCKET temp;
connections = 0;

while(true)

{

temp = accept(listensocket, NULL, NULL); // no other info, we need
only the socket

if(temp == SOCKET_ERROR) return WSAGetLastError();

//now that we got a valid socket, let's start to initialize our
IOCP...

users[connections].socket = temp;

//adding our socket to the IOCP...

CreateIoCompletionPort((HANDLE)temp, DefaultCompletionPort,
(ULONG_PTR)&users[connections], NULL);


//last but not least... initiating the IO_PENDING...

WSARecv(temp, &buf, 1, &bytesrecv, &flags,
&users[connections++].overlapped, NULL);

//let the user know
DisplayConnectionsOnStatusBar(connections);

};

return 0;

}

/************************************************/





and the Worker thread



DWORD Winapi WorkerThread (LPVOID userdata)
{

LPOVERLAPPED lpovp;
OVERLAPPED ovp;
DWORD bytes;
ULONG_PTR completionKey;

LPUserInfo userinfo;

WSABUF buf;

//again, another buffers

buf.len = 512;
buf.buf = new char[buf.len];

lpovp = &ovp;

//now waiting for queued actions...

while (true)
{

GetQueuedCompletionStatus(CompletionPort, &bytes, &completionKey,
&lpovp, INFINITE);

userinfo = (LPUserInfo)completionKey;

if (bytes == 0) // if bytes = 0 then we deal with a disconnect;
{
closesocket(userinfo->socket);
SecureZeroMemory(userinfo, sizeof(UserInfo));

DisplayConnectionsOnStatusBar(--connections);

};

};

return 0;

}

/**************************************************/



That's the whole thing. It works fine with up to 2500 connections. I
wonder why
I tried everything possible, like setsockopt with SO_LINGER on, off,
then I tried to deactivate SO_SNDBUF and SO_RCVBUF. I really don't
understand where I missed something.

I'd really appreciate your help.

Thanks a lot!



Dragos

Just a hint, try to test your code on a checked build of windows, and look
at dbgview what happens when you try to overcome the magic "2500" limit.

Volodymyr
"Dragos Dutu" wrote in message
news:

It's just a snippet of code poor in error handlings. I wrote it with
error handlings too, but it's twice bigger than this.
The problem is not in the code, or at least I didn't notice any so far
and that's why I simplified it.
Interesting is the fact that it works well with up to 2500 connected
sockets. Then IOCP won't send any notification and I don't understand
why...

"Vladimir Petter [MSFT]" wrote in message
news:

I see very little error handling in your code. Are you sure you are not
getting an error that would explain what is happening?
Vladimir.

"Dragos Dutu" wrote in message
news:#

Hi,
I've been looking for an answer to my problem for 2 weeks but I seem to
get stuck into it so I decided to post it here, maybe someone could
help.

Here's what I did:

- I created a dummy TCP/IP IOCP based server with 1 thread for the user
GUI, 1 for the listener and 1 for the worker.

- I also created a dummy console test program that connects a specific
amount of sockets you can type at the console input.

- Everything seems to work fine when I connect 2500 sockets and then
disconnect them one by one (in a for{} loop). If i try to do it with
more than 2500, they get disconnected but the last N-2500 sockets won't
.



Example:



a.. With 3000 connected sockets I only get 2499 disconnected;
b.. With 5000 sockets I get 1106 in a CLOSE_WAIT cycle (seen with
netstat) until i close the server.
I think the problem is that the worker thread does not receive the
completion notification for the rest of them and I'm wondering why??



Here's what I did:



Listening thread...



//Declaring our per-handle data

typedef struct user_info
{


SOCKET usersocket;
OVERLAPPED overlapped;

} UserInfo, *LPUserInfo;

DWORD connections; //global variable for the active connections

LPUserInfo users; //this will be the our per-handle data array

DWORD WINAPI ListeningThread(LPVOID user)
{

//. initializing the listening socket

sockaddr_in* temp = new sockaddr_in;

DWORD bytesrecv, flags = 0;

WSABUF buf;


listensocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP,
NULL, NULL, WSA_FLAG_OVERLAPPED);

temp->sin_addr = default_ip;
temp->sin_family = AF_INET;
temp->sin_port = htons(atoi("30000"));

bind(listensocket, (sockaddr*)temp, sizeof(sockaddr_in));

delete temp;

if(listen(listensocket, 10) == SOCKET_ERROR) return -1;

users = new UserInfo[10000]; //initiating...

buf.len = 512;
buf.buf = new char[buf.len];

// zero-fill our array because of the OVERLAPPED member that needs to
be so..

SecureZeroMemory(users, sizeof(UserInfo)*10000);



// Entering the accept-loop...

SOCKET temp;
connections = 0;

while(true)

{

temp = accept(listensocket, NULL, NULL); // no other info, we need
only the socket

if(temp == SOCKET_ERROR) return WSAGetLastError();

//now that we got a valid socket, let's start to initialize our
IOCP...

users[connections].socket = temp;

//adding our socket to the IOCP...

CreateIoCompletionPort((HANDLE)temp, DefaultCompletionPort,
(ULONG_PTR)&users[connections], NULL);


//last but not least... initiating the IO_PENDING...

WSARecv(temp, &buf, 1, &bytesrecv, &flags,
&users[connections++].overlapped, NULL);

//let the user know
DisplayConnectionsOnStatusBar(connections);

};

return 0;

}

/************************************************/





and the Worker thread



DWORD Winapi WorkerThread (LPVOID userdata)
{

LPOVERLAPPED lpovp;
OVERLAPPED ovp;
DWORD bytes;
ULONG_PTR completionKey;

LPUserInfo userinfo;

WSABUF buf;

//again, another buffers

buf.len = 512;
buf.buf = new char[buf.len];

lpovp = &ovp;

//now waiting for queued actions...

while (true)
{

GetQueuedCompletionStatus(CompletionPort, &bytes,
&completionKey,
&lpovp, INFINITE);

userinfo = (LPUserInfo)completionKey;

if (bytes == 0) // if bytes = 0 then we deal with a disconnect;
{
closesocket(userinfo->socket);
SecureZeroMemory(userinfo, sizeof(UserInfo));

DisplayConnectionsOnStatusBar(--connections);

};

};

return 0;

}

/**************************************************/



That's the whole thing. It works fine with up to 2500 connections. I
wonder why
I tried everything possible, like setsockopt with SO_LINGER on, off,
then I tried to deactivate SO_SNDBUF and SO_RCVBUF. I really don't
understand where I missed something.

I'd really appreciate your help.

Thanks a lot!



Dragos

I tried on 5 different computers with XP SP2 and I got the same. Dbgview
shows me what HAPPENS, it can't show me what doesn't happen. Sockets get
all connected, but when it comes to disconnection, only the first N-2500
get disconnected. The remaining sockets won't send completion notification
at the disconnect time.

"Volodymyr Shcherbyna" wrote in message
news:

Just a hint, try to test your code on a checked build of windows, and
look at dbgview what happens when you try to overcome the magic "2500"
limit.

Volodymyr
"Dragos Dutu" wrote in message
news:

It's just a snippet of code poor in error handlings. I wrote it with
error handlings too, but it's twice bigger than this.
The problem is not in the code, or at least I didn't notice any so far
and that's why I simplified it.
Interesting is the fact that it works well with up to 2500 connected
sockets. Then IOCP won't send any notification and I don't understand
why...

"Vladimir Petter [MSFT]" wrote in message
news:

I see very little error handling in your code. Are you sure you are not
getting an error that would explain what is happening?
Vladimir.

"Dragos Dutu" wrote in message
news:#

Hi,
I've been looking for an answer to my problem for 2 weeks but I seem
to get stuck into it so I decided to post it here, maybe someone could
help.

Here's what I did:

- I created a dummy TCP/IP IOCP based server with 1 thread for the
user GUI, 1 for the listener and 1 for the worker.

- I also created a dummy console test program that connects a specific
amount of sockets you can type at the console input.

- Everything seems to work fine when I connect 2500 sockets and then
disconnect them one by one (in a for{} loop). If i try to do it with
more than 2500, they get disconnected but the last N-2500 sockets
won't .



Example:



a.. With 3000 connected sockets I only get 2499 disconnected;
b.. With 5000 sockets I get 1106 in a CLOSE_WAIT cycle (seen with
netstat) until i close the server.
I think the problem is that the worker thread does not receive the
completion notification for the rest of them and I'm wondering why??



Here's what I did:



Listening thread...



//Declaring our per-handle data

typedef struct user_info
{


SOCKET usersocket;
OVERLAPPED overlapped;

} UserInfo, *LPUserInfo;

DWORD connections; //global variable for the active connections

LPUserInfo users; //this will be the our per-handle data array

DWORD WINAPI ListeningThread(LPVOID user)
{

//. initializing the listening socket

sockaddr_in* temp = new sockaddr_in;

DWORD bytesrecv, flags = 0;

WSABUF buf;


listensocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP,
NULL, NULL, WSA_FLAG_OVERLAPPED);

temp->sin_addr = default_ip;
temp->sin_family = AF_INET;
temp->sin_port = htons(atoi("30000"));

bind(listensocket, (sockaddr*)temp, sizeof(sockaddr_in));

delete temp;

if(listen(listensocket, 10) == SOCKET_ERROR) return -1;

users = new UserInfo[10000]; //initiating...

buf.len = 512;
buf.buf = new char[buf.len];

// zero-fill our array because of the OVERLAPPED member that needs to
be so..

SecureZeroMemory(users, sizeof(UserInfo)*10000);



// Entering the accept-loop...

SOCKET temp;
connections = 0;

while(true)

{

temp = accept(listensocket, NULL, NULL); // no other info, we need
only the socket

if(temp == SOCKET_ERROR) return WSAGetLastError();

//now that we got a valid socket, let's start to initialize our
IOCP...

users[connections].socket = temp;

//adding our socket to the IOCP...

CreateIoCompletionPort((HANDLE)temp, DefaultCompletionPort,
(ULONG_PTR)&users[connections], NULL);


//last but not least... initiating the IO_PENDING...

WSARecv(temp, &buf, 1, &bytesrecv, &flags,
&users[connections++].overlapped, NULL);

//let the user know
DisplayConnectionsOnStatusBar(connections);

};

return 0;

}

/************************************************/





and the Worker thread



DWORD Winapi WorkerThread (LPVOID userdata)
{

LPOVERLAPPED lpovp;
OVERLAPPED ovp;
DWORD bytes;
ULONG_PTR completionKey;

LPUserInfo userinfo;

WSABUF buf;

//again, another buffers

buf.len = 512;
buf.buf = new char[buf.len];

lpovp = &ovp;

//now waiting for queued actions...

while (true)
{

GetQueuedCompletionStatus(CompletionPort, &bytes,
&completionKey,
&lpovp, INFINITE);

userinfo = (LPUserInfo)completionKey;

if (bytes == 0) // if bytes = 0 then we deal with a disconnect;
{
closesocket(userinfo->socket);
SecureZeroMemory(userinfo, sizeof(UserInfo));

DisplayConnectionsOnStatusBar(--connections);

};

};

return 0;

}

/**************************************************/



That's the whole thing. It works fine with up to 2500 connections. I
wonder why
I tried everything possible, like setsockopt with SO_LINGER on, off,
then I tried to deactivate SO_SNDBUF and SO_RCVBUF. I really don't
understand where I missed something.

I'd really appreciate your help.

Thanks a lot!



Dragos