c++ socket accept() get Resource deadlock avoided errno with epoll

Question

I'm a C++ beginner learning I/O multiplexing.

Here is my code:

test.cpp (doesn't use epoll() and works well):

#include <iostream>
#include <cstdio>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>

int main() {
    std::cout << "Hello" << std::endl;

    char buffer[1024];
    buffer[0] = 'f';
    fprintf(stdout, "%s", buffer);
    std::cout << "Hello" << std::endl;

    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }
    struct sockaddr clientAddr;
    unsigned int clientlen = sizeof(clientAddr);
    int acceptFd = accept(serverFd, &clientAddr, &clientlen);
    if (acceptFd < 0) {
        fprintf(stderr, "Fail to create client connection file descriptor\n");
        return 1;
    }
    int fd = acceptFd;
    ssize_t received = recv(fd, &buffer, 1024, 0);
    if (received < 0) {
        fprintf(stderr, "Fail to received bytess from client\n");
        if (errno == EINTR) {
            fprintf(stderr, "Reason: EINTR\n");
        } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
            fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
        } else {
            fprintf(stderr, "Reason: %d\n", errno);
            close(fd);
            return 1;
        }
    } else if (received == 0) {
        close(fd);
    } else {
        buffer[received] = '\0';
        fprintf(stdout, "%s", buffer);
    }
}

test_2.cpp (does use epoll() and doesn't work well):

#include <iostream>
#include <sys/epoll.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <signal.h>
#include <string.h>

int main() {
    // TODO: too much error message to handle, so it's necessary to deal with it (maybe macros can)
    std::cout << "Hello" << std::endl;

    // process ignore SIGPIPE which is caused by send(), or process will exit, which is hard to find out
    signal(SIGPIPE, SIG_IGN);

    // here needs a socket fd or other fd
    // well, AF_INET is okay;socket(PF_INET, SOCK_SEQPACKET, 0) is sctp, tcp cannot use SOCK_SEQPACKET :(
    // when using tcp, watch out **record boundaries**
    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (serverFd < 0) {
        fprintf(stderr, "Fail to create socket file descriptor\n");
        return 1;
    }
    // nonblock
    // nonblock
    int flags = fcntl(serverFd, F_GETFL, 0);
    if (flags < 0) {
        fprintf(stderr, "Fail to get flags\n");
        return 1;
    }
    int setFlagResult = fcntl(serverFd, F_SETFL, flags | O_NONBLOCK);
    if (setFlagResult < 0) {
        fprintf(stderr, "Fail to set flags\n");
        return 1;
    }

    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }

    // epoll fd
    int epollFd = epoll_create(1);
    if (epollFd < 0) {
        fprintf(stderr, "Fail to create epoll file descriptor\n");
        return 1;
    }
    // event
    struct epoll_event event, events[1024];
    event.events = EPOLLIN;
    event.data.fd = serverFd;
    // ctl
    int ctlResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, serverFd, &event);
    if (ctlResult < 0) {
        fprintf(stderr, "Fail to run epoll_ctl\n");
        return 1;
    }
    // wait
    while (1) {
        int event_count = epoll_wait(epollFd, events, 1024, -1);

        for (int i = 0; i < event_count; i++) {
            struct epoll_event event = events[i];
            // accept
            if (event.data.fd == serverFd) {
                unsigned int clientlen = sizeof(clientAddr);
                int acceptFd = accept(serverFd, (struct sockaddr *) &clientAddr, &clientlen);
                if (acceptFd < 0) {
                    fprintf(stderr, "Fail to create client connection file descriptor\n");
                    fprintf(stderr, "Fail Reason: %d\n", errno);
                    return 1;
                }

                // nonblock
                int flags = fcntl(acceptFd, F_GETFL, 0);
                if (flags < 0) {
                    fprintf(stderr, "Fail to get flags\n");
                    return 1;
                }
                int setFlagResult = fcntl(acceptFd, F_SETFL, flags | O_NONBLOCK);
                if (setFlagResult < 0) {
                    fprintf(stderr, "Fail to set flags\n");
                    return 1;
                }

                struct epoll_event event;
                event.events = EPOLLIN;
                event.data.fd = serverFd;
                int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
                if (ctlClientResult < 0) {
                    fprintf(stderr, "Fail to run epoll_ctl\n");
                    return 1;
                }
                // client recv
            } else if (event.events & EPOLLIN) {
                int fd = event.data.fd;
                char buffer[1024+1];
                ssize_t received = recv(fd, &buffer, 1024, 0);
                if (received < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        fprintf(stderr, "Reason: %d\n", errno);
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (received == 0) {
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                    close(fd);
                } else {
                    buffer[received] = '\0';
                    fprintf(stdout, "%s", buffer);
                    // if you want to send something...
                    event.events |= EPOLLOUT;
                    // here is some data that event can hold
                    event.data.u32 = (uint32_t) 1;
                    // you can now send data or just put event in epoll, which is maybe easier
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                }
                // client send
            }  else if (event.events & EPOLLOUT) {
                int fd = event.data.fd;
                char buffer[] = "I see you";
                ssize_t sendResult = send(fd, &buffer, 1024, 0);
                if (sendResult < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        if (errno == EPIPE) {
                            fprintf(stderr, "Reason: EPIPE\n");
                        } else {
                            fprintf(stderr, "Reason: %d\n", errno);
                        }
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (sendResult == 0) {
                    event.events = EPOLLIN;
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                } else {

                    // if you want to recv something...
//                    event.events |= EPOLLIN;
//                    // you can now send data or just put event in epoll, which is maybe easier
//                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
//                    if (ctlClientResult < 0) {
//                        fprintf(stderr, "Fail to run epoll_ctl\n");
//                        return 1;
//                    }
                }
            }


        }
    }

    return 0;
}

When I try to make a TCP socket connection (such as curl -v "http://host:80/", which can make test2.cpp run in line 81), acceptFd is < 0 and errno is 11 according to line 84, which means "Resource deadlock avoided".

Why? There isn't any thread-related code, is it?

"_Here is my code_" All information relevant to the question, including [mcve], must be present in the question itself, and not be hidden behind external links. — Algirdas Preidžius, Feb 20 '19 at 16:11
Your code belongs in the question as text, not at an external link that can go bad. [mcve] — Retired Ninja, Feb 20 '19 at 16:11
Sorry, I will pay attention to how to ask this kind of question in the future — Sora Shiro, Feb 23 '19 at 13:50

score 1 · Answer 1 · answered Feb 20 '19 at 19:54

            struct epoll_event event;
            event.events = EPOLLIN;
            event.data.fd = serverFd;
            int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

When you add the newly-accepted connection to the epoll set, you tell it to report it as a hit on serverFd. So when the client sends you data, you try to accept a new connection instead.

Change event.data.fd = serverFd to event.data.fd = acceptFd.

Then you can move on to your next bug:

            char buffer[] = "I see you";
            ssize_t sendResult = send(fd, &buffer, 1024, 0);

1024?!

Also, any time you use non-blocking sockets, you should add code to handle EAGAIN or EWOULDBLOCK errors as non-fatal.

Thank you! I fixed these bug and it worked well partly :) – Sora Shiro Feb 23 '19 at 13:48 — Sora Shiro, Feb 23 '19 at 13:48

Remy Lebeau · Accepted Answer · 2019-02-20T17:15:02.333

Error code 11 is EAGAIN, which is a very common error code to encounter when dealing with non-blocking socket I/O. It means the requested operation has nothing to do, try it again later.

In the case of accept(), it means:

EAGAIN or EWOULDBLOCK

The socket is marked nonblocking and no connections are present to be accepted. POSIX.1-2001 and POSIX.1-2008 allow either error to be returned for this case, and do not require these constants to have the same value, so a portable application should check for both possibilities.

This means you are calling accept() at the wrong time, when the listening socket does not have any clients to accept. Double check your epoll() usage, it likely has a logic bug in it that is causing you to call accept() prematurely.

For instance, after accept() has successfully accepted a client, you are assigning the listening socket instead of the client socket to event.data.fd when calling epoll_ctl(EPOLL_CTL_ADD):

struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd; // <-- HERE, SHOULD BE acceptFd INSTEAD!
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

As such, when the client has data waiting to be read, your loop will end up calling accept() on the listening socket instead of calling recv() on the client socket.

Also, you are not checking the reported event.events field for socket errors. If an error occurs on a client socket, the reported events may include the EPOLLERR and/or EPOLLHUP flags. You should check for those flags and if present then close the client socket, before checking for the EPOLLIN flag to call recv().

Also, note that in your example, it is not necessary to call epoll_ctl(EPOLL_CTL_DEL) before calling close() on a socket. Closing a socket file descriptor will automatically remove it from the epoll instance that is monitoring it.

Thanks! I changed serverFd to acceptFd and I can recv client bytes(but a new bug occured... when I tried to send "I see you" I got errno == 88, which is "Socket operation on non-socket"); Also, I handled EPOLLERR and EPOLLHUP, and I deleted call epoll_ctl(EPOLL_CTL_DEL) Thank you:) — Sora Shiro, Feb 23 '19 at 13:47
After I commented `event.data.u32 = (uint32_t) 1;` all works well, which is confused me... — Sora Shiro, Feb 23 '19 at 14:09
@ShiroSora that is because `event.data` is a `union` of multiple values. Setting its `u32` field **overwrites** its `fd` field. That is why you end up passing an invalid socket descriptor to `send()` — Remy Lebeau, Feb 23 '19 at 15:16
That's right! I thought **wrongly** that `event.data` is `struct`... — Sora Shiro, Feb 24 '19 at 04:00

c++ socket accept() get Resource deadlock avoided errno with epoll

2 Answers2