Fix oo_exit_hook hang in multithreaded scenarios

[liujinhui-job]

Problem Description：When using my test program to test multithreading, I found that sometimes the main thread gets stuck when the program exits, preventing it from terminating. The call stack shows it is blocked at CITP_FDTABLE_LOCK_RD.
I tried changing CITP_FDTABLE_LOCK_RDto CITP_FDTABLE_LOCK/ CITP_FDTABLE_UNLOCK, and the issue no longer occurs. In fact, the root cause has not been identified; I only believe that replacing it with CITP_FDTABLE_LOCKdoes not affect the program logic, and the problem is no longer reproducible.

Moreover, I analyzed the function uncache_active_netifs called after acquiring the lock, and found that it uses CITP_FDTABLE_ASSERT_LOCKED instead of CITP_FDTABLE_ASSERT_LOCKED_RD. Therefore, I believe CITP_FDTABLE_LOCK should be used instead of CITP_FDTABLE_LOCK_RD. I am not sure if my analysis is correct.

The test code is shown below.

#define _GNU_SOURCE
#include <arpa/inet.h>
#include <pthread.h>
#include <sys/epoll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sched.h>

typedef struct {
    int epoll_fd;
    int cpu;
    int udp_fd;
    struct sockaddr_in dst;
    int pkt_cnt;
    int pkt_size;
} thread_arg_t;

static void bind_cpu(int cpu_id)
{
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(cpu_id, &cpuset);
    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
}

static void *epoll_thread(void *arg)
{
    thread_arg_t *t = arg;
    bind_cpu(t->cpu);

    struct epoll_event ev;
    while (1) {
        int n = epoll_wait(t->epoll_fd, &ev, 1, -1);
    }

    return NULL;
}

int main(int argc, char **argv)
{
    if (argc != 6) {
        fprintf(stderr,
                "Usage: %s <dst_ip> <dst_port> <threads> <pkts_per_thread> <pkt_size> \n",
                argv[0]);
        return 1;
    }

    const char *ip = argv[1];
    int port = atoi(argv[2]);
    int threads = atoi(argv[3]);
    int pkts_per_thread = atoi(argv[4]);
    int pkt_size = atoi(argv[5]);

    int udp_fd = socket(AF_INET, SOCK_DGRAM, 0);
    if (udp_fd < 0) {
        perror("socket");
        return 1;
    }

    struct sockaddr_in dst = {
        .sin_family = AF_INET,
        .sin_port = htons(port)
    };
    inet_pton(AF_INET, ip, &dst.sin_addr);

    /* 主线程绑定 CPU 1 */
    bind_cpu(1);

    pthread_t *pt = malloc(sizeof(pthread_t) * threads);
    thread_arg_t *args = malloc(sizeof(thread_arg_t) * threads);

    for (int i = 0; i < threads; i++) {
        int epfd = epoll_create1(0);
        if (epfd < 0) {
            perror("epoll_create1");
            return 1;
        }

        struct epoll_event ev = {
            .events = EPOLLOUT,
            .data.fd = udp_fd
        };
        epoll_ctl(epfd, EPOLL_CTL_ADD, udp_fd, &ev);

        args[i] = (thread_arg_t){
            .epoll_fd = epfd,
            .cpu = i + 2,
            .udp_fd = udp_fd,
            .dst = dst,
            .pkt_cnt = pkts_per_thread,
            .pkt_size = pkt_size
        };

        pthread_create(&pt[i], NULL, epoll_thread, &args[i]);
    }

    char *buf = malloc(pkt_size);
    memset(buf, 'A', pkt_size);

    while (pkts_per_thread--) {
        sendto(udp_fd, buf, pkt_size, 0,
               (struct sockaddr *)&dst, sizeof(dst));
    }

    close(udp_fd);
    free(pt);
    free(args);
    return 0;
}

Usage: %s <dst_ip> <dst_port> <pkts_per_thread> <pkt_size>
Test method:
./udp_client 10.0.0.33 8888 99 1 512
Yes, I created 99 threads, and I found that the program would hang indefinitely on exit and the process could not terminate.