From 6cc11a5ab821fcc496018478d06f5956b8bd4f20 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Wed, 1 Jul 2026 00:46:36 +0000 Subject: [PATCH 01/14] Implement socket prioritization for system critical events Signed-off-by: Satheesha Gowda --- src/ae.c | 109 +++ src/ae.h | 24 +- src/ae_epoll.c | 5 + src/ae_evport.c | 5 + src/ae_kqueue.c | 5 + src/ae_select.c | 5 + src/cluster.c | 2 + src/cluster_legacy.c | 9 +- src/cluster_migrateslots.c | 6 + src/commands.def | 6 +- src/commands/client-kill.json | 4 + src/commands/client-list.json | 4 + src/connection.c | 52 ++ src/connection.h | 30 +- src/io_threads.c | 240 +++--- src/io_threads.h | 2 +- src/networking.c | 19 +- src/replication.c | 8 + src/server.c | 7 +- src/server.h | 57 +- src/socket.c | 22 +- src/tls.c | 12 +- src/unit/test_socket_prioritization.cpp | 771 +++++++++++++++++++ tests/unit/cluster/links.tcl | 3 +- tests/unit/cluster/socket-prioritization.tcl | 130 ++++ tests/unit/introspection.tcl | 39 +- tests/unit/socket-prioritization.tcl | 153 ++++ 27 files changed, 1546 insertions(+), 183 deletions(-) create mode 100644 src/unit/test_socket_prioritization.cpp create mode 100644 tests/unit/cluster/socket-prioritization.tcl create mode 100644 tests/unit/socket-prioritization.tcl diff --git a/src/ae.c b/src/ae.c index 6d1fac8cada..4968a14683c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -33,6 +33,7 @@ #include "ae.h" #include "anet.h" #include "serverassert.h" +#include "monotonic.h" #include #include @@ -77,6 +78,14 @@ * implementations are AE_READABLE & AE_WRITABLE. */ #define BACKEND_MASK(mask) ((mask) & (AE_READABLE | AE_WRITABLE)) +/* High priority event loop periodic preemptive poll interval in microseconds.*/ +#define AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US 1000 +/* High priority event loop periodic preemptive poll mask.*/ +#define AE_HP_EVENT_PREEMPTIVE_CHECK_MASK 3 + +/* last time high priority event loop was polled */ +static long long hp_event_loop_last_poll_us = 0; + aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; @@ -96,6 +105,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->aftersleep = NULL; eventLoop->custompoll = NULL; eventLoop->flags = 0; + eventLoop->hp_event_loop = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -167,6 +177,9 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { } void aeDeleteEventLoop(aeEventLoop *eventLoop) { + if (eventLoop->hp_event_loop) { + aeDeleteEventLoop(eventLoop->hp_event_loop); + } aeApiFree(eventLoop->apidata); zfree(eventLoop->events); zfree(eventLoop->fired); @@ -187,6 +200,12 @@ void aeStop(aeEventLoop *eventLoop) { } int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc, void *clientData) { + if (mask & AE_HIGH_PRIORITY) { + if (eventLoop->hp_event_loop) { + /* Register events on to high priority event loop */ + return aeCreateFileEvent(eventLoop->hp_event_loop, fd, mask & ~AE_HIGH_PRIORITY, proc, clientData); + } + } AE_LOCK(eventLoop); int ret = AE_ERR; @@ -214,7 +233,16 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { + if (eventLoop->hp_event_loop) { + /* Check if the event exists in HP loop first */ + int hp_mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); + if (hp_mask != AE_NONE) { + aeDeleteFileEvent(eventLoop->hp_event_loop, fd, mask); + return; + } + } AE_LOCK(eventLoop); + mask &= ~AE_HIGH_PRIORITY; if (fd >= eventLoop->setsize) goto done; aeFileEvent *fe = &eventLoop->events[fd]; @@ -251,6 +279,12 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { } void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { + if (eventLoop->hp_event_loop) { + int mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); + if (mask != AE_NONE) { + return aeGetFileClientData(eventLoop->hp_event_loop, fd); + } + } if (fd >= eventLoop->setsize) return NULL; aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) return NULL; @@ -259,6 +293,12 @@ void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { } int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { + if (eventLoop->hp_event_loop) { + int mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); + if (mask != AE_NONE) { + return mask | AE_HIGH_PRIORITY; + } + } if (fd >= eventLoop->setsize) return 0; aeFileEvent *fe = &eventLoop->events[fd]; @@ -403,6 +443,31 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } +/* Process all high priority events immediately */ +static int aeProcessHPEventsNow(aeEventLoop *eventLoop) { + int processed = 0; + if (eventLoop->hp_event_loop != NULL) { + processed = aeProcessEvents(eventLoop->hp_event_loop, AE_ALL_EVENTS | AE_DONT_WAIT); + hp_event_loop_last_poll_us = getMonotonicUs(); + } + return processed; +} + +/* + * Preemptively processes high priority events if the elapsed time since the last poll + * exceeds the AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US threshold and iter count + * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK. + */ +int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { + if (eventLoop->hp_event_loop == NULL) return 0; + if ((iter_count & AE_HP_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { + if (elapsedUs(hp_event_loop_last_poll_us) >= AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US) { + return aeProcessHPEventsNow(eventLoop); + } + } + return 0; +} + /* Process every pending file event, then every pending time event * (that may be registered by file event callbacks just processed). * Without special flags the function sleeps until some file event @@ -467,8 +532,35 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); + /* Process all high-priority events before we start processing the normal events + * if the high-priority event loop's FD (`hp_fd`) was fired.*/ + int hp_fd = -1; + if (eventLoop->hp_event_loop != NULL) { + hp_fd = aeApiGetPollFd(eventLoop->hp_event_loop); + } + int hp_fired = 0; + if (hp_fd != -1) { + for (j = 0; j < numevents; j++) { + if (eventLoop->fired[j].fd == hp_fd) { + hp_fired = 1; + break; + } + } + } + if (hp_fired) { + processed += aeProcessHPEventsNow(eventLoop); + } + for (j = 0; j < numevents; j++) { + /* Periodic Preemptive Poll: + * If processing normal events is taking more than AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US, check for new HP events. + * Batch AE_HP_EVENT_PREEMPTIVE_CHECK_MASK checks to minimize monotonic clock call overhead. */ + processed += aeProcessHPEventsPreemptively(eventLoop, j); + int fd = eventLoop->fired[j].fd; + /* Skip processing HP events here again as they were already processed above */ + if (hp_fired && fd == hp_fd) continue; + aeFileEvent *fe = &eventLoop->events[fd]; int mask = eventLoop->fired[j].mask; int fired = 0; /* Number of events fired for current fd. */ @@ -579,3 +671,20 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { eventLoop->flags &= ~AE_PROTECT_POLL; } } + +/* Event handler for the nested high-priority event loop poll, + * which will immediately process high priority events registered on the high-priority event loop file descriptor.*/ +static void hpEventLoopHandler(aeEventLoop *el, int fd, void *privdata, int mask) { + AE_NOTUSED(fd); + AE_NOTUSED(privdata); + AE_NOTUSED(mask); + aeProcessHPEventsNow(el); +} + +/* Register a high-priority event loop file descriptor in the main event loop, + * which will awake main eventloop when the high-priority event loop file descriptor is fired. */ +void aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop) { + eventLoop->hp_event_loop = hp_event_loop; + int hp_fd = aeApiGetPollFd(hp_event_loop); + if (hp_fd != -1) aeCreateFileEvent(eventLoop, hp_fd, AE_READABLE, hpEventLoopHandler, hp_event_loop); +} diff --git a/src/ae.h b/src/ae.h index 86916bddce8..9099a5bbf53 100644 --- a/src/ae.h +++ b/src/ae.h @@ -39,14 +39,15 @@ #define AE_OK 0 #define AE_ERR -1 -#define AE_NONE 0 /* No events registered. */ -#define AE_READABLE 1 /* Fire when descriptor is readable. */ -#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ -#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ - READABLE event already fired in the same event \ - loop iteration. Useful when you want to persist \ - things to disk before sending replies, and want \ - to do that in a group fashion. */ +#define AE_NONE 0 /* No events registered. */ +#define AE_READABLE 1 /* Fire when descriptor is readable. */ +#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ +#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ + READABLE event already fired in the same event \ + loop iteration. Useful when you want to persist \ + things to disk before sending replies, and want \ + to do that in a group fashion. */ +#define AE_HIGH_PRIORITY 8 /* Route to high-priority nested loop */ #define AE_FILE_EVENTS (1 << 0) #define AE_TIME_EVENTS (1 << 1) @@ -123,6 +124,9 @@ typedef struct aeEventLoop { aeCustomPollProc *custompoll; pthread_mutex_t poll_mutex; int flags; + + /* QoS */ + struct aeEventLoop *hp_event_loop; } aeEventLoop; /* Prototypes */ @@ -151,5 +155,9 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); +/* Process high priority events preemptively. */ +int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count); +/* Link a high-priority event loop to a main event loop for nested polling. */ +void aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop); #endif diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 5ceda92b735..ebe401eb48f 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -134,3 +134,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "epoll"; } + +static int aeApiGetPollFd(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + return state->epfd; +} diff --git a/src/ae_evport.c b/src/ae_evport.c index d730abb097d..e990e141f9c 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -289,3 +289,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "evport"; } + +static int aeApiGetPollFd(aeEventLoop *eventLoop) { + UNUSED(eventLoop); + return -1; +} diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 33615c2da87..703d163c19b 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -180,3 +180,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "kqueue"; } + +static int aeApiGetPollFd(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + return state->kqfd; +} diff --git a/src/ae_select.c b/src/ae_select.c index df19617dacb..69ab3caed20 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -107,3 +107,8 @@ static int aeApiPoll(aeApiState *state, aeFiredEvent *fired, aeFileEvent *events static char *aeApiName(void) { return "select"; } + +static int aeApiGetPollFd(aeEventLoop *eventLoop) { + UNUSED(eventLoop); + return -1; +} diff --git a/src/cluster.c b/src/cluster.c index a453e9f3cfe..1e393ecf822 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -367,6 +367,8 @@ migrateCachedSocket *migrateGetSocket(client *c, robj *host, robj *port, long ti /* Create the connection */ conn = connCreate(connTypeOfCluster()); + /* Set connection to high priority */ + connSetPriority(conn, CONN_PRIORITY_HIGH); if (connBlockingConnect(conn, objectGetVal(host), atoi(objectGetVal(port)), timeout) != C_OK) { addReplyError(c, "-IOERR error or timeout connecting to the client"); connClose(conn); diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index f21d1aa3c40..a819d5e33d6 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2052,6 +2052,8 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); + /* Set connection to high priority */ + connSetPriority(conn, CONN_PRIORITY_HIGH); /* Mark as cluster-owned before any TLS accept retries so generic * accept offload routing can safely avoid client assumptions. */ connSetOwnerKind(conn, CONN_OWNER_CLUSTER_LINK); @@ -6677,6 +6679,8 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long (*cluster_conn_attempts)--; clusterLink *link = createClusterLink(node); link->conn = connCreate(connTypeOfCluster()); + /* Set connection to high priority */ + connSetPriority(link->conn, CONN_PRIORITY_HIGH); connSetPrivateData(link->conn, link); connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); if (connConnect(link->conn, node->ip, node->cport, server.bind_source_addr, 0, clusterLinkConnectHandler) == @@ -7683,7 +7687,7 @@ sds clusterGenNodesDescription(client *c, int filter, int tls_primary) { /* Add to the output buffer of the given client the description of the given cluster link. * The description is a map with each entry being an attribute of the link. */ void addReplyClusterLinkDescription(client *c, clusterLink *link) { - addReplyMapLen(c, 6); + addReplyMapLen(c, 7); addReplyBulkCString(c, "direction"); addReplyBulkCString(c, link->inbound ? "from" : "to"); @@ -7715,6 +7719,9 @@ void addReplyClusterLinkDescription(client *c, clusterLink *link) { addReplyBulkCString(c, "send-buffer-used"); addReplyLongLong(c, link->send_msg_queue_mem); + + addReplyBulkCString(c, "qos"); + addReplyBulkCString(c, getConnectionPriorityName(connGetPriority(link->conn))); } /* Add to the output buffer of the given client an array of cluster link descriptions, diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index fcf7395a94e..67115b46a34 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -851,6 +851,10 @@ slotMigrationJob *createSlotImportJob(client *c, job->state = SLOT_IMPORT_WAIT_ACK; job->client = c; job->client->slot_migration_job = job; + if (c && c->conn) { + /* Upgrade connection to high priority */ + connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); + } /* We treat slot imports like primaries. Primaries are expected to have a * dedicated query buffer and allocated replication data. @@ -1379,6 +1383,8 @@ int connectSlotExportJob(slotMigrationJob *job) { port); job->conn = connCreate(connTypeOfReplication()); + /* Set connection to high priority */ + connSetPriority(job->conn, CONN_PRIORITY_HIGH); if (connConnect(job->conn, n->ip, port, server.bind_source_addr, 0, slotExportConnectHandler) == C_ERR) { return C_ERR; diff --git a/src/commands.def b/src/commands.def index d009b362500..3606e9f1161 100644 --- a/src/commands.def +++ b/src/commands.def @@ -1470,6 +1470,7 @@ commandHistory CLIENT_KILL_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","`ID` option accepts multiple IDs."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, +{"10.0.0","Added `H` flag to indicate high priority clients."}, }; #endif @@ -1565,6 +1566,7 @@ commandHistory CLIENT_LIST_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","Added filters USER, ADDR, LADDR, SKIPME, and MAXAGE."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, +{"10.0.0","Added `H` flag to indicate high priority clients."}, }; #endif @@ -1912,8 +1914,8 @@ struct COMMAND_STRUCT CLIENT_Subcommands[] = { {MAKE_CMD("id","Returns the unique client ID of the connection.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_ID_History,0,CLIENT_ID_Tips,0,clientIDCommand,2,CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_ID_Keyspecs,0,NULL,0)}, {MAKE_CMD("import-source","Marks this client as an import source when the server is in import mode.","O(1)","8.1.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_IMPORT_SOURCE_History,0,CLIENT_IMPORT_SOURCE_Tips,0,clientImportSourceCommand,3,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_IMPORT_SOURCE_Keyspecs,0,NULL,1),.args=CLIENT_IMPORT_SOURCE_Args}, {MAKE_CMD("info","Returns information about the connection.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_INFO_History,0,CLIENT_INFO_Tips,1,clientInfoCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_INFO_Keyspecs,0,NULL,0)}, -{MAKE_CMD("kill","Terminates open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_KILL_History,9,CLIENT_KILL_Tips,0,clientKillCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_KILL_Keyspecs,0,NULL,1),.args=CLIENT_KILL_Args}, -{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,9,CLIENT_LIST_Tips,1,clientListCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_LIST_Keyspecs,0,NULL,27),.args=CLIENT_LIST_Args}, +{MAKE_CMD("kill","Terminates open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_KILL_History,10,CLIENT_KILL_Tips,0,clientKillCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_KILL_Keyspecs,0,NULL,1),.args=CLIENT_KILL_Args}, +{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,10,CLIENT_LIST_Tips,1,clientListCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_LIST_Keyspecs,0,NULL,27),.args=CLIENT_LIST_Args}, {MAKE_CMD("no-evict","Sets the client eviction mode of the connection.","O(1)","7.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_EVICT_History,0,CLIENT_NO_EVICT_Tips,0,clientNoEvictCommand,3,CMD_ALLOW_BUSY|CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_NO_EVICT_Keyspecs,0,NULL,1),.args=CLIENT_NO_EVICT_Args}, {MAKE_CMD("no-touch","Controls whether commands sent by the client affect the LRU/LFU of accessed keys.","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_TOUCH_History,0,CLIENT_NO_TOUCH_Tips,0,clientNoTouchCommand,3,CMD_ALLOW_BUSY|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION|ACL_CATEGORY_SLOW,NULL,CLIENT_NO_TOUCH_Keyspecs,0,NULL,1),.args=CLIENT_NO_TOUCH_Args}, {MAKE_CMD("pause","Suspends commands processing.","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_PAUSE_History,1,CLIENT_PAUSE_Tips,0,clientPauseCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_ADMIN|ACL_CATEGORY_CONNECTION|ACL_CATEGORY_DANGEROUS|ACL_CATEGORY_SLOW,NULL,CLIENT_PAUSE_Keyspecs,0,NULL,2),.args=CLIENT_PAUSE_Args}, diff --git a/src/commands/client-kill.json b/src/commands/client-kill.json index e34ed463fc8..1ccd5ccb224 100644 --- a/src/commands/client-kill.json +++ b/src/commands/client-kill.json @@ -43,6 +43,10 @@ [ "9.0.0", "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." + ], + [ + "10.0.0", + "Added `H` flag to indicate high priority clients." ] ], "command_flags": [ diff --git a/src/commands/client-list.json b/src/commands/client-list.json index 1d8b0a93263..2b15e25fd9e 100644 --- a/src/commands/client-list.json +++ b/src/commands/client-list.json @@ -43,6 +43,10 @@ [ "9.0.0", "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." + ], + [ + "10.0.0", + "Added `H` flag to indicate high priority clients." ] ], "command_flags": [ diff --git a/src/connection.c b/src/connection.c index 819cdeb8741..fd12eec4a74 100644 --- a/src/connection.c +++ b/src/connection.c @@ -168,3 +168,55 @@ sds getListensInfoString(sds info) { return info; } +/* Upgrade connection priority */ +void connUpgradePriority(connection *conn, int priority) { + serverAssert(conn != NULL); + if (conn->priority == priority) return; + + int old_priority = conn->priority; + conn->priority = priority; + + if (conn->fd != -1) { + int mask = aeGetFileEvents(server.el, conn->fd); + if (mask != AE_NONE) { + /* Delete from wherever it is (AE will route delete correctly) */ + aeDeleteFileEvent(server.el, conn->fd, AE_READABLE | AE_WRITABLE); + + /* Re-create with new priority flag */ + int hp_flag = connGetAEPriorityFlag(conn); + if (mask & AE_READABLE) + aeCreateFileEvent(server.el, conn->fd, AE_READABLE | hp_flag, conn->type->ae_handler, conn); + if (mask & AE_WRITABLE) + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | hp_flag, conn->type->ae_handler, conn); + } + } + + serverLog(LL_DEBUG, "Connection fd %d qos upgraded from %s to %s", conn->fd, getConnectionPriorityName(old_priority), getConnectionPriorityName(priority)); +} + +/* Set connection priority */ +void connSetPriority(connection *conn, int priority) { + serverAssert(conn != NULL); + if (conn->priority == priority) return; + conn->priority = priority; +} + +/* Get connection priority */ +int connGetPriority(connection *conn) { + /* Always return CONN_PRIORITY_NORMAL if conn is NULL.*/ + return conn ? conn->priority : CONN_PRIORITY_NORMAL; +} + +/* Get AE priority flag for a connection */ +int connGetAEPriorityFlag(connection *conn) { + return (conn && conn->priority == CONN_PRIORITY_HIGH) ? AE_HIGH_PRIORITY : 0; +} + +/* Get connection priority name from priority value. */ +const char *getConnectionPriorityName(int priority) { + switch (priority) { + case CONN_PRIORITY_NORMAL: return "normal"; + case CONN_PRIORITY_HIGH: return "prioritized"; + default: return "normal"; + } +} diff --git a/src/connection.h b/src/connection.h index b3f94e14e34..e42028e8ca7 100644 --- a/src/connection.h +++ b/src/connection.h @@ -172,6 +172,26 @@ typedef struct ConnectionType { int (*is_closing)(connection *conn); // return 1 if connection is closed } ConnectionType; +/* Connection scheduling priority levels. + * + * High-priority connections receive preferential treatment across two subsystems: + * 1. HP Event loop (ae): Connections that are registered with hp_event_loop are + * processed ahead of normal priority events in the event polling loop. + * 2. I/O threads: Read and write tasks are dispatched to dedicated + * high-priority queues (io_shared_inbox[CONN_PRIORITY_HIGH] and io_shared_outbox[CONN_PRIORITY_HIGH]), + * are processed ahead of normal priority queues (io_shared_inbox[CONN_PRIORITY_NORMAL] + * and io_shared_outbox[CONN_PRIORITY_NORMAL]), when IO threads are enabled. + */ +typedef enum { + /* Normal priority connections - used for normal client connections */ + CONN_PRIORITY_NORMAL = 0, + /* High priority connections - used for critical internal communication such as + * cluster bus messages, slot migration, and replication streams */ + CONN_PRIORITY_HIGH, + /* Number of priority levels */ + CONN_PRIORITY_COUNT +} connPriority; + struct connection { ConnectionType *type; ConnectionState state; @@ -185,6 +205,7 @@ struct connection { ConnectionCallbackFunc conn_handler; ConnectionCallbackFunc write_handler; ConnectionCallbackFunc read_handler; + int priority; /* connPriority value */ }; #define CONFIG_BINDADDR_MAX 16 @@ -282,8 +303,7 @@ static inline int connWritev(connection *conn, const struct iovec *iov, int iovc * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. */ static inline int connRead(connection *conn, void *buf, size_t buf_len) { - int ret = conn->type->read(conn, buf, buf_len); - return ret; + return conn->type->read(conn, buf, buf_len); } /* Register a write handler, to be called when the connection is writable. @@ -537,6 +557,12 @@ static inline aeFileProc *connAcceptHandler(ConnectionType *ct) { /* Get Listeners information, note that caller should free the non-empty string */ sds getListensInfoString(sds info); +/* Connection qos(see connPriority enum). */ +void connSetPriority(connection *conn, int priority); +void connUpgradePriority(connection *conn, int priority); +int connGetPriority(connection *conn); +const char *getConnectionPriorityName(int priority); +int connGetAEPriorityFlag(connection *conn); int RedisRegisterConnectionTypeSocket(void); int RedisRegisterConnectionTypeUnix(void); int RedisRegisterConnectionTypeTLS(void); diff --git a/src/io_threads.c b/src/io_threads.c index e4b00487c1f..df56b1122e3 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -5,6 +5,7 @@ */ #include "io_threads.h" +#include "ae.h" #include "cluster.h" #include "cluster_legacy.h" #include "connhelpers.h" @@ -19,16 +20,16 @@ #define IO_SPSC_QUEUE_SIZE 4096 static _Thread_local int thread_id = 0; -static _Thread_local mpscTicket io_thread_ticket = {0}; +static _Thread_local mpscTicket io_thread_ticket[CONN_PRIORITY_COUNT] = {0}; /* Backlog of responses when io_shared_outbox is full. Should be rare. */ -static _Thread_local list *pending_io_responses = NULL; +static _Thread_local list *pending_io_responses[CONN_PRIORITY_COUNT] = {NULL, NULL}; static pthread_t io_threads[IO_THREADS_MAX_NUM] = {0}; static pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; static int cur_epoll_thread = 0; // Main -> IO: Shared Queue (Single Producer Multi Consumer) where all IO threads pull jobs from -static spmcQueue io_shared_inbox = {0}; +static spmcQueue io_shared_inbox[CONN_PRIORITY_COUNT] = {0}; // IO -> Main: Response Channel (Multi Producer Single Consumer) used by IO threads to send results back to main-thread -static mpscQueue io_shared_outbox = {0}; +static mpscQueue io_shared_outbox[CONN_PRIORITY_COUNT] = {0}; // Main -> IO (Thread-Specific) for tasks that must run on specific IO thread where IO threads check their private inbox before the shared queue static spscQueue io_private_inbox[IO_THREADS_MAX_NUM] = {0}; static size_t io_jobs_submitted; @@ -193,7 +194,7 @@ void IOThreadsAfterSleep(int numevents) { if (now - last_sample_time < IO_SAMPLE_RATE_MS) return; last_sample_time = now; - size_t q_size = spmcSize(&io_shared_inbox); + size_t q_size = spmcSize(&io_shared_inbox[CONN_PRIORITY_NORMAL]) + spmcSize(&io_shared_inbox[CONN_PRIORITY_HIGH]); spmc_size_sum += q_size; sample_count++; @@ -229,7 +230,7 @@ void IOThreadsAfterSleep(int numevents) { /* Don't suspend if work remains in the specific thread's queue... */ if (!spscIsEmpty(&io_private_inbox[tid])) return; /* ...or if we are dropping to 1 thread but the global queue still has work */ - if (target == 1 && !spmcIsEmpty(&io_shared_inbox)) return; + if (target == 1 && (!spmcIsEmpty(&io_shared_inbox[CONN_PRIORITY_NORMAL]) || !spmcIsEmpty(&io_shared_inbox[CONN_PRIORITY_HIGH]))) return; pthread_mutex_lock(&io_threads_mutex[tid]); server.active_io_threads_num--; @@ -246,11 +247,11 @@ void ioThreadPoll(aeEventLoop *el) { atomic_store_explicit(&server.io_poll_state, AE_IO_STATE_DONE, memory_order_release); } -static void flushPendingIOResponses(int blocking) { - if (!pending_io_responses) return; +static void flushPendingList(list **pending_list, mpscQueue *outbox, mpscTicket *ticket, int blocking) { + if (*pending_list == NULL) return; listIter li; listNode *ln; - listRewind(pending_io_responses, &li); + listRewind(*pending_list, &li); while ((ln = listNext(&li))) { void *job = listNodeValue(ln); @@ -258,21 +259,26 @@ static void flushPendingIOResponses(int blocking) { /* Try to enqueue. If blocking is set, retry until success. */ do { - pushed = mpscEnqueue(&io_shared_outbox, job, &io_thread_ticket); + pushed = mpscEnqueue(outbox, job, ticket); if (pushed || !blocking || server.crashed) break; /* On server crash we kill the IO threads, no point in sending back jobs to the main-thread. */ atomic_thread_fence(memory_order_acquire); } while (true); if (pushed) { - listDelNode(pending_io_responses, ln); + listDelNode(*pending_list, ln); } else { return; } } /* List is fully drained */ - listRelease(pending_io_responses); - pending_io_responses = NULL; + listRelease(*pending_list); + *pending_list = NULL; +} + +static void flushPendingIOResponses(int blocking) { + flushPendingList(&pending_io_responses[CONN_PRIORITY_HIGH], &io_shared_outbox[CONN_PRIORITY_HIGH], &io_thread_ticket[CONN_PRIORITY_HIGH], blocking); + flushPendingList(&pending_io_responses[CONN_PRIORITY_NORMAL], &io_shared_outbox[CONN_PRIORITY_NORMAL], &io_thread_ticket[CONN_PRIORITY_NORMAL], blocking); } /* Define a cleanup function that will clean all thread resources */ @@ -286,6 +292,41 @@ void cleanupThreadResources(void *dummy) { freeSharedQueryBuf(); } +static inline void processTaggedSPMCJob(void *tagged_job) { + void *data; + int type; + untagJob(tagged_job, &data, &type); + + switch (type) { + case JOB_REQ_READ_CLIENT: + ioThreadReadQueryFromClient((client *)data); + break; + case JOB_REQ_WRITE_CLIENT: + ioThreadWriteToClient((client *)data); + break; + case JOB_REQ_FREE_OBJ: + decrRefCount(data); + break; + case JOB_REQ_ACCEPT: + ioThreadAccept((client *)data); + break; + case JOB_REQ_POLL: + ioThreadPoll((aeEventLoop *)data); + break; + case JOB_REQ_CLUSTER_READ: + clusterReadJob((clusterLink *)data); + break; + case JOB_REQ_CLUSTER_WRITE: + clusterWriteJob((clusterLink *)data); + break; + case JOB_REQ_CLUSTER_ACCEPT: + clusterAcceptJob((connection *)data); + break; + default: + serverPanic("Invalid SPMC job type: %d", type); + } +} + static void *IOThreadMain(void *myid) { /* The ID is the thread ID number (from 1 to server.io_threads_num-1). ID 0 is the main thread. */ long id = (long)myid; @@ -336,40 +377,14 @@ static void *IOThreadMain(void *myid) { /* PRIORITY 2: Shared Global Queue (SPMC) * Only checked after SPSC is drained. */ - void *tagged_job = spmcDequeue(&io_shared_inbox); - if (tagged_job) { - void *data; - int type; - untagJob(tagged_job, &data, &type); - - switch (type) { - case JOB_REQ_READ_CLIENT: - ioThreadReadQueryFromClient((client *)data); - break; - case JOB_REQ_WRITE_CLIENT: - ioThreadWriteToClient((client *)data); - break; - case JOB_REQ_FREE_OBJ: - decrRefCount(data); - break; - case JOB_REQ_ACCEPT: - ioThreadAccept((client *)data); - break; - case JOB_REQ_POLL: - ioThreadPoll((aeEventLoop *)data); - break; - case JOB_REQ_CLUSTER_READ: - clusterReadJob((clusterLink *)data); - break; - case JOB_REQ_CLUSTER_WRITE: - clusterWriteJob((clusterLink *)data); - break; - case JOB_REQ_CLUSTER_ACCEPT: - clusterAcceptJob((connection *)data); - break; - default: - serverPanic("Invalid SPMC job type: %d", type); - } + void *tagged_job; + if ((tagged_job = spmcDequeue(&io_shared_inbox[CONN_PRIORITY_HIGH])) != NULL) { + processTaggedSPMCJob(tagged_job); + processed++; + } + + if ((tagged_job = spmcDequeue(&io_shared_inbox[CONN_PRIORITY_NORMAL])) != NULL) { + processTaggedSPMCJob(tagged_job); processed++; } @@ -379,7 +394,11 @@ static void *IOThreadMain(void *myid) { /* If both queues were empty (no processing done), wait for signal. */ if (processed == 0) { - if (unlikely(pending_io_responses)) { + int has_pending = 0; + for (int p = 0; p < CONN_PRIORITY_COUNT; p++) { + if (pending_io_responses[p]) has_pending = 1; + } + if (unlikely(has_pending)) { flushPendingIOResponses(0); } else { /* If it is locked. We should block until main thread unlocks it. */ @@ -466,7 +485,7 @@ int updateIOThreads(const char **err) { * in that state, we will deadlock (Main thread waits for worker, Worker waits for queue space). */ size_t pending = getPendingIOResponsesCount(); - if (pending > io_shared_outbox.queue_size) { + if (pending > io_shared_outbox[CONN_PRIORITY_NORMAL].queue_size || pending > io_shared_outbox[CONN_PRIORITY_HIGH].queue_size) { if (err) *err = "Can't update IO threads under load, try again later"; return 0; } @@ -505,8 +524,10 @@ void initIOThreads(int prev_threads_num) { server.active_io_threads_num = 1; /* We start with threads not active. */ server.io_poll_state = AE_IO_STATE_NONE; server.io_ae_fired_events = 0; - spmcInit(&io_shared_inbox, IO_SPMC_QUEUE_SIZE); - mpscInit(&io_shared_outbox, IO_MPSC_QUEUE_SIZE); + for (int p = 0; p < CONN_PRIORITY_COUNT; p++) { + spmcInit(&io_shared_inbox[p], IO_SPMC_QUEUE_SIZE); + mpscInit(&io_shared_outbox[p], IO_MPSC_QUEUE_SIZE); + } io_jobs_submitted = 0; atomic_init(&io_jobs_finished, 0); cluster_io_pending_responses = 0; @@ -590,7 +611,8 @@ int trySendReadToIOThreads(client *c) { c->io_read_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { + int qidx = connGetPriority(c->conn); + if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { c->read_flags = 0; c->io_read_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); @@ -651,7 +673,9 @@ int trySendWriteToIOThreads(client *c) { c->io_write_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_WRITE_CLIENT); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) { + + int qidx = connGetPriority(c->conn); + if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], job) == false)) { c->io_write_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); c->write_flags = 0; @@ -933,7 +957,7 @@ int tryOffloadFreeObjToIOThreads(robj *obj) { if (obj->encoding != OBJ_ENCODING_RAW || obj->type != OBJ_STRING) return C_ERR; void *job = tagJob(obj, JOB_REQ_FREE_OBJ); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) return C_ERR; + if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], job) == false)) return C_ERR; io_jobs_submitted++; server.stat_io_freed_objects++; return C_OK; @@ -980,7 +1004,7 @@ void trySendPollJobToIOThreads(void) { /* Use SPMC to minimize polling overhead. At high thread counts, use private SPSC queues for lower latency. */ if (server.active_io_threads_num <= 9) { - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(server.el, JOB_REQ_POLL)) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], tagJob(server.el, JOB_REQ_POLL)) == false)) { server.io_poll_state = AE_IO_STATE_NONE; aeSetPollProtect(server.el, 0); return; @@ -999,17 +1023,17 @@ void trySendPollJobToIOThreads(void) { io_jobs_submitted++; } -void sendToMainThread(void *data, int type) { - if (unlikely(pending_io_responses)) { - flushPendingIOResponses(0); +void sendToMainThread(client *c, int type) { + int qidx = connGetPriority(c->conn); + if (unlikely(pending_io_responses[qidx])) { + flushPendingList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); } - void *job = tagJob(data, type); - if (unlikely(pending_io_responses || !mpscEnqueue(&io_shared_outbox, job, &io_thread_ticket))) { - /* Failed to push new job: initialize list if needed and save job */ - if (pending_io_responses == NULL) { - pending_io_responses = listCreate(); + void *job = tagJob(c, type); + if (unlikely(pending_io_responses[qidx] || !mpscEnqueue(&io_shared_outbox[qidx], job, &io_thread_ticket[qidx]))) { + if (pending_io_responses[qidx] == NULL) { + pending_io_responses[qidx] = listCreate(); } - listAddNodeTail(pending_io_responses, job); + listAddNodeTail(pending_io_responses[qidx], job); } } @@ -1061,7 +1085,7 @@ int trySendAcceptToIOThreads(connection *conn) { connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_ACCEPT); - if (unlikely(spmcEnqueue(&io_shared_inbox, job) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], job) == false)) { c->io_read_state = CLIENT_IDLE; c->flag.pending_read = 0; connSetPostponeUpdateState(c->conn, 0); @@ -1122,71 +1146,45 @@ static void handleWriteJobs(client **write_jobs, int write_count) { } } -int processIOThreadsResponses(void) { - /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ - - /* Quick check if any pending operations exist */ - if (getPendingIOResponsesCount() == 0) return 0; - - int total_processed = 0; +static int processOutboxBatch(mpscQueue *outbox) { void *jobs[JOB_BATCH_SIZE]; client *read_jobs[JOB_BATCH_SIZE]; client *write_jobs[JOB_BATCH_SIZE]; + int received_responses = 0; + int read_count = 0; + int write_count = 0; - /* Loop until we consume all pending jobs */ - while (1) { - int received_responses = 0; - int dequeued_count = 0; - int read_count = 0; - int write_count = 0; - - /* Try to dequeue JOB_BATCH_SIZE */ - while (received_responses < JOB_BATCH_SIZE) { - dequeued_count = mpscDequeueBatch(&io_shared_outbox, jobs, JOB_BATCH_SIZE - received_responses); + /* Try to dequeue JOB_BATCH_SIZE */ + while (received_responses < JOB_BATCH_SIZE) { + dequeued_count = mpscDequeueBatch(&io_shared_outbox, jobs, JOB_BATCH_SIZE - received_responses); - /* Stop if we can't get more jobs from the queue. */ - if (dequeued_count == 0) break; + /* Stop if we can't get more jobs from the queue. */ + if (dequeued_count == 0) break; - received_responses += dequeued_count; - total_processed += dequeued_count; + received_responses += dequeued_count; + total_processed += dequeued_count; - for (int i = 0; i < dequeued_count; i++) { - void *data; - int job_type; - untagJob(jobs[i], &data, &job_type); - if (job_type == JOB_RES_READ_CLIENT) { - client *c = (client *)data; - serverAssert(c->io_read_state == CLIENT_COMPLETED_IO); - read_jobs[read_count++] = c; - } else if (job_type == JOB_RES_WRITE_CLIENT) { - client *c = (client *)data; - serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); - write_jobs[write_count++] = c; - } else if (job_type == JOB_RES_CLUSTER_READ) { - serverAssert(cluster_io_pending_responses > 0); - cluster_io_pending_responses--; - server.stat_cluster_threaded_reads_processed++; - clusterHandleReadCompletion((struct clusterLink *)data); - } else if (job_type == JOB_RES_CLUSTER_WRITE) { - serverAssert(cluster_io_pending_responses > 0); - cluster_io_pending_responses--; - server.stat_cluster_threaded_writes_processed++; - clusterHandleWriteCompletion((struct clusterLink *)data); - } else if (job_type == JOB_RES_CLUSTER_ACCEPT) { - serverAssert(cluster_io_pending_responses > 0); - cluster_io_pending_responses--; - server.stat_cluster_threaded_accepts_processed++; - clusterHandleAcceptCompletion((connection *)data); - } else { - serverPanic("Unknown job type %d", job_type); - } + for (int i = 0; i < dequeued_count; i++) { + void *data; + int job_type; + untagJob(jobs[i], &data, &job_type); + client *c = (client *)data; + if (job_type == JOB_RES_READ_CLIENT) { + serverAssert(c->io_read_state == CLIENT_COMPLETED_IO); + read_jobs[read_count++] = c; + } else if (job_type == JOB_RES_WRITE_CLIENT) { + serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); + write_jobs[write_count++] = c; + } else { + serverPanic("Unknown job type %d", job_type); } } + } - if (read_count) handleReadJobs(read_jobs, read_count); - if (write_count) handleWriteJobs(write_jobs, write_count); + if (read_count) handleReadJobs(read_jobs, read_count); + if (write_count) handleWriteJobs(write_jobs, write_count); - /* If the queue was empty at the last try - don't try again */ - if (dequeued_count == 0) return total_processed; - } + /* If the queue was empty at the last try - don't try again */ + if (dequeued_count == 0) return total_processed; +} } diff --git a/src/io_threads.h b/src/io_threads.h index 06d4cf889cb..88e440d29c2 100644 --- a/src/io_threads.h +++ b/src/io_threads.h @@ -66,6 +66,6 @@ long long getIOThreadActiveTimeMicroseconds(int id); int clientHasPendingIO(struct client *c); int processIOThreadsResponses(void); int getCurTid(void); -void sendToMainThread(void *data, int type); +void sendToMainThread(client *c, int type); #endif /* IO_THREADS_H */ diff --git a/src/networking.c b/src/networking.c index b827d901e62..8c240693237 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1924,6 +1924,8 @@ void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip) { connClose(conn); return; } + /* Default priority is normal */ + connSetPriority(conn, CONN_PRIORITY_NORMAL); /* Create connection and client */ if ((c = createClient(conn)) == NULL) { @@ -4511,7 +4513,7 @@ int isClientConnIpV6(client *c) { * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (!server.crashed) waitForClientIO(client); - char flags[18], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; + char flags[32], events[3], capa[9], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flag.replica) { @@ -4540,6 +4542,7 @@ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (client->flag.import_source) *p++ = 'I'; if (client->slot_migration_job && isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'i'; if (client->slot_migration_job && !isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'E'; + if (client->conn && connGetPriority(client->conn) == CONN_PRIORITY_HIGH) *p++ = 'H'; if (p == flags) *p++ = 'N'; *p++ = '\0'; @@ -4702,6 +4705,13 @@ int clientSetName(client *c, robj *name, const char **err) { if (c->name) decrRefCount(c->name); c->name = name; incrRefCount(name); + + if (name && len >= 9 && strncmp(objectGetVal(name), "sentinel-", 9) == 0) { + if (c->conn) { + /* Upgrade connection priority to high for sentinel */ + connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); + } + } return C_OK; } @@ -5067,6 +5077,7 @@ static int validateClientFlagFilter(sds flag_filter) { case 'I': case 'i': case 'E': + case 'H': case 'N': /* Valid flag, do nothing. */ break; @@ -5230,6 +5241,9 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { case 'E': /* Slot migration export flag */ if (!c->slot_migration_job || isImportSlotMigrationJob(c->slot_migration_job)) return 0; break; + case 'H': /* High priority connection */ + if (!c->conn || connGetPriority(c->conn) != CONN_PRIORITY_HIGH) return 0; + break; case 'N': /* Check for no flags */ if (c->flag.replica || c->flag.primary || c->flag.pubsub || c->flag.multi || c->flag.blocked || c->flag.tracking || @@ -5238,7 +5252,8 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { c->flag.unblocked || c->flag.close_asap || c->flag.unix_socket || c->flag.readonly || c->flag.no_evict || c->flag.no_touch || c->flag.throttled || - c->flag.import_source || c->slot_migration_job) { + c->flag.import_source || c->slot_migration_job || + (c->conn && connGetPriority(c->conn) == CONN_PRIORITY_HIGH)) { return 0; } break; diff --git a/src/replication.c b/src/replication.c index 78eb153d712..7f921fae695 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1171,6 +1171,10 @@ void syncCommand(client *c) { } serverLog(LL_NOTICE, "Replica %s asks for synchronization", replicationGetReplicaName(c)); + if (c->conn) { + /* Upgrade replica connection to high priority */ + connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); + } /* Try a partial resynchronization if this is a PSYNC command. * If it fails, we continue with usual full resynchronization, however @@ -4385,6 +4389,8 @@ void syncWithPrimary(connection *conn) { if (psync_result == PSYNC_FULLRESYNC_DUAL_CHANNEL) { /* Create RDB connection */ server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); + /* Set connection to high priority */ + connSetPriority(server.repl_rdb_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { dualChannelServerLog(LL_WARNING, "Unable to connect to Primary: %s", @@ -4435,6 +4441,8 @@ void syncWithPrimary(connection *conn) { int connectWithPrimary(void) { server.repl_transfer_s = connCreate(connTypeOfReplication()); + /* Set connection to high priority */ + connSetPriority(server.repl_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { serverLog(LL_WARNING, "Unable to connect to PRIMARY: %s", connGetLastError(server.repl_transfer_s)); diff --git a/src/server.c b/src/server.c index 0b71a99dce6..e9fb7ac4733 100644 --- a/src/server.c +++ b/src/server.c @@ -3095,11 +3095,14 @@ void initServer(void) { const char *clk_msg = monotonicInit(); serverLog(LL_NOTICE, "monotonic clock: %s", clk_msg); server.el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - if (server.el == NULL) { + /*Create high priority event loop*/ + aeEventLoop *hp_el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); + if (server.el == NULL || hp_el == NULL) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - + /*Link high priority event loop with main event loop*/ + aeLinkHighPriorityEventLoop(server.el, hp_el); server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); diff --git a/src/server.h b/src/server.h index 12728df045a..02218f04d2f 100644 --- a/src/server.h +++ b/src/server.h @@ -1809,32 +1809,32 @@ struct valkeyServer { hashtable *commands; /* Command table */ hashtable *orig_commands; /* Command table before command renaming. */ sds command_response_cache[RESP_CACHE_INDEX_MAX]; /* Cached COMMAND response: [0]=RESP2, [1]=RESP3 */ - aeEventLoop *el; - _Atomic(AeIoState) io_poll_state; /* Indicates the state of the IO polling. */ - int io_ae_fired_events; /* Number of poll events received by the IO thread. */ - rax *errors; /* Errors table */ - volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */ - mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */ - int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */ - int shutdown_flags; /* Flags passed to prepareForShutdown(). */ - int activerehashing; /* Incremental rehash in serverCron() */ - int active_defrag_cpu_percent; /* Current desired CPU percentage for active defrag */ - char *pidfile; /* PID file path */ - int arch_bits; /* 32 or 64 depending on sizeof(long) */ - int cronloops; /* Number of times the cron function run */ - char runid[CONFIG_RUN_ID_SIZE + 1]; /* ID always different at every exec. */ - int sentinel_mode; /* True if this instance is a Sentinel. */ - size_t initial_memory_usage; /* Bytes used after initialization. */ - int always_show_logo; /* Show logo even for non-stdout logging. */ - int in_exec; /* Are we inside EXEC? */ - int in_call; /* Nesting level within the call() function. */ - int busy_module_yield_flags; /* Are we inside a busy module? (triggered by RM_Yield). see BUSY_MODULE_YIELD_ flags. */ - const char *busy_module_yield_reply; /* When non-null, we are inside RM_Yield. */ - char *ignore_warnings; /* Config: warnings that should be ignored. */ - int client_pause_in_transaction; /* Was a client pause executed during this Exec? */ - int server_del_keys_in_slot; /* The server is deleting the keys in the dirty slot. */ - int thp_enabled; /* If true, THP is enabled. */ - size_t page_size; /* The page size of OS. */ + aeEventLoop *el; /* Main event loop */ + _Atomic(AeIoState) io_poll_state; /* Indicates the state of the IO polling. */ + int io_ae_fired_events; /* Number of poll events received by the IO thread. */ + rax *errors; /* Errors table */ + volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */ + mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */ + int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */ + int shutdown_flags; /* Flags passed to prepareForShutdown(). */ + int activerehashing; /* Incremental rehash in serverCron() */ + int active_defrag_cpu_percent; /* Current desired CPU percentage for active defrag */ + char *pidfile; /* PID file path */ + int arch_bits; /* 32 or 64 depending on sizeof(long) */ + int cronloops; /* Number of times the cron function run */ + char runid[CONFIG_RUN_ID_SIZE + 1]; /* ID always different at every exec. */ + int sentinel_mode; /* True if this instance is a Sentinel. */ + size_t initial_memory_usage; /* Bytes used after initialization. */ + int always_show_logo; /* Show logo even for non-stdout logging. */ + int in_exec; /* Are we inside EXEC? */ + int in_call; /* Nesting level within the call() function. */ + int busy_module_yield_flags; /* Are we inside a busy module? (triggered by RM_Yield). see BUSY_MODULE_YIELD_ flags. */ + const char *busy_module_yield_reply; /* When non-null, we are inside RM_Yield. */ + char *ignore_warnings; /* Config: warnings that should be ignored. */ + int client_pause_in_transaction; /* Was a client pause executed during this Exec? */ + int server_del_keys_in_slot; /* The server is deleting the keys in the dirty slot. */ + int thp_enabled; /* If true, THP is enabled. */ + size_t page_size; /* The page size of OS. */ /* Modules */ dict *moduleapi; /* Exported core APIs dictionary for modules. */ dict *sharedapi; /* Like moduleapi but containing the APIs that @@ -2054,9 +2054,9 @@ struct valkeyServer { double *latency_tracking_info_percentiles; /* Extended latency tracking info output percentile list configuration. */ int latency_tracking_info_percentiles_len; unsigned int max_new_tls_conns_per_cycle; /* The maximum number of tls connections that will be accepted during each - invocation of the event loop. */ + invocation of the event loop. */ unsigned int max_new_conns_per_cycle; /* The maximum number of tcp connections that will be accepted during each - invocation of the event loop. */ + invocation of the event loop. */ /* AOF persistence */ int aof_enabled; /* AOF configuration */ int aof_state; /* AOF_(ON|OFF|WAIT_REWRITE) */ @@ -3104,6 +3104,7 @@ int closeClientOnOutputBufferLimitReached(client *c, int async); int getClientType(client *c); int getClientTypeByName(char *name); char *getClientTypeName(int client_class); +const char *getClientQoSName(int qos); void flushReplicasOutputBuffers(void); void disconnectReplicas(void); void evictClients(void); diff --git a/src/socket.c b/src/socket.c index f3d2d3bee5a..6a1370196dd 100644 --- a/src/socket.c +++ b/src/socket.c @@ -114,18 +114,23 @@ static int connSocketConnect(connection *conn, ConnectionCallbackFunc connect_handler) { int fd = anetTcpNonBlockBestEffortBindConnect(NULL, addr, port, src_addr, multipath); if (fd == -1) { - conn->state = CONN_STATE_ERROR; - conn->last_errno = errno; - return C_ERR; + goto error; } conn->fd = fd; conn->state = CONN_STATE_CONNECTING; conn->conn_handler = connect_handler; - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, conn->type->ae_handler, conn); + int priority_flag = connGetAEPriorityFlag(conn); + if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) + goto error; return C_OK; + +error: + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; } /* ------ Pure socket connections ------- */ @@ -245,9 +250,10 @@ static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc fu conn->flags |= CONN_FLAG_WRITE_BARRIER; else conn->flags &= ~CONN_FLAG_WRITE_BARRIER; + int priority_flag = connGetAEPriorityFlag(conn); if (!conn->write_handler) aeDeleteFileEvent(server.el, conn->fd, AE_WRITABLE); - else if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, conn->type->ae_handler, conn) == AE_ERR) + else if (aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) return C_ERR; return C_OK; } @@ -259,9 +265,10 @@ static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc fun if (func == conn->read_handler) return C_OK; conn->read_handler = func; + int priority_flag = connGetAEPriorityFlag(conn); if (!conn->read_handler) aeDeleteFileEvent(server.el, conn->fd, AE_READABLE); - else if (aeCreateFileEvent(server.el, conn->fd, AE_READABLE, conn->type->ae_handler, conn) == AE_ERR) + else if (aeCreateFileEvent(server.el, conn->fd, AE_READABLE | priority_flag, conn->type->ae_handler, conn) == AE_ERR) return C_ERR; return C_OK; } @@ -284,7 +291,8 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD conn->state = CONN_STATE_CONNECTED; } - if (!conn->write_handler) aeDeleteFileEvent(server.el, conn->fd, AE_WRITABLE); + if (!conn->write_handler) + aeDeleteFileEvent(el, conn->fd, AE_WRITABLE); if (!callHandler(conn, conn->conn_handler)) return; conn->conn_handler = NULL; diff --git a/src/tls.c b/src/tls.c index 197d1b47e35..490dff52534 100644 --- a/src/tls.c +++ b/src/tls.c @@ -1198,14 +1198,15 @@ static int updateStateAfterSSLIO(tls_connection *conn, int ret_value, int update } static void registerSSLEvent(tls_connection *conn) { + int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); if (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ) { if (mask & AE_WRITABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); - if (!(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, tlsEventHandler, conn); + if (!(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); } else if (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE) { if (mask & AE_READABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (!(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, tlsEventHandler, conn); + if (!(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); } else { serverAssert(0); } @@ -1247,16 +1248,15 @@ void updateSSLPendingFlag(tls_connection *conn) { static void updateSSLEvent(tls_connection *conn) { if (conn->flags & TLS_CONN_FLAG_POSTPONE_UPDATE_STATE) return; + int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); int need_read = conn->c.read_handler || (conn->c.write_handler && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)); int need_write = conn->c.write_handler || (conn->c.read_handler && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)); - if (need_read && !(mask & AE_READABLE)) - aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, tlsEventHandler, conn); + if (need_read && !(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); if (!need_read && (mask & AE_READABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (need_write && !(mask & AE_WRITABLE)) - aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, tlsEventHandler, conn); + if (need_write && !(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); if (!need_write && (mask & AE_WRITABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); } diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp new file mode 100644 index 00000000000..39e057faea7 --- /dev/null +++ b/src/unit/test_socket_prioritization.cpp @@ -0,0 +1,771 @@ +/* + * Copyright (c) Valkey Contributors + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "generated_wrappers.hpp" + +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#define _Static_assert static_assert +extern "C" { +#include "ae.h" +#include "monotonic.h" +#include "connection.h" +#include "io_threads.h" +#include "server.h" +} +#undef _Static_assert + + +/* Static tracking for preemption test callbacks */ +static int g_execution_order[1024]; +static int g_execution_count = 0; + +static monotime g_mock_time_offset = 0; +static monotime (*g_orig_getMonotonicUs)(void) = NULL; +static monotime mockGetMonotonicUs(void) { + return g_orig_getMonotonicUs() + g_mock_time_offset; +} + +static void setupMockClock(void) { + if (g_orig_getMonotonicUs != NULL) return; + g_orig_getMonotonicUs = getMonotonicUs; + getMonotonicUs = mockGetMonotonicUs; + g_mock_time_offset = 0; +} + +static void restoreMockClock(void) { + if (g_orig_getMonotonicUs) { + getMonotonicUs = g_orig_getMonotonicUs; + g_orig_getMonotonicUs = NULL; + } +} + +static void advanceMockTime(uint64_t us) { + g_mock_time_offset += us; +} + +static void testNormalEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = fd; + } + /* Read 1 byte from pipe to clear the event */ + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static void testHighPriorityEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + if (g_execution_count < 1024) { + /* Use 999 to identify high priority execution */ + g_execution_order[g_execution_count++] = 999; + } + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static int g_hp_pipe2_write_fd = -1; +static int g_normal_cb_count = 0; +static void testNormalEventCallbackMaskCheck(aeEventLoop *el, int fd, void *privdata, int mask) { + testNormalEventCallback(el, fd, privdata, mask); + g_normal_cb_count++; + /* Sleep > 1000 us on 3rd normal event (j = 2) and trigger 2nd HP pipe so (itr_cnt & 3) == 0 check at j = 4 polls hp_el */ + if (g_normal_cb_count == 3 && g_hp_pipe2_write_fd != -1) { + usleep(1500); + char c = 'x'; + if (write(g_hp_pipe2_write_fd, &c, 1) < 0) { + } + } +} + +static void testLevelTriggeredCallback(aeEventLoop *el, int fd, void *privdata, int mask) { + UNUSED(el); + UNUSED(mask); + if (privdata) { + int *serve_counts = (int *)privdata; + serve_counts[0]++; + } + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = fd; + } + /* Read only 1 byte per execution to test level-triggered behavior (if more data in buffer, fd remains readable) */ + char buf[1]; + if (read(fd, buf, 1) < 0) { + /* Ignore read error */ + } +} + +static void dummyConnectionHandler(struct connection *conn) { + UNUSED(conn); +} + +class SocketPrioritizationTest : public ::testing::Test { + protected: + void SetUp() override { + monotonicInit(); + setupMockClock(); + /* Initialize minimal server fields needed to prevent crashes in logging/connection layer */ + server.logfile = strdup(""); + server.syslog_enabled = 0; + + /* Initialize connection types registry ONCE */ + static bool conn_types_initialized = false; + if (!conn_types_initialized) { + connTypeInitialize(); + conn_types_initialized = true; + } + + server.el = aeCreateEventLoop(1024); + aeEventLoop *hp_el = aeCreateEventLoop(1024); + if (server.el && hp_el) { + aeLinkHighPriorityEventLoop(server.el, hp_el); + } + g_execution_count = 0; + } + + void TearDown() override { + if (server.el) { + aeDeleteEventLoop(server.el); + server.el = NULL; + } + if (server.logfile) { + free(server.logfile); + server.logfile = NULL; + } + restoreMockClock(); + } +}; + +class SocketPrioritizationConnTest : public SocketPrioritizationTest, public ::testing::WithParamInterface {}; + +TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { + ASSERT_NE(server.el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->hp_event_loop, (aeEventLoop *)NULL); + EXPECT_EQ(server.el->hp_event_loop, server.el->hp_event_loop); + EXPECT_EQ(server.el->hp_event_loop->hp_event_loop, (aeEventLoop *)NULL); +} + +TEST_P(SocketPrioritizationConnTest, ConnectionPriorityMetadataAndHelpers) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + + /* Default priority should be normal */ + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + + /* Update metadata when fd is not yet set (-1) */ + connSetPriority(conn, CONN_PRIORITY_HIGH); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + + connSetPriority(conn, CONN_PRIORITY_NORMAL); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + + conn->state = CONN_STATE_NONE; + connClose(conn); +} + +TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + int fds[2]; + int ret = pipe(fds); + ASSERT_EQ(ret, 0); + int read_fd = fds[0]; + int write_fd = fds[1]; + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = read_fd; + + /* Set a read handler while priority is normal */ + ret = connSetReadHandler(conn, dummyConnectionHandler); + EXPECT_EQ(ret, C_OK); + + /* Event should exist in normal loop, but not in high priority loop */ + EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + + /* Dynamically upgrade connection to high priority */ + connUpgradePriority(conn, CONN_PRIORITY_HIGH); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + int events = aeGetFileEvents(server.el, read_fd); + EXPECT_NE(events & AE_READABLE, 0); + EXPECT_NE(events & AE_HIGH_PRIORITY, 0); + EXPECT_NE(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + + /* Dynamically downgrade connection back to normal priority */ + connUpgradePriority(conn, CONN_PRIORITY_NORMAL); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + + conn->state = CONN_STATE_NONE; + connClose(conn); + close(write_fd); +} + +TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByHighPriorityLoop) { + int normal_pipes[2][2]; + int hp_pipe[2]; + int i; + int ret; + + for (i = 0; i < 2; i++) { + ret = pipe(normal_pipes[i]); + ASSERT_EQ(ret, 0); + /* Set non-blocking */ + fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); + } + ret = pipe(hp_pipe); + ASSERT_EQ(ret, 0); + fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + + /* Register normal events on server.el */ + ret = aeCreateFileEvent(server.el, normal_pipes[0][0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + ret = aeCreateFileEvent(server.el, normal_pipes[1][0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Register high priority event on server.el->hp_event_loop */ + ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE, testHighPriorityEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Write 1 byte to all pipes so they fire */ + char c = 'x'; + if (write(normal_pipes[0][1], &c, 1) < 0) { + } + if (write(normal_pipes[1][1], &c, 1) < 0) { + } + if (write(hp_pipe[1], &c, 1) < 0) { + } + + /* Ensure more than HP_EVENTS_POLL_CHECK_EVERY_US (1000 us) has elapsed for hp_event_loop->last_poll_us */ + advanceMockTime(1000000); + + g_execution_count = 0; + /* Process events on main event loop. At iteration j = 0, PROCESS_HP_EVENTS_IF_NEEDED will trigger + * polling of server.el->hp_event_loop and execute testHighPriorityEventCallback before processing the normal events! */ + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + /* Verify that high priority event (id 999) executed first due to preemption */ + ASSERT_GE(g_execution_count, 1); + EXPECT_EQ(g_execution_order[0], 999); + + /* Cleanup events and file descriptors */ + aeDeleteFileEvent(server.el, normal_pipes[0][0], AE_READABLE); + aeDeleteFileEvent(server.el, normal_pipes[1][0], AE_READABLE); + aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE); + + for (i = 0; i < 2; i++) { + close(normal_pipes[i][0]); + close(normal_pipes[i][1]); + } + close(hp_pipe[0]); + close(hp_pipe[1]); +} + +TEST_F(SocketPrioritizationTest, QoSNames) { + EXPECT_STREQ(getConnectionPriorityName(CONN_PRIORITY_NORMAL), "normal"); + EXPECT_STREQ(getConnectionPriorityName(CONN_PRIORITY_HIGH), "prioritized"); +} + +static void testHighPriorityWriteCallback(struct connection *conn) { + if (g_execution_count < 1024) { + g_execution_order[g_execution_count++] = 999; + } + connSetWriteHandler(conn, NULL); +} + +TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { + ConnectionType *ct = connectionByType(GetParam()); + if (ct == NULL) return; + int normal_pipe[2]; + int hp_pipe[2]; + int ret; + + ret = pipe(normal_pipe); + ASSERT_EQ(ret, 0); + fcntl(normal_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipe[1], F_SETFL, O_NONBLOCK); + + ret = pipe(hp_pipe); + ASSERT_EQ(ret, 0); + fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + + connection *hp_conn = connCreate(ct); + ASSERT_NE(hp_conn, (connection *)NULL); + hp_conn->state = CONN_STATE_CONNECTED; + hp_conn->fd = hp_pipe[1]; + connSetPriority(hp_conn, CONN_PRIORITY_HIGH); + + /* Register normal read event on server.el */ + ret = aeCreateFileEvent(server.el, normal_pipe[0], AE_READABLE, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Register high priority write event on server.el->hp_event_loop via connection wrapper */ + ret = connSetWriteHandler(hp_conn, testHighPriorityWriteCallback); + EXPECT_EQ(ret, C_OK); + + char c = 'w'; + if (write(normal_pipe[1], &c, 1) < 0) { + } + + advanceMockTime(1000000); + g_execution_count = 0; + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + ASSERT_GE(g_execution_count, 1); + EXPECT_EQ(g_execution_order[0], 999); + + aeDeleteFileEvent(server.el, normal_pipe[0], AE_READABLE); + hp_conn->state = CONN_STATE_NONE; + connClose(hp_conn); + close(normal_pipe[0]); + close(normal_pipe[1]); + close(hp_pipe[0]); +} + +TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { + /* Verify that PROCESS_HP_EVENTS_IF_NEEDED checks hp_el on iterations where (itr_cnt & 3) == 0 */ + int normal_pipes[5][2]; + int hp_pipe[2]; + int hp_pipe2[2]; + int i, ret; + + for (i = 0; i < 5; i++) { + ret = pipe(normal_pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); + } + ret = pipe(hp_pipe); + ASSERT_EQ(ret, 0); + fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + + ret = pipe(hp_pipe2); + ASSERT_EQ(ret, 0); + fcntl(hp_pipe2[0], F_SETFL, O_NONBLOCK); + fcntl(hp_pipe2[1], F_SETFL, O_NONBLOCK); + + for (i = 0; i < 5; i++) { + ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackMaskCheck, NULL); + EXPECT_EQ(ret, AE_OK); + } + ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE, testHighPriorityEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe2[0], AE_READABLE, testHighPriorityEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + char c = 'z'; + for (i = 0; i < 5; i++) { + if (write(normal_pipes[i][1], &c, 1) < 0) { + } + } + if (write(hp_pipe[1], &c, 1) < 0) { + } + + g_hp_pipe2_write_fd = hp_pipe2[1]; + g_normal_cb_count = 0; + advanceMockTime(1000000); + g_execution_count = 0; + + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); + + ASSERT_EQ(g_execution_count, 7); + /* At iteration 0 (j = 0), hp_event_loop preempted */ + EXPECT_EQ(g_execution_order[0], 999); + EXPECT_NE(g_execution_order[1], 999); + EXPECT_NE(g_execution_order[2], 999); + EXPECT_NE(g_execution_order[3], 999); + EXPECT_NE(g_execution_order[4], 999); + /* At iteration 4 ((j & 3) == 0 after >1000us elapsed), hp_event_loop is checked and serviced again on later masked iteration */ + EXPECT_EQ(g_execution_order[5], 999); + EXPECT_NE(g_execution_order[6], 999); + + g_hp_pipe2_write_fd = -1; + + for (i = 0; i < 5; i++) { + aeDeleteFileEvent(server.el, normal_pipes[i][0], AE_READABLE); + close(normal_pipes[i][0]); + close(normal_pipes[i][1]); + } + aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE); + close(hp_pipe[0]); + close(hp_pipe[1]); + aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe2[0], AE_READABLE); + close(hp_pipe2[0]); + close(hp_pipe2[1]); +} + +TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { + const int NUM_FDS = 20; + const int BYTES_PER_FD = 5; + int pipes[NUM_FDS][2]; + int counts[NUM_FDS]; + int i, ret; + + g_execution_count = 0; + for (i = 0; i < NUM_FDS; i++) { + counts[i] = 0; + ret = pipe(pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); + + ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + + char data[BYTES_PER_FD] = {'a', 'b', 'c', 'd', 'e'}; + if (write(pipes[i][1], data, BYTES_PER_FD) < 0) { + } + } + + /* Run iterations until all FDs drain their BYTES_PER_FD bytes */ + int max_iterations = NUM_FDS * BYTES_PER_FD + 50; + for (int iter = 0; iter < max_iterations; iter++) { + int total_served = 0; + for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; + if (total_served >= NUM_FDS * BYTES_PER_FD) break; + + aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + } + + /* Verify all FDs were served exactly BYTES_PER_FD times without any level-triggered event being lost or starved */ + for (i = 0; i < NUM_FDS; i++) { + EXPECT_EQ(counts[i], BYTES_PER_FD); + aeDeleteFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE); + close(pipes[i][0]); + close(pipes[i][1]); + } +} + +TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPrevention) { + const int NUM_FDS = 20; + int pipes[NUM_FDS][2]; + int counts[NUM_FDS]; + int i, ret; + + g_execution_count = 0; + for (i = 0; i < NUM_FDS; i++) { + counts[i] = 0; + ret = pipe(pipes[i]); + ASSERT_EQ(ret, 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); + } + + /* 1. Register Head FDs (0..6) with 3 bytes (unprocessed at the head) and Tail FDs (14..19) with 5 bytes */ + for (i = 0; i <= 6; i++) { + ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[3] = {'h', 'e', 'a'}; + if (write(pipes[i][1], data, 3) < 0) { + } + } + for (i = 14; i <= 19; i++) { + ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[5] = {'t', 'a', 'i', 'l', 's'}; + if (write(pipes[i][1], data, 5) < 0) { + } + } + + /* Process batch 1: Head and Tail FDs each fire once (reading 1 byte due to level trigger) */ + aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + for (i = 0; i <= 6; i++) EXPECT_GE(counts[i], 1); + for (i = 14; i <= 19; i++) EXPECT_GE(counts[i], 1); + + /* 2. Register Middle FDs (7..13) with 3 bytes (new events in the middle while head/tail have remaining level-triggered data) */ + for (i = 7; i <= 13; i++) { + ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + EXPECT_EQ(ret, AE_OK); + char data[3] = {'m', 'i', 'd'}; + if (write(pipes[i][1], data, 3) < 0) { + } + } + + /* Process remaining batches until all buffers across all groups are completely drained */ + for (int iter = 0; iter < 200; iter++) { + int total_served = 0; + for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; + /* 7 head FDs x 3 + 7 middle FDs x 3 + 6 tail FDs x 5 = 21 + 21 + 30 = 72 total bytes */ + if (total_served >= 72) break; + + aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + } + + /* Verify every single FD across head (unprocessed), middle (new), and tail (level-triggered remaining data) was fully served */ + for (i = 0; i <= 6; i++) EXPECT_EQ(counts[i], 3); + for (i = 7; i <= 13; i++) EXPECT_EQ(counts[i], 3); + for (i = 14; i <= 19; i++) EXPECT_EQ(counts[i], 5); + + for (i = 0; i < NUM_FDS; i++) { + aeDeleteFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE); + close(pipes[i][0]); + close(pipes[i][1]); + } +} + +INSTANTIATE_TEST_SUITE_P( + ConnTypes, + SocketPrioritizationConnTest, + ::testing::Values(CONN_TYPE_SOCKET, CONN_TYPE_TLS)); + +#ifdef __linux__ +/* This test verifies the Linux kernel epoll starvation prevention behavior + * as discussed in Valkey Issue #3927 (https://github.com/valkey-io/valkey/issues/3927#issuecomment-4664882443). + * + * When epoll_wait is called with a maxevents limit smaller than the number of + * currently ready level-triggered FDs: + * 1. The kernel moves all ready FDs to a temporary transfer list (txlist). + * 2. It copies up to maxevents to userspace. + * 3. The copied FDs (still ready in LT mode) are re-queued to the tail of the ready list (rdllist). + * 4. The remaining unprocessed FDs in txlist are bulk re-spliced to the FRONT of rdllist (via ep_done_scan). + * + * This ensures that: + * - Unprocessed FDs are prioritized over re-queued FDs in the next call. + * - New events arriving later are not starved by persistent LT events, because + * eventually the older unreturned events are promoted to the front. + * + * We verify this by: + * 1. Registering 5 pipes (A, B, C, D, E). + * 2. Making A, B, C, D ready. + * 3. Calling epoll_wait(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + * 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + * 5. Calling epoll_wait(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + * 6. Calling epoll_wait(limit=3) -> must return E (now promoted to front because it was left in txlist). + */ +TEST(EpollFairnessTest, KernelBehavior) { + const int NUM_PIPES = 5; + int pipes[NUM_PIPES][2]; + int epfd = epoll_create1(0); + ASSERT_NE(epfd, -1); + + // 1. Registering 5 pipes (A, B, C, D, E) to epoll (Level-Triggered). + for (int i = 0; i < NUM_PIPES; i++) { + ASSERT_EQ(pipe(pipes[i]), 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + + struct epoll_event ee = {0}; + ee.events = EPOLLIN; + ee.data.fd = pipes[i][0]; + ASSERT_EQ(epoll_ctl(epfd, EPOLL_CTL_ADD, pipes[i][0], &ee), 0); + } + + char c = 'x'; + // 2. Making A, B, C, D ready. + for (int i = 0; i < 4; i++) { + ASSERT_EQ(write(pipes[i][1], &c, 1), 1); + } + + // 3. Calling epoll_wait(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + // The unreturned one (D) is moved to the FRONT of rdllist. + // The returned ones (A, B, C) are re-queued to the tail. + // rdllist is now [D, A, B, C]. + struct epoll_event events[3]; + int ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Track which FD was left out in the first call (D). + int left_out_fd = -1; + for (int i = 0; i < 4; i++) { + int fd = pipes[i][0]; + int found = 0; + for (int j = 0; j < ready; j++) { + if (events[j].data.fd == fd) { + found = 1; + break; + } + } + if (!found) { + left_out_fd = fd; + break; + } + } + ASSERT_NE(left_out_fd, -1); + + // 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + ASSERT_EQ(write(pipes[4][1], &c, 1), 1); + + // 5. Calling epoll_wait(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + // Kernel moves [D, A, B, C, E] to txlist. + // Copies D, A, B (limit 3 reached). Re-queues D, A, B. + // C, E are left in txlist and moved to the FRONT of rdllist -> [C, E, D, A, B]. + ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Verify that the left-out FD (D) WAS returned in this second call + // (proving it was moved to the front and not starved by A, B, C). + int found_left_out = 0; + for (int i = 0; i < ready; i++) { + if (events[i].data.fd == left_out_fd) { + found_left_out = 1; + break; + } + } + EXPECT_TRUE(found_left_out) << "Left-out FD was starved!"; + + // Verify that the new event (E, pipe 4) was NOT returned yet (it was at the tail). + for (int i = 0; i < ready; i++) { + EXPECT_NE(events[i].data.fd, pipes[4][0]) << "New event E should not be returned yet"; + } + + // 6. Calling epoll_wait(limit=3) -> must return E (now promoted to front because it was left in txlist). + // rdllist was [C, E, D, A, B] due to C, E being moved to the front. + // It should return C, E, and one of D, A, B. + ready = epoll_wait(epfd, events, 3, 0); + ASSERT_EQ(ready, 3); + + // Verify that the new event (E, pipe 4) IS returned now + // (proving it was moved to the front in the previous step and not starved). + int found_new_event = 0; + for (int i = 0; i < ready; i++) { + if (events[i].data.fd == pipes[4][0]) { + found_new_event = 1; + break; + } + } + EXPECT_TRUE(found_new_event) << "New event E was starved by persistent LT events!"; + + // Cleanup resources. + for (int i = 0; i < NUM_PIPES; i++) { + close(pipes[i][0]); + close(pipes[i][1]); + } + close(epfd); +} +#endif + +#ifdef HAVE_KQUEUE +#include +#include + +/* This test verifies the kqueue event delivery fairness behavior. + * kqueue generally processes events in FIFO order. When calling kevent + * with a limit smaller than the number of ready events: + * 1. The returned events are popped from the head of the active list. + * 2. Unreturned ready events remain at the head of the list. + * 3. Returned events (if still ready/level-triggered) are re-queued to the tail. + * + * This natural FIFO queuing prevents starvation of both unreturned and new events. + * + * We verify this by: + * 1. Registering 5 pipes (A, B, C, D, E) to kqueue (Level-Triggered). + * 2. Making A, B, C, D ready. + * 3. Calling kevent(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + * 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + * 5. Calling kevent(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + * 6. Calling kevent(limit=3) -> must return E (now promoted to front because it was left out). + */ +TEST(KqueueFairnessTest, KernelBehavior) { + const int NUM_PIPES = 5; + int pipes[NUM_PIPES][2]; + int kq = kqueue(); + ASSERT_NE(kq, -1); + + // 1. Registering 5 pipes (A, B, C, D, E) to kqueue (Level-Triggered). + for (int i = 0; i < NUM_PIPES; i++) { + ASSERT_EQ(pipe(pipes[i]), 0); + fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); + + struct kevent ke; + EV_SET(&ke, pipes[i][0], EVFILT_READ, EV_ADD | EV_ENABLE, 0, 0, (void *)(intptr_t)pipes[i][0]); + ASSERT_NE(kevent(kq, &ke, 1, NULL, 0, NULL), -1); + } + + char c = 'x'; + // 2. Making A, B, C, D ready. + for (int i = 0; i < 4; i++) { + ASSERT_EQ(write(pipes[i][1], &c, 1), 1); + } + + // 3. Calling kevent(limit=3) -> returns 3 (e.g., A, B, C). D is left out. + struct kevent events[3]; + struct timespec timeout = {0, 0}; + int ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Track which FD was left out in the first call (D). + int left_out_fd = -1; + for (int i = 0; i < 4; i++) { + int fd = pipes[i][0]; + int found = 0; + for (int j = 0; j < ready; j++) { + if ((int)events[j].ident == fd) { + found = 1; + break; + } + } + if (!found) { + left_out_fd = fd; + break; + } + } + ASSERT_NE(left_out_fd, -1); + + // 4. Making E (new event) ready (appended to tail -> [D, A, B, C, E]). + ASSERT_EQ(write(pipes[4][1], &c, 1), 1); + + // 5. Calling kevent(limit=3) -> must return D (promoted to front), but NOT E (still at tail). + ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Verify that the left-out FD (D) WAS returned in this second call. + int found_left_out = 0; + for (int i = 0; i < ready; i++) { + if ((int)events[i].ident == left_out_fd) { + found_left_out = 1; + break; + } + } + EXPECT_TRUE(found_left_out) << "Left-out FD was starved!"; + + // Verify that the new event (E, pipe 4) was NOT returned yet (it was at the tail). + for (int i = 0; i < ready; i++) { + EXPECT_NE((int)events[i].ident, pipes[4][0]) << "New event E should not be returned yet"; + } + + // 6. Calling kevent(limit=3) -> must return E (now promoted to front because it was left out). + ready = kevent(kq, NULL, 0, events, 3, &timeout); + ASSERT_EQ(ready, 3); + + // Verify that the new event (E, pipe 4) IS returned now. + int found_new_event = 0; + for (int i = 0; i < ready; i++) { + if ((int)events[i].ident == pipes[4][0]) { + found_new_event = 1; + break; + } + } + EXPECT_TRUE(found_new_event) << "New event E was starved!"; + + // Cleanup resources. + for (int i = 0; i < NUM_PIPES; i++) { + close(pipes[i][0]); + close(pipes[i][1]); + } + close(kq); +} +#endif diff --git a/tests/unit/cluster/links.tcl b/tests/unit/cluster/links.tcl index 9cf61d9f741..2f54551f7fa 100644 --- a/tests/unit/cluster/links.tcl +++ b/tests/unit/cluster/links.tcl @@ -186,13 +186,14 @@ start_cluster 3 0 {tags {external:skip cluster network}} { set lines [R 0 cluster links] foreach l $lines { if {$l eq {}} continue - assert_equal [llength $l] 12 + assert_equal [llength $l] 14 assert_equal 1 [dict exists $l "direction"] assert_equal 1 [dict exists $l "node"] assert_equal 1 [dict exists $l "create-time"] assert_equal 1 [dict exists $l "events"] assert_equal 1 [dict exists $l "send-buffer-allocated"] assert_equal 1 [dict exists $l "send-buffer-used"] + assert_equal 1 [dict exists $l "qos"] } } diff --git a/tests/unit/cluster/socket-prioritization.tcl b/tests/unit/cluster/socket-prioritization.tcl new file mode 100644 index 00000000000..7614c21cd01 --- /dev/null +++ b/tests/unit/cluster/socket-prioritization.tcl @@ -0,0 +1,130 @@ +# Verify socket prioritization in Cluster mode (with and without TLS) +start_cluster 2 2 {tags {socket-prioritization external:skip cluster}} { + test "Cluster is up and running" { + wait_for_cluster_state ok + } + + test "Verify CLIENT LIST qos filters for replica links" { + set high_clients [R 0 client list flags H] + assert_match "*flags=*H*" $high_clients + + set normal_cl [valkey 127.0.0.1 [srv 0 port] 0 $::tls] + $normal_cl client setname clusternorm + + assert_match "*name=clusternorm*flags=N*" [R 0 client list not-flags H name clusternorm] + assert_equal "" [R 0 client list flags H name clusternorm] + $normal_cl close + } + + test "Verify cluster slot synchronization under pipeline load" { + wait_for_cluster_state ok + + set load_clients {} + foreach idx {0 1} { + set port [srv [expr -1*$idx] port] + for {set c 0} {$c < 3} {incr c} { + lappend load_clients [valkey 127.0.0.1 $port 0 $::tls] + } + } + + set val [string repeat "y" 128] + set total_ops 0 + for {set iter 0} {$iter < 5} {incr iter} { + foreach cl $load_clients { + catch { + $cl write "*3\r\n\$3\r\nSET\r\n\$7\r\npipekey\r\n\$128\r\n$val\r\n" + $cl flush + $cl read + incr total_ops 1 + } + } + } + + assert {$total_ops > 0} + + wait_for_condition 100 50 { + [R 0 dbsize] + [R 1 dbsize] == 1 && + [R 2 dbsize] == [R 0 dbsize] && + [R 3 dbsize] == [R 1 dbsize] + } else { + fail "Replicas failed to complete sync during pipelined load" + } + + foreach cl $load_clients { catch { $cl close } } + } + + proc local_slot_ranges_contains_slot {slot_ranges slot} { + set ranges [split $slot_ranges " "] + foreach slot_range $ranges { + lassign [split $slot_range -] start end + if {$end == {}} {set end $start} + if {$slot >= $start && $slot <= $end} { + return 1 + } + } + return 0 + } + + proc local_is_slot_migrated {node_idx slot} { + set target_id [R $node_idx CLUSTER MYID] + set nodes [get_cluster_nodes $node_idx] + foreach n $nodes { + set node_id [dict get $n id] + if {$node_id eq $target_id} { + set slot_ranges [dict get $n slots] + if {[local_slot_ranges_contains_slot $slot_ranges $slot]} { + return 1 + } + } + } + return 0 + } + + proc check_prioritized_client_count {node_idx expected_min} { + set prioritized [string trim [R $node_idx client list flags H]] + if {$prioritized eq ""} { + set count 0 + } else { + set count [llength [split $prioritized "\n"]] + } + return [expr {$count > $expected_min}] + } + + test "Verify CLUSTER MIGRATESLOTS connection is prioritized" { + set target_id [R 1 CLUSTER MYID] + set prioritized_before [string trim [R 1 client list flags H]] + if {$prioritized_before eq ""} { + set expected_min 0 + } else { + set expected_min [llength [split $prioritized_before "\n"]] + } + + R 1 DEBUG slotmigration prevent-failover 1 + + assert_equal "OK" [R 0 CLUSTER MIGRATESLOTS SLOTSRANGE 0 0 NODE $target_id] + + wait_for_condition 100 50 { + [check_prioritized_client_count 1 $expected_min] + } else { + puts "Before: $prioritized_before" + puts "After: [R 1 client list flags H]" + fail "Migration connection did not appear on target as prioritized" + } + + R 1 DEBUG slotmigration prevent-failover 0 + + wait_for_condition 100 50 { + [local_is_slot_migrated 1 0] + } else { + fail "Slot 0 was not migrated to R1" + } + } + + test "Verify cluster nodes connections are prioritized from beginning" { + set links [R 0 CLUSTER LINKS] + assert {[llength $links] > 0} + foreach link $links { + assert_equal "prioritized" [dict get $link qos] + } + } +} diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 71efc7c17d0..ce7511107ea 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1664,7 +1664,7 @@ start_server {tags {"introspection"}} { # Get the tot-net-out of the replica before sending the command. set info_list [$primary client list] foreach info [split $info_list "\r\n"] { - if {[string match "* flags=S *" $info]} { + if {[string match "* flags=*S* *" $info]} { set out_before [get_field_in_client_info $info "tot-net-out"] break } @@ -1677,7 +1677,7 @@ start_server {tags {"introspection"}} { # Get the tot-net-out of the replica after sending the command. set info_list [$primary client list] foreach info [split $info_list "\r\n"] { - if {[string match "* flags=S *" $info]} { + if {[string match "* flags=*S* *" $info]} { set out_after [get_field_in_client_info $info "tot-net-out"] break } @@ -2086,6 +2086,41 @@ test {CONFIG hash-seed is immutable and settable at startup} { } } {} {external:skip} +test {Prioritize Sentinel connections based on client name} { + start_server {tags {"introspection"}} { + set rd [valkey_client] + + proc get_client_qos {id} { + set clients [split [string trim [r client list]] "\n"] + foreach c $clients { + if {[regexp "id=$id .* flags=(\\w+)" $c match flags]} { + if {[string match "*H*" $flags]} { + return "prioritized" + } else { + return "normal" + } + } + } + return "unknown" + } + + set rd_id [$rd client id] + + # Initially, QoS should be normal + assert_equal "normal" [get_client_qos $rd_id] + + # Set name to something that doesn't start with sentinel- + $rd client setname "not-sentinel" + assert_equal "normal" [get_client_qos $rd_id] + + # Set name starting with sentinel- + $rd client setname "sentinel-12345-cmd" + assert_equal "prioritized" [get_client_qos $rd_id] + + $rd close + } +} + start_server {overrides {forkless-infrastructure-enabled yes} tags {"introspection" "external:skip"}} { foreach bgsave_type {"fork" "forkless"} { test "CLIENT KILL close the client connection during bgsave - $bgsave_type" { diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl new file mode 100644 index 00000000000..1690c6914fc --- /dev/null +++ b/tests/unit/socket-prioritization.tcl @@ -0,0 +1,153 @@ +# Copyright (c) Valkey Contributors +# All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +start_server {tags {"socket-prioritization"}} { + test {Socket Prioritization: CLIENT LIST and CLIENT KILL QOS filter} { + set c1 [valkey_client] + $c1 client setname qostestclient + + # By default client is normal priority + set res [r client list not-flags H name qostestclient] + assert_match "*name=qostestclient*flags=N*" $res + + set res_high [r client list flags H name qostestclient] + assert_equal "" $res_high + + # Test invalid flags argument + assert_error "*Unknown flags*" {r client list flags invalid_flag} + + # Kill client by flags + set killed [r client kill not-flags H name qostestclient] + assert_equal 1 $killed + assert_error "*I/O error*" {$c1 ping} + catch {$c1 close} + } + + test {Socket Prioritization: Replication QoS classification and CLIENT KILL} { + # Start a replica server and verify that replication links are upgraded to high priority + start_server {} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + set primary [srv -1 client] + set primary_host [srv -1 host] + set primary_port [srv -1 port] + + # Connect replica to primary + $replica replicaof $primary_host $primary_port + wait_for_condition 50 100 { + [string match "*role:slave*master_link_status:up*" [$replica info replication]] + } else { + fail "Can't turn the instance into a replica" + } + + # On primary server, verify that a client connection has flags=H for the replica + set rep_list [$primary client list flags H] + assert_match "*flags=*H*" $rep_list + + set val [string repeat "a" 1024] + for {set i 0} {$i < 50} {incr i} { + $primary set "key:$i" $val + } + $primary ping + + # Wait for replica to sync the keys + wait_for_condition 50 100 { + [$replica dbsize] == 50 + } else { + fail "Replica failed to sync 50 keys" + } + + # Verify CLIENT KILL flags H kills the replica connection + set killed [$primary client kill flags H] + assert {$killed >= 1} + } + } {} {external:skip} + + foreach io_threads {1 4} { + test "Verify replication connection upgrade (io-threads=$io_threads)" { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary CONFIG SET io-threads $io_threads + + set replica_mock [valkey $primary_host $primary_port 0 $::tls] + $replica_mock client setname replica_mock_$io_threads + + set res [$primary client list not-flags H name replica_mock_$io_threads] + assert_match "*name=replica_mock_$io_threads*flags=N*" $res + assert_equal "" [$primary client list flags H name replica_mock_$io_threads] + + $replica_mock write "PSYNC ? -1\r\n" + $replica_mock flush + + wait_for_condition 50 100 { + [string match "*name=replica_mock_${io_threads}*flags=*H*" [$primary client list flags H name replica_mock_$io_threads]] + } else { + fail "Replica connection was not upgraded to high priority (io-threads=$io_threads)" + } + + $replica_mock close + $primary CONFIG SET io-threads 1 + } + } +} + +start_server {tags {"socket-prioritization external:skip"}} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + test "Populate standalone primary with 1000 keys" { + for {set k 0} {$k < 1000} {incr k} { + $primary set "benchkey:$k" [string repeat "x" 128] + } + } + + start_server {} { + set replica1 [srv 0 client] + start_server {} { + set replica2 [srv 0 client] + + test "Benchmark standalone replica sync under pipeline load with high-priority event loop" { + set load_clients {} + for {set c 0} {$c < 5} {incr c} { + lappend load_clients [valkey $primary_host $primary_port 0 $::tls] + } + + set val [string repeat "x" 256] + set pipeline "" + for {set p 0} {$p < 20} {incr p} { + append pipeline "*3\r\n\$3\r\nSET\r\n\$7\r\npipekey\r\n\$256\r\n$val\r\n" + } + + $replica1 replicaof $primary_host $primary_port + $replica2 replicaof $primary_host $primary_port + + set total_ops 0 + for {set iter 0} {$iter < 10} {incr iter} { + foreach cl $load_clients { + catch { + $cl write $pipeline + $cl flush + incr total_ops 20 + for {set r 0} {$r < 20} {incr r} { $cl read } + } + } + } + + wait_for_condition 100 50 { + [status $primary connected_slaves] == 2 && + [$replica1 dbsize] == 1000 && + [$replica2 dbsize] == 1000 + } else { + fail "Replicas failed to complete sync during pipelined load" + } + + foreach cl $load_clients { $cl close } + } + } + } +} From 4eaf6512e743a438a4ddae03aee3d80cbe1509e3 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Mon, 17 Aug 2026 02:59:53 +0000 Subject: [PATCH 02/14] Address review comments Signed-off-by: Satheesha Gowda --- src/ae.c | 70 +++++++++------- src/ae.h | 22 ++++-- src/cluster.c | 4 +- src/cluster_legacy.c | 6 +- src/cluster_migrateslots.c | 4 +- src/commands/cluster-links.json | 4 + src/connection.c | 83 ++++++++++++------- src/connection.h | 29 +++++-- src/io_threads.c | 31 +++++++- src/networking.c | 7 -- src/replication.c | 13 ++- src/server.c | 13 ++- src/server.h | 1 - src/unit/test_socket_prioritization.cpp | 101 +++++++++++++++++++++++- tests/unit/introspection.tcl | 35 -------- 15 files changed, 295 insertions(+), 128 deletions(-) diff --git a/src/ae.c b/src/ae.c index 4968a14683c..998c3b48bc3 100644 --- a/src/ae.c +++ b/src/ae.c @@ -78,13 +78,14 @@ * implementations are AE_READABLE & AE_WRITABLE. */ #define BACKEND_MASK(mask) ((mask) & (AE_READABLE | AE_WRITABLE)) -/* High priority event loop periodic preemptive poll interval in microseconds.*/ +/* High priority event loop periodic preemptive poll interval in microseconds. + * High-priority events are checked periodically during normal event loops to prevent + * long batches of normal client commands from starving control plane traffic. */ #define AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US 1000 -/* High priority event loop periodic preemptive poll mask.*/ -#define AE_HP_EVENT_PREEMPTIVE_CHECK_MASK 3 -/* last time high priority event loop was polled */ -static long long hp_event_loop_last_poll_us = 0; +/* High priority event loop periodic preemptive poll mask. + * Used to amortize getMonotonicUs() clock reads across every 4th iteration ((iter & 3) == 0). */ +#define AE_HP_EVENT_PREEMPTIVE_CHECK_MASK 3 aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; @@ -106,6 +107,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->custompoll = NULL; eventLoop->flags = 0; eventLoop->hp_event_loop = NULL; + eventLoop->hp_last_poll_us = 0; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -160,6 +162,10 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) goto err; if (aeApiResize(eventLoop->apidata, setsize) == -1) goto err; + if (eventLoop->hp_event_loop) { + if (aeResizeSetSize(eventLoop->hp_event_loop, setsize) == AE_ERR) goto err; + } + eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * setsize); eventLoop->fired = zrealloc(eventLoop->fired, sizeof(aeFiredEvent) * setsize); eventLoop->setsize = setsize; @@ -178,6 +184,8 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { void aeDeleteEventLoop(aeEventLoop *eventLoop) { if (eventLoop->hp_event_loop) { + int hp_fd = aeApiGetPollFd(eventLoop->hp_event_loop); + if (hp_fd != -1) aeDeleteFileEvent(eventLoop, hp_fd, AE_READABLE); aeDeleteEventLoop(eventLoop->hp_event_loop); } aeApiFree(eventLoop->apidata); @@ -205,6 +213,8 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc /* Register events on to high priority event loop */ return aeCreateFileEvent(eventLoop->hp_event_loop, fd, mask & ~AE_HIGH_PRIORITY, proc, clientData); } + /* If hp_event_loop is not initialized, strip the high priority flag */ + mask &= ~AE_HIGH_PRIORITY; } AE_LOCK(eventLoop); int ret = AE_ERR; @@ -448,20 +458,18 @@ static int aeProcessHPEventsNow(aeEventLoop *eventLoop) { int processed = 0; if (eventLoop->hp_event_loop != NULL) { processed = aeProcessEvents(eventLoop->hp_event_loop, AE_ALL_EVENTS | AE_DONT_WAIT); - hp_event_loop_last_poll_us = getMonotonicUs(); + eventLoop->hp_last_poll_us = getMonotonicUs(); } return processed; } -/* - * Preemptively processes high priority events if the elapsed time since the last poll +/* Preemptively processes high priority events if the elapsed time since the last poll * exceeds the AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US threshold and iter count - * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK. - */ + * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK. */ int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { - if (eventLoop->hp_event_loop == NULL) return 0; + if (eventLoop->hp_event_loop == NULL || iter_count == 0) return 0; if ((iter_count & AE_HP_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { - if (elapsedUs(hp_event_loop_last_poll_us) >= AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US) { + if (elapsedUs(eventLoop->hp_last_poll_us) >= AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US) { return aeProcessHPEventsNow(eventLoop); } } @@ -532,24 +540,20 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); - /* Process all high-priority events before we start processing the normal events - * if the high-priority event loop's FD (`hp_fd`) was fired.*/ + /* Prioritize high-priority event loop: if hp_fd fired, drain HP events + * immediately before processing normal events. */ int hp_fd = -1; if (eventLoop->hp_event_loop != NULL) { hp_fd = aeApiGetPollFd(eventLoop->hp_event_loop); - } - int hp_fired = 0; - if (hp_fd != -1) { - for (j = 0; j < numevents; j++) { - if (eventLoop->fired[j].fd == hp_fd) { - hp_fired = 1; - break; + if (hp_fd != -1) { + for (j = 0; j < numevents; j++) { + if (eventLoop->fired[j].fd == hp_fd) { + processed += aeProcessHPEventsNow(eventLoop); + break; + } } } } - if (hp_fired) { - processed += aeProcessHPEventsNow(eventLoop); - } for (j = 0; j < numevents; j++) { /* Periodic Preemptive Poll: @@ -559,8 +563,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { int fd = eventLoop->fired[j].fd; /* Skip processing HP events here again as they were already processed above */ - if (hp_fired && fd == hp_fd) continue; - + if (fd == hp_fd) continue; aeFileEvent *fe = &eventLoop->events[fd]; int mask = eventLoop->fired[j].mask; int fired = 0; /* Number of events fired for current fd. */ @@ -683,8 +686,17 @@ static void hpEventLoopHandler(aeEventLoop *el, int fd, void *privdata, int mask /* Register a high-priority event loop file descriptor in the main event loop, * which will awake main eventloop when the high-priority event loop file descriptor is fired. */ -void aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop) { - eventLoop->hp_event_loop = hp_event_loop; +int aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop) { + if (eventLoop == NULL || hp_event_loop == NULL) return AE_ERR; int hp_fd = aeApiGetPollFd(hp_event_loop); - if (hp_fd != -1) aeCreateFileEvent(eventLoop, hp_fd, AE_READABLE, hpEventLoopHandler, hp_event_loop); + if (hp_fd == -1) { + eventLoop->hp_event_loop = NULL; + return AE_ERR; + } + if (aeCreateFileEvent(eventLoop, hp_fd, AE_READABLE, hpEventLoopHandler, hp_event_loop) == AE_ERR) { + eventLoop->hp_event_loop = NULL; + return AE_ERR; + } + eventLoop->hp_event_loop = hp_event_loop; + return AE_OK; } diff --git a/src/ae.h b/src/ae.h index 9099a5bbf53..9306030db4b 100644 --- a/src/ae.h +++ b/src/ae.h @@ -47,7 +47,9 @@ loop iteration. Useful when you want to persist \ things to disk before sending replies, and want \ to do that in a group fashion. */ -#define AE_HIGH_PRIORITY 8 /* Route to high-priority nested loop */ +#define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ + * the event is registered on hp_event_loop if available. \ + * Stripped before passing to the underlying OS multiplexer. */ #define AE_FILE_EVENTS (1 << 0) #define AE_TIME_EVENTS (1 << 1) @@ -125,8 +127,13 @@ typedef struct aeEventLoop { pthread_mutex_t poll_mutex; int flags; - /* QoS */ + /* High-Priority Quality of Service (QoS): + * hp_event_loop points to a secondary, nested event loop dedicated to critical + * internal channels (cluster bus, slot migration, replication). Sockets registered + * here are processed ahead of normal client traffic and periodically preempt + * normal event iterations. NULL if unsupported by the OS multiplexer or disabled. */ struct aeEventLoop *hp_event_loop; + monotime hp_last_poll_us; /* Timestamp (microseconds) when hp_event_loop was last drained */ } aeEventLoop; /* Prototypes */ @@ -155,9 +162,14 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); -/* Process high priority events preemptively. */ + +/* Preemptively processes pending high-priority events during long-running normal event loops. + * Returns the number of high-priority events processed. */ int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count); -/* Link a high-priority event loop to a main event loop for nested polling. */ -void aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop); + +/* Links a nested high-priority event loop to the main event loop by registering the child loop's + * multiplexer descriptor (e.g. epoll fd) on the parent loop with AE_READABLE. + * Returns AE_OK on success or AE_ERR if multiplexer nesting is unsupported or registration fails. */ +int aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop); #endif diff --git a/src/cluster.c b/src/cluster.c index 1e393ecf822..3e34ed2d240 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -365,9 +365,9 @@ migrateCachedSocket *migrateGetSocket(client *c, robj *host, robj *port, long ti dictDelete(server.migrate_cached_sockets, dictGetKey(de)); } - /* Create the connection */ + /* Create the connection and tag as high-priority so key/slot migration + * packets are not delayed by normal tenant commands. */ conn = connCreate(connTypeOfCluster()); - /* Set connection to high priority */ connSetPriority(conn, CONN_PRIORITY_HIGH); if (connBlockingConnect(conn, objectGetVal(host), atoi(objectGetVal(port)), timeout) != C_OK) { addReplyError(c, "-IOERR error or timeout connecting to the client"); diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index a819d5e33d6..cb5c0f17544 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2052,7 +2052,8 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); - /* Set connection to high priority */ + /* Tag inbound cluster bus link as high-priority so cluster gossip and heartbeats + * are processed in hp_event_loop ahead of normal client traffic. */ connSetPriority(conn, CONN_PRIORITY_HIGH); /* Mark as cluster-owned before any TLS accept retries so generic * accept offload routing can safely avoid client assumptions. */ @@ -6679,7 +6680,8 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long (*cluster_conn_attempts)--; clusterLink *link = createClusterLink(node); link->conn = connCreate(connTypeOfCluster()); - /* Set connection to high priority */ + /* Tag outbound cluster bus link as high-priority so node reconnects, gossip ping/pong, + * and failure detection heartbeats operate within hp_event_loop. */ connSetPriority(link->conn, CONN_PRIORITY_HIGH); connSetPrivateData(link->conn, link); connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index 67115b46a34..204f3ecc4a9 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -853,7 +853,9 @@ slotMigrationJob *createSlotImportJob(client *c, job->client->slot_migration_job = job; if (c && c->conn) { /* Upgrade connection to high priority */ - connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); + if (connSetPriority(c->conn, CONN_PRIORITY_HIGH) == C_ERR) { + serverLog(LL_WARNING, "Failed to upgrade priority for slot migration connection %d", c->conn->fd); + } } /* We treat slot imports like primaries. Primaries are expected to have a diff --git a/src/commands/cluster-links.json b/src/commands/cluster-links.json index eef363367ca..865289ff2f6 100644 --- a/src/commands/cluster-links.json +++ b/src/commands/cluster-links.json @@ -51,6 +51,10 @@ "send-buffer-used": { "description": "Size of the portion of the link's send buffer that is currently holding data(messages).", "type": "integer" + }, + "qos": { + "description": "Quality of service priority level of the link connection.", + "type": "string" } }, "additionalProperties": false diff --git a/src/connection.c b/src/connection.c index fd12eec4a74..dd45675c68e 100644 --- a/src/connection.c +++ b/src/connection.c @@ -168,37 +168,71 @@ sds getListensInfoString(sds info) { return info; } -/* Upgrade connection priority */ -void connUpgradePriority(connection *conn, int priority) { +/* Set connection priority. If the connection already has active events + * registered in the event loop, migrate them to the new priority level. + * Handles postponed state safely if the socket is offloaded to IO threads, + * preserves AE_BARRIER ordering flags, and performs atomic rollback on failure. + * Returns C_OK on success, or C_ERR if event migration fails. */ +int connSetPriority(connection *conn, int priority) { serverAssert(conn != NULL); - if (conn->priority == priority) return; + if (conn->priority == priority) return C_OK; + /* Fast path: if no socket exists or no event loop is active, update field directly */ + if (conn->fd == -1 || server.el == NULL) { + conn->priority = priority; + return C_OK; + } + + int mask = aeGetFileEvents(server.el, conn->fd); + if (mask == AE_NONE) { + conn->priority = priority; + return C_OK; + } + + /* If socket state update is postponed by IO threads, update priority field only; + * connUpdateState() will register with the new priority upon IO completion. */ + if (conn->flags & CONN_FLAG_POSTPONE_UPDATE_STATE) { + conn->priority = priority; + return C_OK; + } + + /* Dynamic migration: active events exist on this socket */ int old_priority = conn->priority; conn->priority = priority; - if (conn->fd != -1) { - int mask = aeGetFileEvents(server.el, conn->fd); - if (mask != AE_NONE) { - /* Delete from wherever it is (AE will route delete correctly) */ - aeDeleteFileEvent(server.el, conn->fd, AE_READABLE | AE_WRITABLE); - - /* Re-create with new priority flag */ - int hp_flag = connGetAEPriorityFlag(conn); - if (mask & AE_READABLE) - aeCreateFileEvent(server.el, conn->fd, AE_READABLE | hp_flag, conn->type->ae_handler, conn); - if (mask & AE_WRITABLE) - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | hp_flag, conn->type->ae_handler, conn); + /* Delete existing registration (AE routes delete to the correct loop) */ + aeDeleteFileEvent(server.el, conn->fd, AE_READABLE | AE_WRITABLE); + + /* If transport has custom state updater (e.g. TLS), delegate to it */ + if (conn->type && conn->type->update_state) { + conn->type->update_state(conn); + } else { + int barrier = mask & AE_BARRIER; + int ae_hp_flag = connGetAEPriorityFlag(conn); + if ((mask & AE_READABLE) && + aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | ae_hp_flag, conn->type->ae_handler, conn) == AE_ERR) { + goto rollback; + } + if ((mask & AE_WRITABLE) && + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | ae_hp_flag, conn->type->ae_handler, conn) == AE_ERR) { + goto rollback; } } - serverLog(LL_DEBUG, "Connection fd %d qos upgraded from %s to %s", conn->fd, getConnectionPriorityName(old_priority), getConnectionPriorityName(priority)); -} + serverLog(LL_DEBUG, "Connection fd %d priority updated from %s to %s", + conn->fd, getConnectionPriorityName(old_priority), getConnectionPriorityName(priority)); + return C_OK; -/* Set connection priority */ -void connSetPriority(connection *conn, int priority) { - serverAssert(conn != NULL); - if (conn->priority == priority) return; - conn->priority = priority; +rollback: + /* Rollback priority and restore previous event registrations */ + conn->priority = old_priority; + int barrier = mask & AE_BARRIER; + int old_ae_hp_flag = connGetAEPriorityFlag(conn); + if (mask & AE_READABLE) + aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | old_ae_hp_flag, conn->type->ae_handler, conn); + if (mask & AE_WRITABLE) + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | old_ae_hp_flag, conn->type->ae_handler, conn); + return C_ERR; } /* Get connection priority */ @@ -207,11 +241,6 @@ int connGetPriority(connection *conn) { return conn ? conn->priority : CONN_PRIORITY_NORMAL; } -/* Get AE priority flag for a connection */ -int connGetAEPriorityFlag(connection *conn) { - return (conn && conn->priority == CONN_PRIORITY_HIGH) ? AE_HIGH_PRIORITY : 0; -} - /* Get connection priority name from priority value. */ const char *getConnectionPriorityName(int priority) { switch (priority) { diff --git a/src/connection.h b/src/connection.h index e42028e8ca7..858d375310b 100644 --- a/src/connection.h +++ b/src/connection.h @@ -72,6 +72,9 @@ typedef enum { #define CONN_FLAG_WRITE_BARRIER (1 << 1) /* Write barrier requested */ #define CONN_FLAG_ALLOW_ACCEPT_OFFLOAD (1 << 2) /* Connection accept can be offloaded to IO threads. */ #define CONN_FLAG_ACCEPT_OFFLOAD_PENDING (1 << 3) /* Accept offload job is currently in flight. */ +#define CONN_FLAG_POSTPONE_UPDATE_STATE (1 << 4) /* Connection update state is postponed by IO threads \ + * to prevent main thread event loop races while worker \ + * threads access the socket buffers. */ #define CONN_POSTPONE_READ (1 << 0) #define CONN_POSTPONE_WRITE (1 << 1) @@ -558,11 +561,21 @@ static inline aeFileProc *connAcceptHandler(ConnectionType *ct) { sds getListensInfoString(sds info); /* Connection qos(see connPriority enum). */ -void connSetPriority(connection *conn, int priority); -void connUpgradePriority(connection *conn, int priority); +int connSetPriority(connection *conn, int priority); int connGetPriority(connection *conn); const char *getConnectionPriorityName(int priority); -int connGetAEPriorityFlag(connection *conn); + +/* Get AE priority flag for a connection. */ +static inline int connGetAEPriorityFlag(const connection *conn) { + if (conn == NULL) return AE_NONE; + switch (conn->priority) { + case CONN_PRIORITY_HIGH: + return AE_HIGH_PRIORITY; + case CONN_PRIORITY_NORMAL: + default: + return AE_NONE; + } +} int RedisRegisterConnectionTypeSocket(void); int RedisRegisterConnectionTypeUnix(void); int RedisRegisterConnectionTypeTLS(void); @@ -580,8 +593,14 @@ static inline void connUpdateState(connection *conn) { } static inline void connSetPostponeUpdateState(connection *conn, int postpone_mask) { - if (conn && conn->type && conn->type->postpone_update_state) { - conn->type->postpone_update_state(conn, postpone_mask); + if (conn) { + if (postpone_mask) + conn->flags |= CONN_FLAG_POSTPONE_UPDATE_STATE; + else + conn->flags &= ~CONN_FLAG_POSTPONE_UPDATE_STATE; + if (conn->type && conn->type->postpone_update_state) { + conn->type->postpone_update_state(conn, postpone_mask); + } } } diff --git a/src/io_threads.c b/src/io_threads.c index df56b1122e3..917723ae184 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -1183,8 +1183,33 @@ static int processOutboxBatch(mpscQueue *outbox) { if (read_count) handleReadJobs(read_jobs, read_count); if (write_count) handleWriteJobs(write_jobs, write_count); - - /* If the queue was empty at the last try - don't try again */ - if (dequeued_count == 0) return total_processed; + return received_responses; } + +/* Process completed IO jobs from worker threads back onto the main thread. + * Drains the high-priority outbox first to guarantee control-plane responsiveness, + * and performs periodic preemptive polling of hp_event_loop while consuming normal jobs. */ +int processIOThreadsResponses(void) { + /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ + + /* Quick check if any pending operations exist across any priority level */ + if (getPendingIOResponsesCount() == 0) return 0; + + int total_processed = 0; + int counter = 0; + /* Loop until we consume all pending jobs */ + while (1) { + /* 1. Strict Priority: First, drain high-priority events (cluster bus, replication, and slot migration jobs) */ + int processed = processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_HIGH]); + if (processed == 0) { + /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new + * high-priority events arrived on hp_event_loop before processing normal traffic. */ + aeProcessHPEventsPreemptively(server.el, counter); + } + /* 3. Drain normal client events */ + processed += processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_NORMAL]); + total_processed += processed; + counter++; + if (processed == 0) return total_processed; + } } diff --git a/src/networking.c b/src/networking.c index 8c240693237..fe7be2e2eca 100644 --- a/src/networking.c +++ b/src/networking.c @@ -4705,13 +4705,6 @@ int clientSetName(client *c, robj *name, const char **err) { if (c->name) decrRefCount(c->name); c->name = name; incrRefCount(name); - - if (name && len >= 9 && strncmp(objectGetVal(name), "sentinel-", 9) == 0) { - if (c->conn) { - /* Upgrade connection priority to high for sentinel */ - connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); - } - } return C_OK; } diff --git a/src/replication.c b/src/replication.c index 7f921fae695..9b3c827f0ba 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1172,8 +1172,11 @@ void syncCommand(client *c) { serverLog(LL_NOTICE, "Replica %s asks for synchronization", replicationGetReplicaName(c)); if (c->conn) { - /* Upgrade replica connection to high priority */ - connUpgradePriority(c->conn, CONN_PRIORITY_HIGH); + /* Upgrade incoming replica connection to high priority so that replication + * command streaming and ACK heartbeats are not delayed by normal client commands. */ + if (connSetPriority(c->conn, CONN_PRIORITY_HIGH) == C_ERR) { + serverLog(LL_WARNING, "Failed to upgrade priority for replica connection %d", c->conn->fd); + } } /* Try a partial resynchronization if this is a PSYNC command. @@ -4389,7 +4392,8 @@ void syncWithPrimary(connection *conn) { if (psync_result == PSYNC_FULLRESYNC_DUAL_CHANNEL) { /* Create RDB connection */ server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); - /* Set connection to high priority */ + /* Tag connection as high-priority before connecting so non-blocking connect and + * subsequent RDB transfer events are registered directly on hp_event_loop. */ connSetPriority(server.repl_rdb_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { @@ -4441,7 +4445,8 @@ void syncWithPrimary(connection *conn) { int connectWithPrimary(void) { server.repl_transfer_s = connCreate(connTypeOfReplication()); - /* Set connection to high priority */ + /* Tag main replication connection as high-priority before connecting so handshake, + * heartbeat pings, and PSYNC streaming are processed in hp_event_loop. */ connSetPriority(server.repl_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { diff --git a/src/server.c b/src/server.c index e9fb7ac4733..dc317e2285d 100644 --- a/src/server.c +++ b/src/server.c @@ -3095,14 +3095,21 @@ void initServer(void) { const char *clk_msg = monotonicInit(); serverLog(LL_NOTICE, "monotonic clock: %s", clk_msg); server.el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - /*Create high priority event loop*/ + /* Create high-priority event loop for internal connections (cluster bus, replication, and slot migration jobs) */ aeEventLoop *hp_el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); if (server.el == NULL || hp_el == NULL) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - /*Link high priority event loop with main event loop*/ - aeLinkHighPriorityEventLoop(server.el, hp_el); + /* Link high priority event loop with main event loop via nested multiplexer polling. + * If multiplexer nesting is unsupported (e.g. evport, select), gracefully + * free hp_el and fall back to main event loop event processing without QoS. */ + if (aeLinkHighPriorityEventLoop(server.el, hp_el) == AE_ERR) { + serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", + aeGetApiName()); + aeDeleteEventLoop(hp_el); + hp_el = NULL; + } server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); diff --git a/src/server.h b/src/server.h index 02218f04d2f..115d01f0419 100644 --- a/src/server.h +++ b/src/server.h @@ -3104,7 +3104,6 @@ int closeClientOnOutputBufferLimitReached(client *c, int async); int getClientType(client *c); int getClientTypeByName(char *name); char *getClientTypeName(int client_class); -const char *getClientQoSName(int qos); void flushReplicasOutputBuffers(void); void disconnectReplicas(void); void evictClients(void); diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 39e057faea7..0a083df6d6a 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -17,9 +17,9 @@ #define _Static_assert static_assert extern "C" { #include "ae.h" -#include "monotonic.h" #include "connection.h" #include "io_threads.h" +#include "monotonic.h" #include "server.h" } #undef _Static_assert @@ -204,7 +204,7 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); /* Dynamically upgrade connection to high priority */ - connUpgradePriority(conn, CONN_PRIORITY_HIGH); + EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_HIGH), C_OK); EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); int events = aeGetFileEvents(server.el, read_fd); EXPECT_NE(events & AE_READABLE, 0); @@ -212,7 +212,7 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { EXPECT_NE(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); /* Dynamically downgrade connection back to normal priority */ - connUpgradePriority(conn, CONN_PRIORITY_NORMAL); + EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_NORMAL), C_OK); EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); @@ -394,7 +394,7 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); ASSERT_EQ(g_execution_count, 7); - /* At iteration 0 (j = 0), hp_event_loop preempted */ + /* At start (pre-loop), hp_event_loop processed */ EXPECT_EQ(g_execution_order[0], 999); EXPECT_NE(g_execution_order[1], 999); EXPECT_NE(g_execution_order[2], 999); @@ -528,6 +528,99 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve } } +TEST_F(SocketPrioritizationTest, DualEventLoopResize) { + ASSERT_NE(server.el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->hp_event_loop, (aeEventLoop *)NULL); + EXPECT_EQ(aeGetSetSize(server.el), 1024); + EXPECT_EQ(aeGetSetSize(server.el->hp_event_loop), 1024); + + int ret = aeResizeSetSize(server.el, 2048); + EXPECT_EQ(ret, AE_OK); + EXPECT_EQ(aeGetSetSize(server.el), 2048); + EXPECT_EQ(aeGetSetSize(server.el->hp_event_loop), 2048); +} + +TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { + aeEventLoop *standalone_el = aeCreateEventLoop(64); + ASSERT_NE(standalone_el, (aeEventLoop *)NULL); + EXPECT_EQ(standalone_el->hp_event_loop, (aeEventLoop *)NULL); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + /* Register with AE_HIGH_PRIORITY on standalone event loop with no hp_event_loop */ + int ret = aeCreateFileEvent(standalone_el, fds[0], AE_READABLE | AE_HIGH_PRIORITY, testNormalEventCallback, NULL); + EXPECT_EQ(ret, AE_OK); + + /* Mask should only have AE_READABLE, and NOT retain AE_HIGH_PRIORITY (0x8) */ + int events = aeGetFileEvents(standalone_el, fds[0]); + EXPECT_EQ(events, AE_READABLE); + + /* Delete the readable event */ + aeDeleteFileEvent(standalone_el, fds[0], AE_READABLE); + EXPECT_EQ(aeGetFileEvents(standalone_el, fds[0]), AE_NONE); + + close(fds[0]); + close(fds[1]); + aeDeleteEventLoop(standalone_el); +} + +TEST_F(SocketPrioritizationTest, PostponedStateDynamicPriorityUpdate) { + ConnectionType *ct = connectionByType(CONN_TYPE_SOCKET); + ASSERT_NE(ct, (ConnectionType *)NULL); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = fds[0]; + + int ret = connSetReadHandler(conn, dummyConnectionHandler); + EXPECT_EQ(ret, C_OK); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + + /* Set postponed state (e.g. while offloaded to IO threads) */ + conn->flags |= CONN_FLAG_POSTPONE_UPDATE_STATE; + + /* Upgrading priority while postponed must update priority field but defer event loop migration */ + ret = connSetPriority(conn, CONN_PRIORITY_HIGH); + EXPECT_EQ(ret, C_OK); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + + conn->flags &= ~CONN_FLAG_POSTPONE_UPDATE_STATE; + conn->state = CONN_STATE_NONE; + connClose(conn); + close(fds[1]); +} + +TEST_F(SocketPrioritizationTest, WriteBarrierPreservation) { + ConnectionType *ct = connectionByType(CONN_TYPE_SOCKET); + ASSERT_NE(ct, (ConnectionType *)NULL); + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + connection *conn = connCreate(ct); + ASSERT_NE(conn, (connection *)NULL); + conn->fd = fds[0]; + + int ret = connSetWriteHandlerWithBarrier(conn, dummyConnectionHandler, 1); + EXPECT_EQ(ret, C_OK); + + int events = aeGetFileEvents(server.el, fds[0]); + EXPECT_NE(events & AE_WRITABLE, 0); + + /* Upgrade priority: verify migration succeeds with barrier */ + ret = connSetPriority(conn, CONN_PRIORITY_HIGH); + EXPECT_EQ(ret, C_OK); + EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + + conn->state = CONN_STATE_NONE; + connClose(conn); + close(fds[1]); +} + INSTANTIATE_TEST_SUITE_P( ConnTypes, SocketPrioritizationConnTest, diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index ce7511107ea..e4c6d7a11c6 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -2086,41 +2086,6 @@ test {CONFIG hash-seed is immutable and settable at startup} { } } {} {external:skip} -test {Prioritize Sentinel connections based on client name} { - start_server {tags {"introspection"}} { - set rd [valkey_client] - - proc get_client_qos {id} { - set clients [split [string trim [r client list]] "\n"] - foreach c $clients { - if {[regexp "id=$id .* flags=(\\w+)" $c match flags]} { - if {[string match "*H*" $flags]} { - return "prioritized" - } else { - return "normal" - } - } - } - return "unknown" - } - - set rd_id [$rd client id] - - # Initially, QoS should be normal - assert_equal "normal" [get_client_qos $rd_id] - - # Set name to something that doesn't start with sentinel- - $rd client setname "not-sentinel" - assert_equal "normal" [get_client_qos $rd_id] - - # Set name starting with sentinel- - $rd client setname "sentinel-12345-cmd" - assert_equal "prioritized" [get_client_qos $rd_id] - - $rd close - } -} - start_server {overrides {forkless-infrastructure-enabled yes} tags {"introspection" "external:skip"}} { foreach bgsave_type {"fork" "forkless"} { test "CLIENT KILL close the client connection during bgsave - $bgsave_type" { From d109fc8e7eb10e43223a334c6236f78bd77fa4aa Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Tue, 18 Aug 2026 06:54:09 +0000 Subject: [PATCH 03/14] QoS preemptive polling interval dynamically configurable Signed-off-by: Satheesha Gowda --- src/ae.c | 29 +++++++++++++++++++------ src/ae.h | 7 +++++- src/config.c | 11 ++++++++++ src/server.c | 1 + src/server.h | 1 + src/unit/test_socket_prioritization.cpp | 19 ++++++++++++++-- tests/unit/socket-prioritization.tcl | 12 ++++++++++ valkey.conf | 28 ++++++++++++++++++++++++ 8 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/ae.c b/src/ae.c index 998c3b48bc3..a14c90a8215 100644 --- a/src/ae.c +++ b/src/ae.c @@ -78,10 +78,10 @@ * implementations are AE_READABLE & AE_WRITABLE. */ #define BACKEND_MASK(mask) ((mask) & (AE_READABLE | AE_WRITABLE)) -/* High priority event loop periodic preemptive poll interval in microseconds. +/* High priority event loop periodic preemptive poll default interval in microseconds. * High-priority events are checked periodically during normal event loops to prevent * long batches of normal client commands from starving control plane traffic. */ -#define AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US 1000 +#define AE_HP_DEFAULT_PREEMPT_CHECK_INTERVAL_US 2000 /* High priority event loop periodic preemptive poll mask. * Used to amortize getMonotonicUs() clock reads across every 4th iteration ((iter & 3) == 0). */ @@ -108,6 +108,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->flags = 0; eventLoop->hp_event_loop = NULL; eventLoop->hp_last_poll_us = 0; + eventLoop->hp_preempt_check_interval_us = AE_HP_DEFAULT_PREEMPT_CHECK_INTERVAL_US; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -463,13 +464,27 @@ static int aeProcessHPEventsNow(aeEventLoop *eventLoop) { return processed; } +/* Set the preemptive poll interval in microseconds for high-priority events. + * High-priority events are checked periodically during normal event processing + * when processing batches of normal events exceeds this interval. + * Setting this interval to 0 disables preemptive polling. */ +void aeSetHPPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { + if (eventLoop) eventLoop->hp_preempt_check_interval_us = interval_us; +} + +/* Get the current preemptive poll interval in microseconds for high-priority events. + * Returns 0 if eventLoop is NULL or preemption is disabled. */ +uint64_t aeGetHPPreemptCheckInterval(aeEventLoop *eventLoop) { + return eventLoop ? eventLoop->hp_preempt_check_interval_us : 0; +} + /* Preemptively processes high priority events if the elapsed time since the last poll - * exceeds the AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US threshold and iter count - * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK. */ + * exceeds the hp_preempt_check_interval_us threshold and iter count + * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK (0 disables preemption). */ int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { - if (eventLoop->hp_event_loop == NULL || iter_count == 0) return 0; + if (eventLoop->hp_event_loop == NULL || eventLoop->hp_preempt_check_interval_us == 0 || iter_count == 0) return 0; if ((iter_count & AE_HP_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { - if (elapsedUs(eventLoop->hp_last_poll_us) >= AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US) { + if (elapsedUs(eventLoop->hp_last_poll_us) >= eventLoop->hp_preempt_check_interval_us) { return aeProcessHPEventsNow(eventLoop); } } @@ -557,7 +572,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { for (j = 0; j < numevents; j++) { /* Periodic Preemptive Poll: - * If processing normal events is taking more than AE_HP_EVENT_PREEMPTIVE_CHECK_INTERVAL_US, check for new HP events. + * If processing normal events is taking more than hp_preempt_check_interval_us, check for new HP events. * Batch AE_HP_EVENT_PREEMPTIVE_CHECK_MASK checks to minimize monotonic clock call overhead. */ processed += aeProcessHPEventsPreemptively(eventLoop, j); diff --git a/src/ae.h b/src/ae.h index 9306030db4b..d79d83eddac 100644 --- a/src/ae.h +++ b/src/ae.h @@ -133,7 +133,8 @@ typedef struct aeEventLoop { * here are processed ahead of normal client traffic and periodically preempt * normal event iterations. NULL if unsupported by the OS multiplexer or disabled. */ struct aeEventLoop *hp_event_loop; - monotime hp_last_poll_us; /* Timestamp (microseconds) when hp_event_loop was last drained */ + monotime hp_last_poll_us; /* Timestamp (microseconds) when hp_event_loop was last drained */ + uint64_t hp_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ } aeEventLoop; /* Prototypes */ @@ -167,6 +168,10 @@ void aeSetDontWait(aeEventLoop *eventLoop, int noWait); * Returns the number of high-priority events processed. */ int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count); +/* Sets and gets the high-priority event preemption check interval in microseconds. */ +void aeSetHPPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); +uint64_t aeGetHPPreemptCheckInterval(aeEventLoop *eventLoop); + /* Links a nested high-priority event loop to the main event loop by registering the child loop's * multiplexer descriptor (e.g. epoll fd) on the parent loop with AE_READABLE. * Returns AE_OK on success or AE_ERR if multiplexer nesting is unsupported or registration fails. */ diff --git a/src/config.c b/src/config.c index 89f3e9e56a3..9b643c3cbc7 100644 --- a/src/config.c +++ b/src/config.c @@ -2677,6 +2677,16 @@ static int updateDefragConfiguration(const char **err) { return 1; } +/* Dynamic configuration apply callback for qos-preemptive-poll-interval-us. + * Updates the preemption threshold on the active event loop. */ +static int updateQoSPreemptivePollInterval(const char **err) { + UNUSED(err); + if (server.el) { + aeSetHPPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); + } + return 1; +} + static int updateJemallocBgThread(const char **err) { UNUSED(err); set_jemalloc_bg_thread(server.jemalloc_bg_thread); @@ -3508,6 +3518,7 @@ standardConfig static_configs[] = { createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, updateDefragConfiguration), /* Default: maximum defrag force at 100% fragmentation */ createIntConfig("active-defrag-cycle-us", NULL, MODIFIABLE_CONFIG, 0, 100000, server.active_defrag_cycle_us, 500, INTEGER_CONFIG, NULL, updateDefragConfiguration), + createIntConfig("qos-preemptive-poll-interval-us", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.qos_preemptive_poll_interval_us, 2000, INTEGER_CONFIG, NULL, updateQoSPreemptivePollInterval), createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_log_factor, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_decay_time, 1, INTEGER_CONFIG, NULL, NULL), createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.replica_priority, 100, INTEGER_CONFIG, NULL, NULL), diff --git a/src/server.c b/src/server.c index dc317e2285d..4a4da22d14e 100644 --- a/src/server.c +++ b/src/server.c @@ -3110,6 +3110,7 @@ void initServer(void) { aeDeleteEventLoop(hp_el); hp_el = NULL; } + aeSetHPPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); diff --git a/src/server.h b/src/server.h index 115d01f0419..dfc4da17b5a 100644 --- a/src/server.h +++ b/src/server.h @@ -2057,6 +2057,7 @@ struct valkeyServer { invocation of the event loop. */ unsigned int max_new_conns_per_cycle; /* The maximum number of tcp connections that will be accepted during each invocation of the event loop. */ + int qos_preemptive_poll_interval_us; /* High-priority event loop preemptive poll interval in microseconds */ /* AOF persistence */ int aof_enabled; /* AOF configuration */ int aof_state; /* AOF_(ON|OFF|WAIT_REWRITE) */ diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 0a083df6d6a..1e2f527e6e2 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -86,9 +86,9 @@ static int g_normal_cb_count = 0; static void testNormalEventCallbackMaskCheck(aeEventLoop *el, int fd, void *privdata, int mask) { testNormalEventCallback(el, fd, privdata, mask); g_normal_cb_count++; - /* Sleep > 1000 us on 3rd normal event (j = 2) and trigger 2nd HP pipe so (itr_cnt & 3) == 0 check at j = 4 polls hp_el */ + /* Sleep > 2000 us on 3rd normal event (j = 2) and trigger 2nd HP pipe so (itr_cnt & 3) == 0 check at j = 4 polls hp_el */ if (g_normal_cb_count == 3 && g_hp_pipe2_write_fd != -1) { - usleep(1500); + usleep(2500); char c = 'x'; if (write(g_hp_pipe2_write_fd, &c, 1) < 0) { } @@ -160,6 +160,21 @@ TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { ASSERT_NE(server.el->hp_event_loop, (aeEventLoop *)NULL); EXPECT_EQ(server.el->hp_event_loop, server.el->hp_event_loop); EXPECT_EQ(server.el->hp_event_loop->hp_event_loop, (aeEventLoop *)NULL); + EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 2000ULL); + aeSetHPPreemptCheckInterval(server.el, 5000ULL); + EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 5000ULL); + aeSetHPPreemptCheckInterval(server.el, 2000ULL); +} + +TEST_F(SocketPrioritizationTest, DynamicPreemptionIntervalThreshold) { + /* Test getter and setter */ + aeSetHPPreemptCheckInterval(server.el, 500ULL); + EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 500ULL); + + /* Test disabling preemption (interval = 0) */ + aeSetHPPreemptCheckInterval(server.el, 0ULL); + EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 0ULL); + aeSetHPPreemptCheckInterval(server.el, 2000ULL); } TEST_P(SocketPrioritizationConnTest, ConnectionPriorityMetadataAndHelpers) { diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl index 1690c6914fc..b0bd7b2544f 100644 --- a/tests/unit/socket-prioritization.tcl +++ b/tests/unit/socket-prioritization.tcl @@ -149,5 +149,17 @@ start_server {tags {"socket-prioritization external:skip"}} { foreach cl $load_clients { $cl close } } } + + test {Dynamic configuration of qos-preemptive-poll-interval-us} { + assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 2000 + r config set qos-preemptive-poll-interval-us 500 + assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 500 + r config set qos-preemptive-poll-interval-us 0 + assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 0 + assert_error "*argument must be between*" {r config set qos-preemptive-poll-interval-us -1} + assert_error "*argument couldn't be parsed into an integer*" {r config set qos-preemptive-poll-interval-us invalid} + r config set qos-preemptive-poll-interval-us 2000 + assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 2000 + } } } diff --git a/valkey.conf b/valkey.conf index e417bb547e6..0670d2b5d85 100644 --- a/valkey.conf +++ b/valkey.conf @@ -2771,6 +2771,34 @@ rdb-save-incremental-fsync yes # Jemalloc background thread for purging will be enabled by default jemalloc-bg-thread yes +################################ QUALITY OF SERVICE ############################### +# +# Valkey prioritizes system-critical internal connections (such as cluster bus +# heartbeats/gossip, primary-replica replication streams, and slot migration links) +# ahead of standard client traffic to maintain control-plane responsiveness and +# prevent false failovers under heavy command workloads. +# +# When the main eventloop processes large batches of normal client events (such as +# heavy pipelined commands or multi-key operations), batch execution can take +# several milliseconds. To prevent head-of-line blocking and ensure system-critical +# internal events are not starved, Valkey periodically interrupts normal events +# processing to check and drain pending high-priority events. +# +# The `qos-preemptive-poll-interval-us` configuration defines the maximum elapsed +# time (in microseconds) between preemptive poll of the high-priority events +# while iterating through normal client events. +# +# Tuning guidelines: +# - Smaller values (e.g., 500 to 1000 us) ensure tighter latency bounds for cluster +# heartbeats and replication under intense client query bursts. +# - Larger values (e.g., 5000 to 10000 us) maximize client query batching throughput. +# - Setting this value to 0 disables mid-batch preemption entirely; high-priority +# events will only be processed at the start of each event loop cycle. +# +# Default: 2000 microseconds (2 milliseconds) +# qos-preemptive-poll-interval-us 2000 + + ########################### HOT KEY DETECTION ################################# # Valkey can sample client key accesses and report the hottest keys through the From 8f20183c14899d99e0ec700f716c9cebe1903a6d Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Wed, 19 Aug 2026 18:38:08 +0000 Subject: [PATCH 04/14] Add QoS metrics for observability Signed-off-by: Satheesha Gowda --- src/ae.c | 14 ++++++++- src/ae.h | 5 +++ src/latency.h | 12 +++++--- src/server.c | 39 +++++++++++++++++++++-- src/server.h | 5 +++ src/unit/test_socket_prioritization.cpp | 41 +++++++++++++++++++++++++ tests/unit/info.tcl | 8 +++++ tests/unit/socket-prioritization.tcl | 19 ++++++++++++ 8 files changed, 135 insertions(+), 8 deletions(-) diff --git a/src/ae.c b/src/ae.c index a14c90a8215..1784b98d8cf 100644 --- a/src/ae.c +++ b/src/ae.c @@ -109,6 +109,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->hp_event_loop = NULL; eventLoop->hp_last_poll_us = 0; eventLoop->hp_preempt_check_interval_us = AE_HP_DEFAULT_PREEMPT_CHECK_INTERVAL_US; + eventLoop->hp_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -454,16 +455,27 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } -/* Process all high priority events immediately */ +/* Process all high priority events immediately and invoke the duration callback + * with the elapsed time in microseconds if registered. */ static int aeProcessHPEventsNow(aeEventLoop *eventLoop) { int processed = 0; if (eventLoop->hp_event_loop != NULL) { + monotime start = getMonotonicUs(); processed = aeProcessEvents(eventLoop->hp_event_loop, AE_ALL_EVENTS | AE_DONT_WAIT); eventLoop->hp_last_poll_us = getMonotonicUs(); + + if (eventLoop->hp_stats_callback != NULL) { + eventLoop->hp_stats_callback(eventLoop, eventLoop->hp_last_poll_us - start); + } } return processed; } +/* Set callback to receive elapsed duration of high-priority event processing */ +void aeSetHPStatsCallback(aeEventLoop *eventLoop, aeHPStatsProc *cb) { + if (eventLoop) eventLoop->hp_stats_callback = cb; +} + /* Set the preemptive poll interval in microseconds for high-priority events. * High-priority events are checked periodically during normal event processing * when processing batches of normal events exceeds this interval. diff --git a/src/ae.h b/src/ae.h index d79d83eddac..6518b96c8ec 100644 --- a/src/ae.h +++ b/src/ae.h @@ -83,6 +83,8 @@ typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientDat typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); typedef void aeAfterSleepProc(struct aeEventLoop *eventLoop, int numevents); typedef int aeCustomPollProc(struct aeEventLoop *eventLoop); +/* Callback invoked with elapsed microseconds after high-priority events are processed. */ +typedef void aeHPStatsProc(struct aeEventLoop *eventLoop, uint64_t duration_us); /* File event structure */ typedef struct aeFileEvent { @@ -135,6 +137,7 @@ typedef struct aeEventLoop { struct aeEventLoop *hp_event_loop; monotime hp_last_poll_us; /* Timestamp (microseconds) when hp_event_loop was last drained */ uint64_t hp_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ + aeHPStatsProc *hp_stats_callback; /* Callback invoked with elapsed microseconds after draining hp_event_loop */ } aeEventLoop; /* Prototypes */ @@ -171,6 +174,8 @@ int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count); /* Sets and gets the high-priority event preemption check interval in microseconds. */ void aeSetHPPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); uint64_t aeGetHPPreemptCheckInterval(aeEventLoop *eventLoop); +/* Sets the callback invoked with elapsed duration whenever high-priority events are processed. */ +void aeSetHPStatsCallback(aeEventLoop *eventLoop, aeHPStatsProc *cb); /* Links a nested high-priority event loop to the main event loop by registering the child loop's * multiplexer descriptor (e.g. epoll fd) on the parent loop with AE_READABLE. diff --git a/src/latency.h b/src/latency.h index a1e327bbba7..ff937e07ac0 100644 --- a/src/latency.h +++ b/src/latency.h @@ -103,11 +103,13 @@ typedef struct durationStats { } durationStats; typedef enum { - EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop - EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands - EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop - EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and - // AOF) + EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop + EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands + EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop + EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and + // AOF) + EL_DURATION_TYPE_QOS_EL, // cumulative time duration metric of QoS high-priority eventloop + EL_DURATION_TYPE_QOS_CMD, // cumulative time duration metric of QoS high-priority handler execution EL_DURATION_TYPE_NUM } DurationType; diff --git a/src/server.c b/src/server.c index 4a4da22d14e..c053ed4d824 100644 --- a/src/server.c +++ b/src/server.c @@ -1630,6 +1630,10 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa factor); trackInstantaneousMetric(STATS_METRIC_EL_DURATION, server.duration_stats[EL_DURATION_TYPE_EL].sum, server.duration_stats[EL_DURATION_TYPE_EL].cnt, 1); + trackInstantaneousMetric(STATS_METRIC_QOS_EL_CYCLE, server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, current_time, + factor); + trackInstantaneousMetric(STATS_METRIC_QOS_EL_DURATION, server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, + server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, 1); } cronUpdateMemoryStats(); @@ -2163,6 +2167,21 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { IOThreadsAfterSleep(numevents); } +/* Callback invoked by the event loop after draining high-priority events. + * Records QoS eventloop duration and updates peak commands executed per QoS cycle. */ +static void hpStatsCallback(struct aeEventLoop *el, uint64_t duration_us) { + UNUSED(el); + durationAddSample(EL_DURATION_TYPE_QOS_EL, duration_us); + unsigned long long qos_cmds = server.duration_stats[EL_DURATION_TYPE_QOS_CMD].cnt; + if (qos_cmds > (unsigned long long)server.qos_el_cmd_cnt_prev) { + long long el_cmd_cnt = qos_cmds - server.qos_el_cmd_cnt_prev; + if (el_cmd_cnt > server.qos_el_cmd_cnt_max) { + server.qos_el_cmd_cnt_max = el_cmd_cnt; + } + server.qos_el_cmd_cnt_prev = qos_cmds; + } +} + /* =========================== Server initialization ======================== */ static inline robj *createSharedString(const char *string) { @@ -2946,6 +2965,8 @@ void resetServerStats(void) { server.stat_reply_buffer_expands = 0; memset(server.duration_stats, 0, sizeof(durationStats) * EL_DURATION_TYPE_NUM); server.el_cmd_cnt_max = 0; + server.qos_el_cmd_cnt_max = 0; + server.qos_el_cmd_cnt_prev = 0; server.stat_active_time = 0; server.el_iteration_active = false; server.stat_total_prefetch_batches = 0; @@ -3110,6 +3131,7 @@ void initServer(void) { aeDeleteEventLoop(hp_el); hp_el = NULL; } + aeSetHPStatsCallback(server.el, hpStatsCallback); aeSetHPPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); @@ -4233,7 +4255,13 @@ void call(client *c, int flags) { } else { latencyTraceIfNeeded(server, command, duration); } - if (server.execution_nesting == 0) durationAddSample(EL_DURATION_TYPE_CMD, duration); + if (server.execution_nesting == 0) { + durationAddSample(EL_DURATION_TYPE_CMD, duration); + /* Attribute command execution latency for high-priority client connections. */ + if (c->conn && connGetPriority(c->conn) == CONN_PRIORITY_HIGH) { + durationAddSample(EL_DURATION_TYPE_QOS_CMD, duration); + } + } } /* Log the command into the commandlog if needed. @@ -6807,7 +6835,12 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_EL].sum, "eventloop_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CMD].sum, "instantaneous_eventloop_cycles_per_sec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_CYCLE), - "instantaneous_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_DURATION))); + "instantaneous_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_DURATION), + "qos_eventloop_cycles:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, + "qos_eventloop_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, + "qos_eventloop_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_CMD].sum, + "instantaneous_qos_eventloop_cycles_per_sec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_QOS_EL_CYCLE), + "instantaneous_qos_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_QOS_EL_DURATION))); info = genValkeyInfoStringACLStats(info); } @@ -7094,6 +7127,8 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_duration_cron_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CRON].sum, "eventloop_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_EL].max, "eventloop_cmd_per_cycle_max:%lld\r\n", server.el_cmd_cnt_max, + "qos_eventloop_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].max, + "qos_eventloop_cmd_per_cycle_max:%lld\r\n", server.qos_el_cmd_cnt_max, "io_threaded_reads_pending:%lld\r\n", server.stat_io_reads_pending, "io_threaded_writes_pending:%lld\r\n", server.stat_io_writes_pending)); diff --git a/src/server.h b/src/server.h index dfc4da17b5a..647a9203dd3 100644 --- a/src/server.h +++ b/src/server.h @@ -217,6 +217,8 @@ typedef enum { STATS_METRIC_EL_DURATION, /* Eventloop duration. */ STATS_METRIC_IO_WAIT, /* IO queue size */ STATS_METRIC_MAIN_THREAD_ACTIVE_TIME, /* Main-thread active time */ + STATS_METRIC_QOS_EL_CYCLE, /* Number of QoS high-priority eventloop cycles per sec. */ + STATS_METRIC_QOS_EL_DURATION, /* QoS high-priority eventloop duration in microseconds. */ STATS_METRIC_COUNT /* Total count */ } instantaneous_metric_type; @@ -2011,6 +2013,9 @@ struct valkeyServer { * Note that commands in transactions are also counted. */ long long el_cmd_cnt_start; long long el_cmd_cnt_max; + /* Record the previous baseline and peak number of high-priority commands executed in one QoS cycle. */ + long long qos_el_cmd_cnt_prev; + long long qos_el_cmd_cnt_max; /* The sum of active-expire, active-defrag and all other tasks done by cron and beforeSleep, but excluding read, write and AOF, which are counted by other sets of metrics. */ monotime el_cron_duration; diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 1e2f527e6e2..12d94d568d8 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -877,3 +877,44 @@ TEST(KqueueFairnessTest, KernelBehavior) { close(kq); } #endif + +static void qosMetricTestCb(struct aeEventLoop *el, uint64_t duration_us) { + (void)el; + durationAddSample(EL_DURATION_TYPE_QOS_EL, duration_us); +} + +static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { + (void)el; + (void)privdata; + (void)mask; + char b; + (void)read(fd, &b, 1); +} + +/* Test that high-priority event loop duration callback correctly samples QoS metrics. */ +TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { + aeEventLoop *main_loop = aeCreateEventLoop(64); + aeEventLoop *hp_loop = aeCreateEventLoop(64); + ASSERT_NE(main_loop, (aeEventLoop *)NULL); + ASSERT_NE(hp_loop, (aeEventLoop *)NULL); + ASSERT_EQ(aeLinkHighPriorityEventLoop(main_loop, hp_loop), AE_OK); + aeSetHPStatsCallback(main_loop, qosMetricTestCb); + + unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt; + unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum; + + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + ASSERT_EQ(aeCreateFileEvent(main_loop, fds[0], AE_READABLE | AE_HIGH_PRIORITY, qosTestFileProc, NULL), AE_OK); + + char b = 'x'; + ASSERT_EQ(write(fds[1], &b, 1), 1); + aeProcessEvents(main_loop, AE_DONT_WAIT | AE_ALL_EVENTS); + + EXPECT_GT(server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, orig_cnt); + EXPECT_GE(server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, orig_sum); + + close(fds[0]); + close(fds[1]); + aeDeleteEventLoop(main_loop); +} diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 86f3b4a7390..caf96423797 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -356,8 +356,12 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { # make sure debug info is hidden set info [r info] assert_equal [getInfoProperty $info eventloop_duration_aof_sum] {} + assert_equal [getInfoProperty $info qos_eventloop_duration_max] {} + assert_equal [getInfoProperty $info qos_eventloop_cmd_per_cycle_max] {} set info_all [r info all] assert_equal [getInfoProperty $info_all eventloop_duration_aof_sum] {} + assert_equal [getInfoProperty $info_all qos_eventloop_duration_max] {} + assert_equal [getInfoProperty $info_all qos_eventloop_cmd_per_cycle_max] {} set info1 [r info debug] @@ -369,6 +373,10 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { assert {$cycle_max1 > 0} set duration_max1 [getInfoProperty $info1 eventloop_duration_max] assert {$duration_max1 > 0} + set qos_duration_max1 [getInfoProperty $info1 qos_eventloop_duration_max] + assert {$qos_duration_max1 >= 0} + set qos_cycle_max1 [getInfoProperty $info1 qos_eventloop_cmd_per_cycle_max] + assert {$qos_cycle_max1 >= 0} after 110 ;# hz is 10, wait for a cron tick. set info2 [r info debug] diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl index b0bd7b2544f..9d588f2e6f0 100644 --- a/tests/unit/socket-prioritization.tcl +++ b/tests/unit/socket-prioritization.tcl @@ -161,5 +161,24 @@ start_server {tags {"socket-prioritization external:skip"}} { r config set qos-preemptive-poll-interval-us 2000 assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 2000 } + + test {Socket Prioritization: INFO stats and debug QoS metrics} { + set info_stats [$primary info stats] + set info_debug [$primary info debug] + + assert_morethan [getInfoProperty $info_stats qos_eventloop_cycles] 0 + assert_morethan [getInfoProperty $info_stats qos_eventloop_duration_sum] 0 + assert_morethan [getInfoProperty $info_stats qos_eventloop_duration_cmd_sum] 0 + assert {[getInfoProperty $info_debug qos_eventloop_duration_max] >= 0} + assert {[getInfoProperty $info_debug qos_eventloop_cmd_per_cycle_max] >= 0} + + # Reset stats and verify + $primary config resetstat + set info_stats_reset [$primary info stats] + assert_equal [getInfoProperty $info_stats_reset qos_eventloop_cycles] 0 + assert_equal [getInfoProperty $info_stats_reset qos_eventloop_duration_sum] 0 + assert_equal [getInfoProperty $info_stats_reset qos_eventloop_duration_cmd_sum] 0 + assert_equal [getInfoProperty [$primary info debug] qos_eventloop_cmd_per_cycle_max] 0 + } } } From 1f296af5407da5e05240e1cca37cae544a1b4d4d Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Wed, 19 Aug 2026 19:19:54 +0000 Subject: [PATCH 05/14] Refactored the kwyword High Priority to QoS to be consistent with the feature Signed-off-by: Satheesha Gowda --- src/ae.c | 160 +++++++++---------- src/ae.h | 38 ++--- src/cluster_legacy.c | 4 +- src/config.c | 2 +- src/connection.c | 12 +- src/connection.h | 2 +- src/io_threads.c | 6 +- src/latency.h | 4 +- src/replication.c | 4 +- src/server.c | 24 +-- src/server.h | 8 +- src/unit/test_socket_prioritization.cpp | 196 ++++++++++++------------ tests/unit/socket-prioritization.tcl | 6 +- 13 files changed, 234 insertions(+), 232 deletions(-) diff --git a/src/ae.c b/src/ae.c index 1784b98d8cf..043aa5d7ab4 100644 --- a/src/ae.c +++ b/src/ae.c @@ -78,14 +78,14 @@ * implementations are AE_READABLE & AE_WRITABLE. */ #define BACKEND_MASK(mask) ((mask) & (AE_READABLE | AE_WRITABLE)) -/* High priority event loop periodic preemptive poll default interval in microseconds. - * High-priority events are checked periodically during normal event loops to prevent +/* QoS event loop periodic preemptive poll default interval in microseconds. + * QoS events are checked periodically during normal event loops to prevent * long batches of normal client commands from starving control plane traffic. */ -#define AE_HP_DEFAULT_PREEMPT_CHECK_INTERVAL_US 2000 +#define AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US 2000 -/* High priority event loop periodic preemptive poll mask. +/* QoS event loop periodic preemptive poll mask. * Used to amortize getMonotonicUs() clock reads across every 4th iteration ((iter & 3) == 0). */ -#define AE_HP_EVENT_PREEMPTIVE_CHECK_MASK 3 +#define AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK 3 aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; @@ -106,10 +106,10 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->aftersleep = NULL; eventLoop->custompoll = NULL; eventLoop->flags = 0; - eventLoop->hp_event_loop = NULL; - eventLoop->hp_last_poll_us = 0; - eventLoop->hp_preempt_check_interval_us = AE_HP_DEFAULT_PREEMPT_CHECK_INTERVAL_US; - eventLoop->hp_stats_callback = NULL; + eventLoop->qos_el = NULL; + eventLoop->qos_el_last_poll_us = 0; + eventLoop->qos_el_preempt_check_interval_us = AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US; + eventLoop->qos_el_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -164,8 +164,8 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) goto err; if (aeApiResize(eventLoop->apidata, setsize) == -1) goto err; - if (eventLoop->hp_event_loop) { - if (aeResizeSetSize(eventLoop->hp_event_loop, setsize) == AE_ERR) goto err; + if (eventLoop->qos_el) { + if (aeResizeSetSize(eventLoop->qos_el, setsize) == AE_ERR) goto err; } eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * setsize); @@ -185,10 +185,10 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { } void aeDeleteEventLoop(aeEventLoop *eventLoop) { - if (eventLoop->hp_event_loop) { - int hp_fd = aeApiGetPollFd(eventLoop->hp_event_loop); - if (hp_fd != -1) aeDeleteFileEvent(eventLoop, hp_fd, AE_READABLE); - aeDeleteEventLoop(eventLoop->hp_event_loop); + if (eventLoop->qos_el) { + int qos_fd = aeApiGetPollFd(eventLoop->qos_el); + if (qos_fd != -1) aeDeleteFileEvent(eventLoop, qos_fd, AE_READABLE); + aeDeleteEventLoop(eventLoop->qos_el); } aeApiFree(eventLoop->apidata); zfree(eventLoop->events); @@ -211,11 +211,11 @@ void aeStop(aeEventLoop *eventLoop) { int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc, void *clientData) { if (mask & AE_HIGH_PRIORITY) { - if (eventLoop->hp_event_loop) { - /* Register events on to high priority event loop */ - return aeCreateFileEvent(eventLoop->hp_event_loop, fd, mask & ~AE_HIGH_PRIORITY, proc, clientData); + if (eventLoop->qos_el) { + /* Register events on to QoS event loop */ + return aeCreateFileEvent(eventLoop->qos_el, fd, mask & ~AE_HIGH_PRIORITY, proc, clientData); } - /* If hp_event_loop is not initialized, strip the high priority flag */ + /* If qos_el is not initialized, strip the high priority flag */ mask &= ~AE_HIGH_PRIORITY; } AE_LOCK(eventLoop); @@ -245,11 +245,11 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { - if (eventLoop->hp_event_loop) { - /* Check if the event exists in HP loop first */ - int hp_mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); - if (hp_mask != AE_NONE) { - aeDeleteFileEvent(eventLoop->hp_event_loop, fd, mask); + if (eventLoop->qos_el) { + /* Check if the event exists in QoS loop first */ + int qos_mask = aeGetFileEvents(eventLoop->qos_el, fd); + if (qos_mask != AE_NONE) { + aeDeleteFileEvent(eventLoop->qos_el, fd, mask); return; } } @@ -291,10 +291,10 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { } void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { - if (eventLoop->hp_event_loop) { - int mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); + if (eventLoop->qos_el) { + int mask = aeGetFileEvents(eventLoop->qos_el, fd); if (mask != AE_NONE) { - return aeGetFileClientData(eventLoop->hp_event_loop, fd); + return aeGetFileClientData(eventLoop->qos_el, fd); } } if (fd >= eventLoop->setsize) return NULL; @@ -305,8 +305,8 @@ void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { } int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { - if (eventLoop->hp_event_loop) { - int mask = aeGetFileEvents(eventLoop->hp_event_loop, fd); + if (eventLoop->qos_el) { + int mask = aeGetFileEvents(eventLoop->qos_el, fd); if (mask != AE_NONE) { return mask | AE_HIGH_PRIORITY; } @@ -455,49 +455,49 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } -/* Process all high priority events immediately and invoke the duration callback +/* Process all QoS events immediately and invoke the duration callback * with the elapsed time in microseconds if registered. */ -static int aeProcessHPEventsNow(aeEventLoop *eventLoop) { +static int aeProcessQoSEventsNow(aeEventLoop *eventLoop) { int processed = 0; - if (eventLoop->hp_event_loop != NULL) { + if (eventLoop->qos_el != NULL) { monotime start = getMonotonicUs(); - processed = aeProcessEvents(eventLoop->hp_event_loop, AE_ALL_EVENTS | AE_DONT_WAIT); - eventLoop->hp_last_poll_us = getMonotonicUs(); + processed = aeProcessEvents(eventLoop->qos_el, AE_ALL_EVENTS | AE_DONT_WAIT); + eventLoop->qos_el_last_poll_us = getMonotonicUs(); - if (eventLoop->hp_stats_callback != NULL) { - eventLoop->hp_stats_callback(eventLoop, eventLoop->hp_last_poll_us - start); + if (eventLoop->qos_el_stats_callback != NULL) { + eventLoop->qos_el_stats_callback(eventLoop, eventLoop->qos_el_last_poll_us - start); } } return processed; } -/* Set callback to receive elapsed duration of high-priority event processing */ -void aeSetHPStatsCallback(aeEventLoop *eventLoop, aeHPStatsProc *cb) { - if (eventLoop) eventLoop->hp_stats_callback = cb; +/* Set callback to receive elapsed duration of QoS event processing */ +void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb) { + if (eventLoop) eventLoop->qos_el_stats_callback = cb; } -/* Set the preemptive poll interval in microseconds for high-priority events. - * High-priority events are checked periodically during normal event processing +/* Set the preemptive poll interval in microseconds for QoS events. + * QoS events are checked periodically during normal event processing * when processing batches of normal events exceeds this interval. * Setting this interval to 0 disables preemptive polling. */ -void aeSetHPPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { - if (eventLoop) eventLoop->hp_preempt_check_interval_us = interval_us; +void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { + if (eventLoop) eventLoop->qos_el_preempt_check_interval_us = interval_us; } -/* Get the current preemptive poll interval in microseconds for high-priority events. +/* Get the current preemptive poll interval in microseconds for QoS events. * Returns 0 if eventLoop is NULL or preemption is disabled. */ -uint64_t aeGetHPPreemptCheckInterval(aeEventLoop *eventLoop) { - return eventLoop ? eventLoop->hp_preempt_check_interval_us : 0; +uint64_t aeGetQoSPreemptCheckInterval(aeEventLoop *eventLoop) { + return eventLoop ? eventLoop->qos_el_preempt_check_interval_us : 0; } -/* Preemptively processes high priority events if the elapsed time since the last poll - * exceeds the hp_preempt_check_interval_us threshold and iter count - * is a multiple of AE_HP_EVENT_PREEMPTIVE_CHECK_MASK (0 disables preemption). */ -int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { - if (eventLoop->hp_event_loop == NULL || eventLoop->hp_preempt_check_interval_us == 0 || iter_count == 0) return 0; - if ((iter_count & AE_HP_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { - if (elapsedUs(eventLoop->hp_last_poll_us) >= eventLoop->hp_preempt_check_interval_us) { - return aeProcessHPEventsNow(eventLoop); +/* Preemptively processes QoS events if the elapsed time since the last poll + * exceeds the qos_el_preempt_check_interval_us threshold and iter count + * is a multiple of AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK (0 disables preemption). */ +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { + if (eventLoop->qos_el == NULL || eventLoop->qos_el_preempt_check_interval_us == 0 || iter_count == 0) return 0; + if ((iter_count & AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { + if (elapsedUs(eventLoop->qos_el_last_poll_us) >= eventLoop->qos_el_preempt_check_interval_us) { + return aeProcessQoSEventsNow(eventLoop); } } return 0; @@ -567,15 +567,15 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); - /* Prioritize high-priority event loop: if hp_fd fired, drain HP events + /* Prioritize QoS event loop: if qos_fd fired, drain QoS events * immediately before processing normal events. */ - int hp_fd = -1; - if (eventLoop->hp_event_loop != NULL) { - hp_fd = aeApiGetPollFd(eventLoop->hp_event_loop); - if (hp_fd != -1) { + int qos_fd = -1; + if (eventLoop->qos_el != NULL) { + qos_fd = aeApiGetPollFd(eventLoop->qos_el); + if (qos_fd != -1) { for (j = 0; j < numevents; j++) { - if (eventLoop->fired[j].fd == hp_fd) { - processed += aeProcessHPEventsNow(eventLoop); + if (eventLoop->fired[j].fd == qos_fd) { + processed += aeProcessQoSEventsNow(eventLoop); break; } } @@ -584,13 +584,13 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { for (j = 0; j < numevents; j++) { /* Periodic Preemptive Poll: - * If processing normal events is taking more than hp_preempt_check_interval_us, check for new HP events. - * Batch AE_HP_EVENT_PREEMPTIVE_CHECK_MASK checks to minimize monotonic clock call overhead. */ - processed += aeProcessHPEventsPreemptively(eventLoop, j); + * If processing normal events is taking more than qos_el_preempt_check_interval_us, check for new QoS events. + * Batch AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK checks to minimize monotonic clock call overhead. */ + processed += aeProcessQoSEventsPreemptively(eventLoop, j); int fd = eventLoop->fired[j].fd; - /* Skip processing HP events here again as they were already processed above */ - if (fd == hp_fd) continue; + /* Skip processing QoS events here again as they were already processed above */ + if (fd == qos_fd) continue; aeFileEvent *fe = &eventLoop->events[fd]; int mask = eventLoop->fired[j].mask; int fired = 0; /* Number of events fired for current fd. */ @@ -702,28 +702,28 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { } } -/* Event handler for the nested high-priority event loop poll, - * which will immediately process high priority events registered on the high-priority event loop file descriptor.*/ -static void hpEventLoopHandler(aeEventLoop *el, int fd, void *privdata, int mask) { +/* Event handler for the nested QoS event loop poll, + * which will immediately process QoS events registered on the QoS event loop file descriptor.*/ +static void qosEventLoopHandler(aeEventLoop *el, int fd, void *privdata, int mask) { AE_NOTUSED(fd); AE_NOTUSED(privdata); AE_NOTUSED(mask); - aeProcessHPEventsNow(el); + aeProcessQoSEventsNow(el); } -/* Register a high-priority event loop file descriptor in the main event loop, - * which will awake main eventloop when the high-priority event loop file descriptor is fired. */ -int aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop) { - if (eventLoop == NULL || hp_event_loop == NULL) return AE_ERR; - int hp_fd = aeApiGetPollFd(hp_event_loop); - if (hp_fd == -1) { - eventLoop->hp_event_loop = NULL; +/* Register a QoS event loop file descriptor in the main event loop, + * which will awake main eventloop when the QoS event loop file descriptor is fired. */ +int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el) { + if (eventLoop == NULL || qos_el == NULL) return AE_ERR; + int qos_fd = aeApiGetPollFd(qos_el); + if (qos_fd == -1) { + eventLoop->qos_el = NULL; return AE_ERR; } - if (aeCreateFileEvent(eventLoop, hp_fd, AE_READABLE, hpEventLoopHandler, hp_event_loop) == AE_ERR) { - eventLoop->hp_event_loop = NULL; + if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, qosEventLoopHandler, qos_el) == AE_ERR) { + eventLoop->qos_el = NULL; return AE_ERR; } - eventLoop->hp_event_loop = hp_event_loop; + eventLoop->qos_el = qos_el; return AE_OK; } diff --git a/src/ae.h b/src/ae.h index 6518b96c8ec..0fba3df6ae0 100644 --- a/src/ae.h +++ b/src/ae.h @@ -48,7 +48,7 @@ things to disk before sending replies, and want \ to do that in a group fashion. */ #define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ - * the event is registered on hp_event_loop if available. \ + * the event is registered on qos_el if available. \ * Stripped before passing to the underlying OS multiplexer. */ #define AE_FILE_EVENTS (1 << 0) @@ -83,8 +83,8 @@ typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientDat typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); typedef void aeAfterSleepProc(struct aeEventLoop *eventLoop, int numevents); typedef int aeCustomPollProc(struct aeEventLoop *eventLoop); -/* Callback invoked with elapsed microseconds after high-priority events are processed. */ -typedef void aeHPStatsProc(struct aeEventLoop *eventLoop, uint64_t duration_us); +/* Callback invoked with elapsed microseconds after QoS events are processed. */ +typedef void aeQoSStatsProc(struct aeEventLoop *eventLoop, uint64_t duration_us); /* File event structure */ typedef struct aeFileEvent { @@ -129,15 +129,15 @@ typedef struct aeEventLoop { pthread_mutex_t poll_mutex; int flags; - /* High-Priority Quality of Service (QoS): - * hp_event_loop points to a secondary, nested event loop dedicated to critical + /* Quality of Service (QoS): + * qos_el points to a secondary, nested event loop dedicated to critical * internal channels (cluster bus, slot migration, replication). Sockets registered * here are processed ahead of normal client traffic and periodically preempt * normal event iterations. NULL if unsupported by the OS multiplexer or disabled. */ - struct aeEventLoop *hp_event_loop; - monotime hp_last_poll_us; /* Timestamp (microseconds) when hp_event_loop was last drained */ - uint64_t hp_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ - aeHPStatsProc *hp_stats_callback; /* Callback invoked with elapsed microseconds after draining hp_event_loop */ + struct aeEventLoop *qos_el; + monotime qos_el_last_poll_us; /* Timestamp (microseconds) when qos_el was last drained */ + uint64_t qos_el_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ + aeQoSStatsProc *qos_el_stats_callback; /* Callback invoked with elapsed microseconds after draining qos_el */ } aeEventLoop; /* Prototypes */ @@ -167,19 +167,19 @@ int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); -/* Preemptively processes pending high-priority events during long-running normal event loops. - * Returns the number of high-priority events processed. */ -int aeProcessHPEventsPreemptively(aeEventLoop *eventLoop, int iter_count); +/* Preemptively processes pending QoS events during long-running normal event loops. + * Returns the number of QoS events processed. */ +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop, int iter_count); -/* Sets and gets the high-priority event preemption check interval in microseconds. */ -void aeSetHPPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); -uint64_t aeGetHPPreemptCheckInterval(aeEventLoop *eventLoop); -/* Sets the callback invoked with elapsed duration whenever high-priority events are processed. */ -void aeSetHPStatsCallback(aeEventLoop *eventLoop, aeHPStatsProc *cb); +/* Sets and gets the QoS event preemption check interval in microseconds. */ +void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); +uint64_t aeGetQoSPreemptCheckInterval(aeEventLoop *eventLoop); +/* Sets the callback invoked with elapsed duration whenever QoS events are processed. */ +void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb); -/* Links a nested high-priority event loop to the main event loop by registering the child loop's +/* Links a nested QoS event loop to the main event loop by registering the child loop's * multiplexer descriptor (e.g. epoll fd) on the parent loop with AE_READABLE. * Returns AE_OK on success or AE_ERR if multiplexer nesting is unsupported or registration fails. */ -int aeLinkHighPriorityEventLoop(aeEventLoop *eventLoop, aeEventLoop *hp_event_loop); +int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el); #endif diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index cb5c0f17544..eeed8bebbb1 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2053,7 +2053,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); /* Tag inbound cluster bus link as high-priority so cluster gossip and heartbeats - * are processed in hp_event_loop ahead of normal client traffic. */ + * are processed in qos_el ahead of normal client traffic. */ connSetPriority(conn, CONN_PRIORITY_HIGH); /* Mark as cluster-owned before any TLS accept retries so generic * accept offload routing can safely avoid client assumptions. */ @@ -6681,7 +6681,7 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long clusterLink *link = createClusterLink(node); link->conn = connCreate(connTypeOfCluster()); /* Tag outbound cluster bus link as high-priority so node reconnects, gossip ping/pong, - * and failure detection heartbeats operate within hp_event_loop. */ + * and failure detection heartbeats operate within qos_el. */ connSetPriority(link->conn, CONN_PRIORITY_HIGH); connSetPrivateData(link->conn, link); connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); diff --git a/src/config.c b/src/config.c index 9b643c3cbc7..12555a68852 100644 --- a/src/config.c +++ b/src/config.c @@ -2682,7 +2682,7 @@ static int updateDefragConfiguration(const char **err) { static int updateQoSPreemptivePollInterval(const char **err) { UNUSED(err); if (server.el) { - aeSetHPPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); + aeSetQoSPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); } return 1; } diff --git a/src/connection.c b/src/connection.c index dd45675c68e..b9d796c9c6c 100644 --- a/src/connection.c +++ b/src/connection.c @@ -208,13 +208,13 @@ int connSetPriority(connection *conn, int priority) { conn->type->update_state(conn); } else { int barrier = mask & AE_BARRIER; - int ae_hp_flag = connGetAEPriorityFlag(conn); + int ae_qos_flag = connGetAEPriorityFlag(conn); if ((mask & AE_READABLE) && - aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | ae_hp_flag, conn->type->ae_handler, conn) == AE_ERR) { + aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | ae_qos_flag, conn->type->ae_handler, conn) == AE_ERR) { goto rollback; } if ((mask & AE_WRITABLE) && - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | ae_hp_flag, conn->type->ae_handler, conn) == AE_ERR) { + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | ae_qos_flag, conn->type->ae_handler, conn) == AE_ERR) { goto rollback; } } @@ -227,11 +227,11 @@ int connSetPriority(connection *conn, int priority) { /* Rollback priority and restore previous event registrations */ conn->priority = old_priority; int barrier = mask & AE_BARRIER; - int old_ae_hp_flag = connGetAEPriorityFlag(conn); + int old_ae_qos_flag = connGetAEPriorityFlag(conn); if (mask & AE_READABLE) - aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | old_ae_hp_flag, conn->type->ae_handler, conn); + aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | old_ae_qos_flag, conn->type->ae_handler, conn); if (mask & AE_WRITABLE) - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | old_ae_hp_flag, conn->type->ae_handler, conn); + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | old_ae_qos_flag, conn->type->ae_handler, conn); return C_ERR; } diff --git a/src/connection.h b/src/connection.h index 858d375310b..4db999c72a6 100644 --- a/src/connection.h +++ b/src/connection.h @@ -178,7 +178,7 @@ typedef struct ConnectionType { /* Connection scheduling priority levels. * * High-priority connections receive preferential treatment across two subsystems: - * 1. HP Event loop (ae): Connections that are registered with hp_event_loop are + * 1. QoS Event loop (ae): Connections that are registered with qos_el are * processed ahead of normal priority events in the event polling loop. * 2. I/O threads: Read and write tasks are dispatched to dedicated * high-priority queues (io_shared_inbox[CONN_PRIORITY_HIGH] and io_shared_outbox[CONN_PRIORITY_HIGH]), diff --git a/src/io_threads.c b/src/io_threads.c index 917723ae184..2b0f8ae9f61 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -1188,7 +1188,7 @@ static int processOutboxBatch(mpscQueue *outbox) { /* Process completed IO jobs from worker threads back onto the main thread. * Drains the high-priority outbox first to guarantee control-plane responsiveness, - * and performs periodic preemptive polling of hp_event_loop while consuming normal jobs. */ + * and performs periodic preemptive polling of qos_el while consuming normal jobs. */ int processIOThreadsResponses(void) { /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ @@ -1203,8 +1203,8 @@ int processIOThreadsResponses(void) { int processed = processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_HIGH]); if (processed == 0) { /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new - * high-priority events arrived on hp_event_loop before processing normal traffic. */ - aeProcessHPEventsPreemptively(server.el, counter); + * high-priority events arrived on qos_el before processing normal traffic. */ + aeProcessQoSEventsPreemptively(server.el, counter); } /* 3. Drain normal client events */ processed += processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_NORMAL]); diff --git a/src/latency.h b/src/latency.h index ff937e07ac0..af0c0285579 100644 --- a/src/latency.h +++ b/src/latency.h @@ -108,8 +108,8 @@ typedef enum { EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and // AOF) - EL_DURATION_TYPE_QOS_EL, // cumulative time duration metric of QoS high-priority eventloop - EL_DURATION_TYPE_QOS_CMD, // cumulative time duration metric of QoS high-priority handler execution + EL_DURATION_TYPE_QOS_EL, // cumulative time duration metric of QoS eventloop + EL_DURATION_TYPE_QOS_CMD, // cumulative time duration metric of QoS handler execution EL_DURATION_TYPE_NUM } DurationType; diff --git a/src/replication.c b/src/replication.c index 9b3c827f0ba..98cc046d355 100644 --- a/src/replication.c +++ b/src/replication.c @@ -4393,7 +4393,7 @@ void syncWithPrimary(connection *conn) { /* Create RDB connection */ server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); /* Tag connection as high-priority before connecting so non-blocking connect and - * subsequent RDB transfer events are registered directly on hp_event_loop. */ + * subsequent RDB transfer events are registered directly on qos_el. */ connSetPriority(server.repl_rdb_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { @@ -4446,7 +4446,7 @@ void syncWithPrimary(connection *conn) { int connectWithPrimary(void) { server.repl_transfer_s = connCreate(connTypeOfReplication()); /* Tag main replication connection as high-priority before connecting so handshake, - * heartbeat pings, and PSYNC streaming are processed in hp_event_loop. */ + * heartbeat pings, and PSYNC streaming are processed in qos_el. */ connSetPriority(server.repl_transfer_s, CONN_PRIORITY_HIGH); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { diff --git a/src/server.c b/src/server.c index c053ed4d824..931d431dab4 100644 --- a/src/server.c +++ b/src/server.c @@ -2167,9 +2167,9 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { IOThreadsAfterSleep(numevents); } -/* Callback invoked by the event loop after draining high-priority events. +/* Callback invoked by the event loop after draining QoS events. * Records QoS eventloop duration and updates peak commands executed per QoS cycle. */ -static void hpStatsCallback(struct aeEventLoop *el, uint64_t duration_us) { +static void qosStatsCallback(struct aeEventLoop *el, uint64_t duration_us) { UNUSED(el); durationAddSample(EL_DURATION_TYPE_QOS_EL, duration_us); unsigned long long qos_cmds = server.duration_stats[EL_DURATION_TYPE_QOS_CMD].cnt; @@ -3116,23 +3116,23 @@ void initServer(void) { const char *clk_msg = monotonicInit(); serverLog(LL_NOTICE, "monotonic clock: %s", clk_msg); server.el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - /* Create high-priority event loop for internal connections (cluster bus, replication, and slot migration jobs) */ - aeEventLoop *hp_el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - if (server.el == NULL || hp_el == NULL) { + /* Create QoS event loop for internal connections (cluster bus, replication, and slot migration jobs) */ + aeEventLoop *qos_el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); + if (server.el == NULL || qos_el == NULL) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - /* Link high priority event loop with main event loop via nested multiplexer polling. + /* Link QoS event loop with main event loop via nested multiplexer polling. * If multiplexer nesting is unsupported (e.g. evport, select), gracefully - * free hp_el and fall back to main event loop event processing without QoS. */ - if (aeLinkHighPriorityEventLoop(server.el, hp_el) == AE_ERR) { + * free qos_el and fall back to main event loop event processing without QoS. */ + if (aeLinkQoSEventLoop(server.el, qos_el) == AE_ERR) { serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", aeGetApiName()); - aeDeleteEventLoop(hp_el); - hp_el = NULL; + aeDeleteEventLoop(qos_el); + qos_el = NULL; } - aeSetHPStatsCallback(server.el, hpStatsCallback); - aeSetHPPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); + aeSetQoSStatsCallback(server.el, qosStatsCallback); + aeSetQoSPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); diff --git a/src/server.h b/src/server.h index 647a9203dd3..445fab6ab83 100644 --- a/src/server.h +++ b/src/server.h @@ -217,8 +217,8 @@ typedef enum { STATS_METRIC_EL_DURATION, /* Eventloop duration. */ STATS_METRIC_IO_WAIT, /* IO queue size */ STATS_METRIC_MAIN_THREAD_ACTIVE_TIME, /* Main-thread active time */ - STATS_METRIC_QOS_EL_CYCLE, /* Number of QoS high-priority eventloop cycles per sec. */ - STATS_METRIC_QOS_EL_DURATION, /* QoS high-priority eventloop duration in microseconds. */ + STATS_METRIC_QOS_EL_CYCLE, /* Number of QoS eventloop cycles per sec. */ + STATS_METRIC_QOS_EL_DURATION, /* QoS eventloop duration in microseconds. */ STATS_METRIC_COUNT /* Total count */ } instantaneous_metric_type; @@ -2013,7 +2013,7 @@ struct valkeyServer { * Note that commands in transactions are also counted. */ long long el_cmd_cnt_start; long long el_cmd_cnt_max; - /* Record the previous baseline and peak number of high-priority commands executed in one QoS cycle. */ + /* Record the previous baseline and peak number of QoS commands executed in one QoS cycle. */ long long qos_el_cmd_cnt_prev; long long qos_el_cmd_cnt_max; /* The sum of active-expire, active-defrag and all other tasks done by cron and beforeSleep, @@ -2062,7 +2062,7 @@ struct valkeyServer { invocation of the event loop. */ unsigned int max_new_conns_per_cycle; /* The maximum number of tcp connections that will be accepted during each invocation of the event loop. */ - int qos_preemptive_poll_interval_us; /* High-priority event loop preemptive poll interval in microseconds */ + int qos_preemptive_poll_interval_us; /* QoS event loop preemptive poll interval in microseconds */ /* AOF persistence */ int aof_enabled; /* AOF configuration */ int aof_state; /* AOF_(ON|OFF|WAIT_REWRITE) */ diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 12d94d568d8..29b4e8e603f 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -67,12 +67,12 @@ static void testNormalEventCallback(aeEventLoop *el, int fd, void *privdata, int } } -static void testHighPriorityEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { +static void testQoSEventCallback(aeEventLoop *el, int fd, void *privdata, int mask) { UNUSED(el); UNUSED(mask); UNUSED(privdata); if (g_execution_count < 1024) { - /* Use 999 to identify high priority execution */ + /* Use 999 to identify QoS execution */ g_execution_order[g_execution_count++] = 999; } char buf[1]; @@ -81,16 +81,16 @@ static void testHighPriorityEventCallback(aeEventLoop *el, int fd, void *privdat } } -static int g_hp_pipe2_write_fd = -1; +static int g_qos_pipe2_write_fd = -1; static int g_normal_cb_count = 0; static void testNormalEventCallbackMaskCheck(aeEventLoop *el, int fd, void *privdata, int mask) { testNormalEventCallback(el, fd, privdata, mask); g_normal_cb_count++; - /* Sleep > 2000 us on 3rd normal event (j = 2) and trigger 2nd HP pipe so (itr_cnt & 3) == 0 check at j = 4 polls hp_el */ - if (g_normal_cb_count == 3 && g_hp_pipe2_write_fd != -1) { + /* Sleep > 2000 us on 3rd normal event (j = 2) and trigger 2nd QoS pipe so (itr_cnt & 3) == 0 check at j = 4 polls qos_el */ + if (g_normal_cb_count == 3 && g_qos_pipe2_write_fd != -1) { usleep(2500); char c = 'x'; - if (write(g_hp_pipe2_write_fd, &c, 1) < 0) { + if (write(g_qos_pipe2_write_fd, &c, 1) < 0) { } } } @@ -133,9 +133,9 @@ class SocketPrioritizationTest : public ::testing::Test { } server.el = aeCreateEventLoop(1024); - aeEventLoop *hp_el = aeCreateEventLoop(1024); - if (server.el && hp_el) { - aeLinkHighPriorityEventLoop(server.el, hp_el); + aeEventLoop *qos_el = aeCreateEventLoop(1024); + if (server.el && qos_el) { + aeLinkQoSEventLoop(server.el, qos_el); } g_execution_count = 0; } @@ -157,24 +157,24 @@ class SocketPrioritizationConnTest : public SocketPrioritizationTest, public ::t TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->hp_event_loop, (aeEventLoop *)NULL); - EXPECT_EQ(server.el->hp_event_loop, server.el->hp_event_loop); - EXPECT_EQ(server.el->hp_event_loop->hp_event_loop, (aeEventLoop *)NULL); - EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 2000ULL); - aeSetHPPreemptCheckInterval(server.el, 5000ULL); - EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 5000ULL); - aeSetHPPreemptCheckInterval(server.el, 2000ULL); + ASSERT_NE(server.el->qos_el, (aeEventLoop *)NULL); + EXPECT_EQ(server.el->qos_el, server.el->qos_el); + EXPECT_EQ(server.el->qos_el->qos_el, (aeEventLoop *)NULL); + EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 2000ULL); + aeSetQoSPreemptCheckInterval(server.el, 5000ULL); + EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 5000ULL); + aeSetQoSPreemptCheckInterval(server.el, 2000ULL); } TEST_F(SocketPrioritizationTest, DynamicPreemptionIntervalThreshold) { /* Test getter and setter */ - aeSetHPPreemptCheckInterval(server.el, 500ULL); - EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 500ULL); + aeSetQoSPreemptCheckInterval(server.el, 500ULL); + EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 500ULL); /* Test disabling preemption (interval = 0) */ - aeSetHPPreemptCheckInterval(server.el, 0ULL); - EXPECT_EQ(aeGetHPPreemptCheckInterval(server.el), 0ULL); - aeSetHPPreemptCheckInterval(server.el, 2000ULL); + aeSetQoSPreemptCheckInterval(server.el, 0ULL); + EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 0ULL); + aeSetQoSPreemptCheckInterval(server.el, 2000ULL); } TEST_P(SocketPrioritizationConnTest, ConnectionPriorityMetadataAndHelpers) { @@ -214,9 +214,9 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { ret = connSetReadHandler(conn, dummyConnectionHandler); EXPECT_EQ(ret, C_OK); - /* Event should exist in normal loop, but not in high priority loop */ + /* Event should exist in normal loop, but not in QoS loop */ EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); - EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); /* Dynamically upgrade connection to high priority */ EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_HIGH), C_OK); @@ -224,22 +224,22 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { int events = aeGetFileEvents(server.el, read_fd); EXPECT_NE(events & AE_READABLE, 0); EXPECT_NE(events & AE_HIGH_PRIORITY, 0); - EXPECT_NE(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + EXPECT_NE(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); /* Dynamically downgrade connection back to normal priority */ EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_NORMAL), C_OK); EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); - EXPECT_EQ(aeGetFileEvents(server.el->hp_event_loop, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); conn->state = CONN_STATE_NONE; connClose(conn); close(write_fd); } -TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByHighPriorityLoop) { +TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { int normal_pipes[2][2]; - int hp_pipe[2]; + int qos_pipe[2]; int i; int ret; @@ -250,10 +250,10 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByHighPriorityLoop) { fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); } - ret = pipe(hp_pipe); + ret = pipe(qos_pipe); ASSERT_EQ(ret, 0); - fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); - fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); /* Register normal events on server.el */ ret = aeCreateFileEvent(server.el, normal_pipes[0][0], AE_READABLE, testNormalEventCallback, NULL); @@ -261,8 +261,8 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByHighPriorityLoop) { ret = aeCreateFileEvent(server.el, normal_pipes[1][0], AE_READABLE, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - /* Register high priority event on server.el->hp_event_loop */ - ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE, testHighPriorityEventCallback, NULL); + /* Register QoS event on server.el->qos_el */ + ret = aeCreateFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); /* Write 1 byte to all pipes so they fire */ @@ -271,32 +271,32 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByHighPriorityLoop) { } if (write(normal_pipes[1][1], &c, 1) < 0) { } - if (write(hp_pipe[1], &c, 1) < 0) { + if (write(qos_pipe[1], &c, 1) < 0) { } - /* Ensure more than HP_EVENTS_POLL_CHECK_EVERY_US (1000 us) has elapsed for hp_event_loop->last_poll_us */ + /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el->qos_el_last_poll_us */ advanceMockTime(1000000); g_execution_count = 0; - /* Process events on main event loop. At iteration j = 0, PROCESS_HP_EVENTS_IF_NEEDED will trigger - * polling of server.el->hp_event_loop and execute testHighPriorityEventCallback before processing the normal events! */ + /* Process events on main event loop. Preemption check will trigger + * polling of server.el->qos_el and execute testQoSEventCallback before processing the normal events! */ aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); - /* Verify that high priority event (id 999) executed first due to preemption */ + /* Verify that QoS event (id 999) executed first due to preemption */ ASSERT_GE(g_execution_count, 1); EXPECT_EQ(g_execution_order[0], 999); /* Cleanup events and file descriptors */ aeDeleteFileEvent(server.el, normal_pipes[0][0], AE_READABLE); aeDeleteFileEvent(server.el, normal_pipes[1][0], AE_READABLE); - aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE); + aeDeleteFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE); for (i = 0; i < 2; i++) { close(normal_pipes[i][0]); close(normal_pipes[i][1]); } - close(hp_pipe[0]); - close(hp_pipe[1]); + close(qos_pipe[0]); + close(qos_pipe[1]); } TEST_F(SocketPrioritizationTest, QoSNames) { @@ -304,7 +304,7 @@ TEST_F(SocketPrioritizationTest, QoSNames) { EXPECT_STREQ(getConnectionPriorityName(CONN_PRIORITY_HIGH), "prioritized"); } -static void testHighPriorityWriteCallback(struct connection *conn) { +static void testQoSWriteCallback(struct connection *conn) { if (g_execution_count < 1024) { g_execution_order[g_execution_count++] = 999; } @@ -315,7 +315,7 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { ConnectionType *ct = connectionByType(GetParam()); if (ct == NULL) return; int normal_pipe[2]; - int hp_pipe[2]; + int qos_pipe[2]; int ret; ret = pipe(normal_pipe); @@ -323,23 +323,23 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { fcntl(normal_pipe[0], F_SETFL, O_NONBLOCK); fcntl(normal_pipe[1], F_SETFL, O_NONBLOCK); - ret = pipe(hp_pipe); + ret = pipe(qos_pipe); ASSERT_EQ(ret, 0); - fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); - fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); - connection *hp_conn = connCreate(ct); - ASSERT_NE(hp_conn, (connection *)NULL); - hp_conn->state = CONN_STATE_CONNECTED; - hp_conn->fd = hp_pipe[1]; - connSetPriority(hp_conn, CONN_PRIORITY_HIGH); + connection *qos_conn = connCreate(ct); + ASSERT_NE(qos_conn, (connection *)NULL); + qos_conn->state = CONN_STATE_CONNECTED; + qos_conn->fd = qos_pipe[1]; + connSetPriority(qos_conn, CONN_PRIORITY_HIGH); /* Register normal read event on server.el */ ret = aeCreateFileEvent(server.el, normal_pipe[0], AE_READABLE, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - /* Register high priority write event on server.el->hp_event_loop via connection wrapper */ - ret = connSetWriteHandler(hp_conn, testHighPriorityWriteCallback); + /* Register QoS write event on server.el->qos_el via connection wrapper */ + ret = connSetWriteHandler(qos_conn, testQoSWriteCallback); EXPECT_EQ(ret, C_OK); char c = 'w'; @@ -354,18 +354,18 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { EXPECT_EQ(g_execution_order[0], 999); aeDeleteFileEvent(server.el, normal_pipe[0], AE_READABLE); - hp_conn->state = CONN_STATE_NONE; - connClose(hp_conn); + qos_conn->state = CONN_STATE_NONE; + connClose(qos_conn); close(normal_pipe[0]); close(normal_pipe[1]); - close(hp_pipe[0]); + close(qos_pipe[0]); } TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { - /* Verify that PROCESS_HP_EVENTS_IF_NEEDED checks hp_el on iterations where (itr_cnt & 3) == 0 */ + /* Verify that preemption check services qos_el on iterations where (itr_cnt & 3) == 0 */ int normal_pipes[5][2]; - int hp_pipe[2]; - int hp_pipe2[2]; + int qos_pipe[2]; + int qos_pipe2[2]; int i, ret; for (i = 0; i < 5; i++) { @@ -374,23 +374,23 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { fcntl(normal_pipes[i][0], F_SETFL, O_NONBLOCK); fcntl(normal_pipes[i][1], F_SETFL, O_NONBLOCK); } - ret = pipe(hp_pipe); + ret = pipe(qos_pipe); ASSERT_EQ(ret, 0); - fcntl(hp_pipe[0], F_SETFL, O_NONBLOCK); - fcntl(hp_pipe[1], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe[1], F_SETFL, O_NONBLOCK); - ret = pipe(hp_pipe2); + ret = pipe(qos_pipe2); ASSERT_EQ(ret, 0); - fcntl(hp_pipe2[0], F_SETFL, O_NONBLOCK); - fcntl(hp_pipe2[1], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe2[0], F_SETFL, O_NONBLOCK); + fcntl(qos_pipe2[1], F_SETFL, O_NONBLOCK); for (i = 0; i < 5; i++) { ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackMaskCheck, NULL); EXPECT_EQ(ret, AE_OK); } - ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE, testHighPriorityEventCallback, NULL); + ret = aeCreateFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - ret = aeCreateFileEvent(server.el->hp_event_loop, hp_pipe2[0], AE_READABLE, testHighPriorityEventCallback, NULL); + ret = aeCreateFileEvent(server.el->qos_el, qos_pipe2[0], AE_READABLE, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); char c = 'z'; @@ -398,10 +398,10 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { if (write(normal_pipes[i][1], &c, 1) < 0) { } } - if (write(hp_pipe[1], &c, 1) < 0) { + if (write(qos_pipe[1], &c, 1) < 0) { } - g_hp_pipe2_write_fd = hp_pipe2[1]; + g_qos_pipe2_write_fd = qos_pipe2[1]; g_normal_cb_count = 0; advanceMockTime(1000000); g_execution_count = 0; @@ -409,29 +409,29 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); ASSERT_EQ(g_execution_count, 7); - /* At start (pre-loop), hp_event_loop processed */ + /* At start (pre-loop), qos_el processed */ EXPECT_EQ(g_execution_order[0], 999); EXPECT_NE(g_execution_order[1], 999); EXPECT_NE(g_execution_order[2], 999); EXPECT_NE(g_execution_order[3], 999); EXPECT_NE(g_execution_order[4], 999); - /* At iteration 4 ((j & 3) == 0 after >1000us elapsed), hp_event_loop is checked and serviced again on later masked iteration */ + /* At iteration 4 ((j & 3) == 0 after >2000us elapsed), qos_el is checked and serviced again on later masked iteration */ EXPECT_EQ(g_execution_order[5], 999); EXPECT_NE(g_execution_order[6], 999); - g_hp_pipe2_write_fd = -1; + g_qos_pipe2_write_fd = -1; for (i = 0; i < 5; i++) { aeDeleteFileEvent(server.el, normal_pipes[i][0], AE_READABLE); close(normal_pipes[i][0]); close(normal_pipes[i][1]); } - aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe[0], AE_READABLE); - close(hp_pipe[0]); - close(hp_pipe[1]); - aeDeleteFileEvent(server.el->hp_event_loop, hp_pipe2[0], AE_READABLE); - close(hp_pipe2[0]); - close(hp_pipe2[1]); + aeDeleteFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE); + close(qos_pipe[0]); + close(qos_pipe[1]); + aeDeleteFileEvent(server.el->qos_el, qos_pipe2[0], AE_READABLE); + close(qos_pipe2[0]); + close(qos_pipe2[1]); } TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { @@ -449,7 +449,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); - ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[BYTES_PER_FD] = {'a', 'b', 'c', 'd', 'e'}; @@ -464,13 +464,13 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; if (total_served >= NUM_FDS * BYTES_PER_FD) break; - aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); } /* Verify all FDs were served exactly BYTES_PER_FD times without any level-triggered event being lost or starved */ for (i = 0; i < NUM_FDS; i++) { EXPECT_EQ(counts[i], BYTES_PER_FD); - aeDeleteFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE); + aeDeleteFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE); close(pipes[i][0]); close(pipes[i][1]); } @@ -493,14 +493,14 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve /* 1. Register Head FDs (0..6) with 3 bytes (unprocessed at the head) and Tail FDs (14..19) with 5 bytes */ for (i = 0; i <= 6; i++) { - ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[3] = {'h', 'e', 'a'}; if (write(pipes[i][1], data, 3) < 0) { } } for (i = 14; i <= 19; i++) { - ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[5] = {'t', 'a', 'i', 'l', 's'}; if (write(pipes[i][1], data, 5) < 0) { @@ -508,13 +508,13 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve } /* Process batch 1: Head and Tail FDs each fire once (reading 1 byte due to level trigger) */ - aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); for (i = 0; i <= 6; i++) EXPECT_GE(counts[i], 1); for (i = 14; i <= 19; i++) EXPECT_GE(counts[i], 1); /* 2. Register Middle FDs (7..13) with 3 bytes (new events in the middle while head/tail have remaining level-triggered data) */ for (i = 7; i <= 13; i++) { - ret = aeCreateFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[3] = {'m', 'i', 'd'}; if (write(pipes[i][1], data, 3) < 0) { @@ -528,7 +528,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve /* 7 head FDs x 3 + 7 middle FDs x 3 + 6 tail FDs x 5 = 21 + 21 + 30 = 72 total bytes */ if (total_served >= 72) break; - aeProcessEvents(server.el->hp_event_loop, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); } /* Verify every single FD across head (unprocessed), middle (new), and tail (level-triggered remaining data) was fully served */ @@ -537,7 +537,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve for (i = 14; i <= 19; i++) EXPECT_EQ(counts[i], 5); for (i = 0; i < NUM_FDS; i++) { - aeDeleteFileEvent(server.el->hp_event_loop, pipes[i][0], AE_READABLE); + aeDeleteFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE); close(pipes[i][0]); close(pipes[i][1]); } @@ -545,25 +545,25 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve TEST_F(SocketPrioritizationTest, DualEventLoopResize) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->hp_event_loop, (aeEventLoop *)NULL); + ASSERT_NE(server.el->qos_el, (aeEventLoop *)NULL); EXPECT_EQ(aeGetSetSize(server.el), 1024); - EXPECT_EQ(aeGetSetSize(server.el->hp_event_loop), 1024); + EXPECT_EQ(aeGetSetSize(server.el->qos_el), 1024); int ret = aeResizeSetSize(server.el, 2048); EXPECT_EQ(ret, AE_OK); EXPECT_EQ(aeGetSetSize(server.el), 2048); - EXPECT_EQ(aeGetSetSize(server.el->hp_event_loop), 2048); + EXPECT_EQ(aeGetSetSize(server.el->qos_el), 2048); } TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { aeEventLoop *standalone_el = aeCreateEventLoop(64); ASSERT_NE(standalone_el, (aeEventLoop *)NULL); - EXPECT_EQ(standalone_el->hp_event_loop, (aeEventLoop *)NULL); + EXPECT_EQ(standalone_el->qos_el, (aeEventLoop *)NULL); int fds[2]; ASSERT_EQ(pipe(fds), 0); - /* Register with AE_HIGH_PRIORITY on standalone event loop with no hp_event_loop */ + /* Register with AE_HIGH_PRIORITY on standalone event loop with no qos_el */ int ret = aeCreateFileEvent(standalone_el, fds[0], AE_READABLE | AE_HIGH_PRIORITY, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); @@ -888,17 +888,19 @@ static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { (void)privdata; (void)mask; char b; - (void)read(fd, &b, 1); + if (read(fd, &b, 1) < 0) { + /* Ignore read error in test callback */ + } } -/* Test that high-priority event loop duration callback correctly samples QoS metrics. */ +/* Test that QoS event loop duration callback correctly samples QoS metrics. */ TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { aeEventLoop *main_loop = aeCreateEventLoop(64); - aeEventLoop *hp_loop = aeCreateEventLoop(64); + aeEventLoop *qos_loop = aeCreateEventLoop(64); ASSERT_NE(main_loop, (aeEventLoop *)NULL); - ASSERT_NE(hp_loop, (aeEventLoop *)NULL); - ASSERT_EQ(aeLinkHighPriorityEventLoop(main_loop, hp_loop), AE_OK); - aeSetHPStatsCallback(main_loop, qosMetricTestCb); + ASSERT_NE(qos_loop, (aeEventLoop *)NULL); + ASSERT_EQ(aeLinkQoSEventLoop(main_loop, qos_loop), AE_OK); + aeSetQoSStatsCallback(main_loop, qosMetricTestCb); unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt; unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum; diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl index 9d588f2e6f0..690fc553204 100644 --- a/tests/unit/socket-prioritization.tcl +++ b/tests/unit/socket-prioritization.tcl @@ -25,7 +25,7 @@ start_server {tags {"socket-prioritization"}} { } test {Socket Prioritization: Replication QoS classification and CLIENT KILL} { - # Start a replica server and verify that replication links are upgraded to high priority + # Start a replica server and verify that replication links are upgraded to QoS priority start_server {} { set replica [srv 0 client] set replica_host [srv 0 host] @@ -86,7 +86,7 @@ start_server {tags {"socket-prioritization"}} { wait_for_condition 50 100 { [string match "*name=replica_mock_${io_threads}*flags=*H*" [$primary client list flags H name replica_mock_$io_threads]] } else { - fail "Replica connection was not upgraded to high priority (io-threads=$io_threads)" + fail "Replica connection was not upgraded to QoS priority (io-threads=$io_threads)" } $replica_mock close @@ -111,7 +111,7 @@ start_server {tags {"socket-prioritization external:skip"}} { start_server {} { set replica2 [srv 0 client] - test "Benchmark standalone replica sync under pipeline load with high-priority event loop" { + test "Benchmark standalone replica sync under pipeline load with QoS event loop" { set load_clients {} for {set c 0} {$c < 5} {incr c} { lappend load_clients [valkey $primary_host $primary_port 0 $::tls] From 7c0eae7baec25fa6870065a5bdeed3e6c12f541b Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Fri, 21 Aug 2026 01:38:52 +0000 Subject: [PATCH 06/14] Address Review comments from Jim Signed-off-by: Satheesha Gowda --- src/ae.c | 110 ++++++++++++------------ src/ae.h | 16 +--- src/cluster.c | 2 +- src/cluster_legacy.c | 6 +- src/cluster_migrateslots.c | 4 +- src/connection.c | 62 ++++--------- src/connection.h | 23 ++--- src/io_threads.c | 10 +-- src/networking.c | 8 +- src/replication.c | 14 ++- src/server.c | 17 ++-- src/socket.c | 3 +- src/unit/test_socket_prioritization.cpp | 82 +++++++++--------- valkey.conf | 2 +- 14 files changed, 153 insertions(+), 206 deletions(-) diff --git a/src/ae.c b/src/ae.c index 043aa5d7ab4..3c24e066f5b 100644 --- a/src/ae.c +++ b/src/ae.c @@ -78,15 +78,6 @@ * implementations are AE_READABLE & AE_WRITABLE. */ #define BACKEND_MASK(mask) ((mask) & (AE_READABLE | AE_WRITABLE)) -/* QoS event loop periodic preemptive poll default interval in microseconds. - * QoS events are checked periodically during normal event loops to prevent - * long batches of normal client commands from starving control plane traffic. */ -#define AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US 2000 - -/* QoS event loop periodic preemptive poll mask. - * Used to amortize getMonotonicUs() clock reads across every 4th iteration ((iter & 3) == 0). */ -#define AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK 3 - aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; @@ -108,7 +99,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->flags = 0; eventLoop->qos_el = NULL; eventLoop->qos_el_last_poll_us = 0; - eventLoop->qos_el_preempt_check_interval_us = AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US; + eventLoop->qos_el_preempt_check_interval_us = 0; eventLoop->qos_el_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; @@ -455,16 +446,16 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } -/* Process all QoS events immediately and invoke the duration callback +/* Process all QoS events and invoke the stats callback * with the elapsed time in microseconds if registered. */ -static int aeProcessQoSEventsNow(aeEventLoop *eventLoop) { +static int aeProcessQoSEvents(aeEventLoop *eventLoop) { int processed = 0; if (eventLoop->qos_el != NULL) { monotime start = getMonotonicUs(); processed = aeProcessEvents(eventLoop->qos_el, AE_ALL_EVENTS | AE_DONT_WAIT); eventLoop->qos_el_last_poll_us = getMonotonicUs(); - - if (eventLoop->qos_el_stats_callback != NULL) { + /* Update INFO stats if registered and QoS events were processed */ + if (eventLoop->qos_el_stats_callback != NULL && processed > 0) { eventLoop->qos_el_stats_callback(eventLoop, eventLoop->qos_el_last_poll_us - start); } } @@ -472,8 +463,8 @@ static int aeProcessQoSEventsNow(aeEventLoop *eventLoop) { } /* Set callback to receive elapsed duration of QoS event processing */ -void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb) { - if (eventLoop) eventLoop->qos_el_stats_callback = cb; +static void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb) { + eventLoop->qos_el_stats_callback = cb; } /* Set the preemptive poll interval in microseconds for QoS events. @@ -481,26 +472,16 @@ void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb) { * when processing batches of normal events exceeds this interval. * Setting this interval to 0 disables preemptive polling. */ void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { - if (eventLoop) eventLoop->qos_el_preempt_check_interval_us = interval_us; -} - -/* Get the current preemptive poll interval in microseconds for QoS events. - * Returns 0 if eventLoop is NULL or preemption is disabled. */ -uint64_t aeGetQoSPreemptCheckInterval(aeEventLoop *eventLoop) { - return eventLoop ? eventLoop->qos_el_preempt_check_interval_us : 0; + eventLoop->qos_el_preempt_check_interval_us = interval_us; } /* Preemptively processes QoS events if the elapsed time since the last poll - * exceeds the qos_el_preempt_check_interval_us threshold and iter count - * is a multiple of AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK (0 disables preemption). */ -int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop, int iter_count) { - if (eventLoop->qos_el == NULL || eventLoop->qos_el_preempt_check_interval_us == 0 || iter_count == 0) return 0; - if ((iter_count & AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK) == 0) { - if (elapsedUs(eventLoop->qos_el_last_poll_us) >= eventLoop->qos_el_preempt_check_interval_us) { - return aeProcessQoSEventsNow(eventLoop); - } - } - return 0; + * exceeds the qos_el_preempt_check_interval_us threshold (0 disables preemption). */ +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop) { + /* Skip if the multiplexer backend doesn't support nested event loop or preemptive polling is disabled */ + if (eventLoop->qos_el == NULL || eventLoop->qos_el_preempt_check_interval_us == 0) return 0; + if (elapsedUs(eventLoop->qos_el_last_poll_us) < eventLoop->qos_el_preempt_check_interval_us) return 0; + return aeProcessQoSEvents(eventLoop); } /* Process every pending file event, then every pending time event @@ -575,7 +556,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { if (qos_fd != -1) { for (j = 0; j < numevents; j++) { if (eventLoop->fired[j].fd == qos_fd) { - processed += aeProcessQoSEventsNow(eventLoop); + processed += aeProcessQoSEvents(eventLoop); break; } } @@ -584,9 +565,8 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { for (j = 0; j < numevents; j++) { /* Periodic Preemptive Poll: - * If processing normal events is taking more than qos_el_preempt_check_interval_us, check for new QoS events. - * Batch AE_QOS_EVENT_PREEMPTIVE_CHECK_MASK checks to minimize monotonic clock call overhead. */ - processed += aeProcessQoSEventsPreemptively(eventLoop, j); + * If processing normal events is taking more than qos_el_preempt_check_interval_us, check for new QoS events. */ + processed += aeProcessQoSEventsPreemptively(eventLoop); int fd = eventLoop->fired[j].fd; /* Skip processing QoS events here again as they were already processed above */ @@ -702,28 +682,52 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { } } -/* Event handler for the nested QoS event loop poll, - * which will immediately process QoS events registered on the QoS event loop file descriptor.*/ -static void qosEventLoopHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - AE_NOTUSED(fd); - AE_NOTUSED(privdata); - AE_NOTUSED(mask); - aeProcessQoSEventsNow(el); -} - /* Register a QoS event loop file descriptor in the main event loop, * which will awake main eventloop when the QoS event loop file descriptor is fired. */ -int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el) { - if (eventLoop == NULL || qos_el == NULL) return AE_ERR; +static int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el) { + assert(eventLoop != NULL && qos_el != NULL); int qos_fd = aeApiGetPollFd(qos_el); - if (qos_fd == -1) { - eventLoop->qos_el = NULL; + if (qos_fd == -1) return AE_ERR; + /* Register qos_fd with NULL callback: qos_fd is only used to wake up the main loop's + * multiplexer when QoS traffic arrives. aeProcessEvents() checks for qos_fd and drains + * qos_el directly before dispatching normal events, and explicitly skips callback + * dispatch for qos_fd. */ + if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, NULL, NULL) == AE_ERR) { return AE_ERR; } - if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, qosEventLoopHandler, qos_el) == AE_ERR) { - eventLoop->qos_el = NULL; + eventLoop->qos_el = qos_el; + return AE_OK; +} +/* Actuate QoS event loop if supported: creates a nested eventloop for internal + * connections (cluster bus, replication, and slot migration jobs). It installs + * a preemptive polling mechanism that wakes the main event loop and processes + * QoS events at regular interval. If qosPreemptPollIntervalUs is 0, standard + * event loop behavior is preserved. The provided stats callback is invoked with the + * elapsed duration of each QoS event processing cycle, enabling monitoring and + * statistics for QoS processing. + * Returns AE_OK on success, or AE_ERR if QoS eventloop actuation fails + * (e.g., unsupported platform, memory allocation failure, or I/O error). + */ +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { + assert(eventLoop != NULL); + + /* Create QoS event loop for internal connections (cluster bus, replication, and slot migration jobs) */ + aeEventLoop *qos_el = aeCreateEventLoop(setsize); + if (qos_el == NULL) return AE_ERR; + + /* Set the QoS event preemption check interval in microseconds. */ + aeSetQoSPreemptCheckInterval(eventLoop, qosPreemptPollIntervalUs); + + /* Sets the stats callback invoked with elapsed duration whenever QoS events are processed. */ + aeSetQoSStatsCallback(eventLoop, qosStatsCallback); + + /* Link QoS eventloop with main eventloop via nested multiplexer polling. + * If multiplexer nesting is unsupported (e.g. evport, select), gracefully + * free QoS eventloop and fall back to main eventloop event processing without QoS. */ + if (aeLinkQoSEventLoop(eventLoop, qos_el) == AE_ERR) { + /* gracefully free QoS eventloop and fall back to main eventloop event processing without QoS */ + aeDeleteEventLoop(qos_el); return AE_ERR; } - eventLoop->qos_el = qos_el; return AE_OK; } diff --git a/src/ae.h b/src/ae.h index 0fba3df6ae0..c03d08e415a 100644 --- a/src/ae.h +++ b/src/ae.h @@ -167,19 +167,9 @@ int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); -/* Preemptively processes pending QoS events during long-running normal event loops. - * Returns the number of QoS events processed. */ -int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop, int iter_count); - -/* Sets and gets the QoS event preemption check interval in microseconds. */ +/* QoS event loop prototypes */ +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback); +int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop); void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); -uint64_t aeGetQoSPreemptCheckInterval(aeEventLoop *eventLoop); -/* Sets the callback invoked with elapsed duration whenever QoS events are processed. */ -void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb); - -/* Links a nested QoS event loop to the main event loop by registering the child loop's - * multiplexer descriptor (e.g. epoll fd) on the parent loop with AE_READABLE. - * Returns AE_OK on success or AE_ERR if multiplexer nesting is unsupported or registration fails. */ -int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el); #endif diff --git a/src/cluster.c b/src/cluster.c index 3e34ed2d240..01d160cfa59 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -368,7 +368,7 @@ migrateCachedSocket *migrateGetSocket(client *c, robj *host, robj *port, long ti /* Create the connection and tag as high-priority so key/slot migration * packets are not delayed by normal tenant commands. */ conn = connCreate(connTypeOfCluster()); - connSetPriority(conn, CONN_PRIORITY_HIGH); + connSetPriority(conn, true); if (connBlockingConnect(conn, objectGetVal(host), atoi(objectGetVal(port)), timeout) != C_OK) { addReplyError(c, "-IOERR error or timeout connecting to the client"); connClose(conn); diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index eeed8bebbb1..2a0f08e14ab 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2054,7 +2054,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); /* Tag inbound cluster bus link as high-priority so cluster gossip and heartbeats * are processed in qos_el ahead of normal client traffic. */ - connSetPriority(conn, CONN_PRIORITY_HIGH); + connSetPriority(conn, true); /* Mark as cluster-owned before any TLS accept retries so generic * accept offload routing can safely avoid client assumptions. */ connSetOwnerKind(conn, CONN_OWNER_CLUSTER_LINK); @@ -6682,7 +6682,7 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long link->conn = connCreate(connTypeOfCluster()); /* Tag outbound cluster bus link as high-priority so node reconnects, gossip ping/pong, * and failure detection heartbeats operate within qos_el. */ - connSetPriority(link->conn, CONN_PRIORITY_HIGH); + connSetPriority(link->conn, true); connSetPrivateData(link->conn, link); connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); if (connConnect(link->conn, node->ip, node->cport, server.bind_source_addr, 0, clusterLinkConnectHandler) == @@ -7723,7 +7723,7 @@ void addReplyClusterLinkDescription(client *c, clusterLink *link) { addReplyLongLong(c, link->send_msg_queue_mem); addReplyBulkCString(c, "qos"); - addReplyBulkCString(c, getConnectionPriorityName(connGetPriority(link->conn))); + addReplyBulkCString(c, connIsPriority(link->conn) ? "prioritized" : "normal"); } /* Add to the output buffer of the given client an array of cluster link descriptions, diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index 204f3ecc4a9..18785e16c3d 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -853,7 +853,7 @@ slotMigrationJob *createSlotImportJob(client *c, job->client->slot_migration_job = job; if (c && c->conn) { /* Upgrade connection to high priority */ - if (connSetPriority(c->conn, CONN_PRIORITY_HIGH) == C_ERR) { + if (connSetPriority(c->conn, true) == C_ERR) { serverLog(LL_WARNING, "Failed to upgrade priority for slot migration connection %d", c->conn->fd); } } @@ -1386,7 +1386,7 @@ int connectSlotExportJob(slotMigrationJob *job) { job->conn = connCreate(connTypeOfReplication()); /* Set connection to high priority */ - connSetPriority(job->conn, CONN_PRIORITY_HIGH); + connSetPriority(job->conn, true); if (connConnect(job->conn, n->ip, port, server.bind_source_addr, 0, slotExportConnectHandler) == C_ERR) { return C_ERR; diff --git a/src/connection.c b/src/connection.c index b9d796c9c6c..055cb10d2c6 100644 --- a/src/connection.c +++ b/src/connection.c @@ -171,34 +171,34 @@ sds getListensInfoString(sds info) { /* Set connection priority. If the connection already has active events * registered in the event loop, migrate them to the new priority level. * Handles postponed state safely if the socket is offloaded to IO threads, - * preserves AE_BARRIER ordering flags, and performs atomic rollback on failure. + * and preserves AE_BARRIER ordering flags. * Returns C_OK on success, or C_ERR if event migration fails. */ -int connSetPriority(connection *conn, int priority) { +int connSetPriority(connection *conn, bool is_priority) { serverAssert(conn != NULL); - if (conn->priority == priority) return C_OK; + if (conn->is_priority == is_priority) return C_OK; - /* Fast path: if no socket exists or no event loop is active, update field directly */ - if (conn->fd == -1 || server.el == NULL) { - conn->priority = priority; + /* Fast path: if no socket exists yet, update priority field directly */ + if (conn->fd == -1) { + conn->is_priority = is_priority; return C_OK; } int mask = aeGetFileEvents(server.el, conn->fd); if (mask == AE_NONE) { - conn->priority = priority; + conn->is_priority = is_priority; return C_OK; } /* If socket state update is postponed by IO threads, update priority field only; * connUpdateState() will register with the new priority upon IO completion. */ if (conn->flags & CONN_FLAG_POSTPONE_UPDATE_STATE) { - conn->priority = priority; + conn->is_priority = is_priority; return C_OK; } /* Dynamic migration: active events exist on this socket */ - int old_priority = conn->priority; - conn->priority = priority; + bool old_priority = conn->is_priority; + conn->is_priority = is_priority; /* Delete existing registration (AE routes delete to the correct loop) */ aeDeleteFileEvent(server.el, conn->fd, AE_READABLE | AE_WRITABLE); @@ -207,45 +207,15 @@ int connSetPriority(connection *conn, int priority) { if (conn->type && conn->type->update_state) { conn->type->update_state(conn); } else { - int barrier = mask & AE_BARRIER; - int ae_qos_flag = connGetAEPriorityFlag(conn); - if ((mask & AE_READABLE) && - aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | ae_qos_flag, conn->type->ae_handler, conn) == AE_ERR) { - goto rollback; - } - if ((mask & AE_WRITABLE) && - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | ae_qos_flag, conn->type->ae_handler, conn) == AE_ERR) { - goto rollback; + mask = (mask & ~AE_HIGH_PRIORITY); /* Strip off old priority flag */ + if (conn->is_priority) mask |= AE_HIGH_PRIORITY; /* Add new priority flag */ + + if (aeCreateFileEvent(server.el, conn->fd, mask, conn->type->ae_handler, conn) == AE_ERR) { + return C_ERR; } } serverLog(LL_DEBUG, "Connection fd %d priority updated from %s to %s", - conn->fd, getConnectionPriorityName(old_priority), getConnectionPriorityName(priority)); + conn->fd, old_priority ? "prioritized" : "normal", is_priority ? "prioritized" : "normal"); return C_OK; - -rollback: - /* Rollback priority and restore previous event registrations */ - conn->priority = old_priority; - int barrier = mask & AE_BARRIER; - int old_ae_qos_flag = connGetAEPriorityFlag(conn); - if (mask & AE_READABLE) - aeCreateFileEvent(server.el, conn->fd, AE_READABLE | barrier | old_ae_qos_flag, conn->type->ae_handler, conn); - if (mask & AE_WRITABLE) - aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE | barrier | old_ae_qos_flag, conn->type->ae_handler, conn); - return C_ERR; -} - -/* Get connection priority */ -int connGetPriority(connection *conn) { - /* Always return CONN_PRIORITY_NORMAL if conn is NULL.*/ - return conn ? conn->priority : CONN_PRIORITY_NORMAL; -} - -/* Get connection priority name from priority value. */ -const char *getConnectionPriorityName(int priority) { - switch (priority) { - case CONN_PRIORITY_NORMAL: return "normal"; - case CONN_PRIORITY_HIGH: return "prioritized"; - default: return "normal"; - } } diff --git a/src/connection.h b/src/connection.h index 4db999c72a6..19b6c364708 100644 --- a/src/connection.h +++ b/src/connection.h @@ -32,9 +32,10 @@ #define VALKEY_CONNECTION_H #include +#include #include -#include #include +#include #include #include "ae.h" @@ -208,7 +209,7 @@ struct connection { ConnectionCallbackFunc conn_handler; ConnectionCallbackFunc write_handler; ConnectionCallbackFunc read_handler; - int priority; /* connPriority value */ + bool is_priority; /* true if connection is prioritized for QoS */ }; #define CONFIG_BINDADDR_MAX 16 @@ -560,21 +561,15 @@ static inline aeFileProc *connAcceptHandler(ConnectionType *ct) { /* Get Listeners information, note that caller should free the non-empty string */ sds getListensInfoString(sds info); -/* Connection qos(see connPriority enum). */ -int connSetPriority(connection *conn, int priority); -int connGetPriority(connection *conn); -const char *getConnectionPriorityName(int priority); +/* Connection QoS / Priority. */ +int connSetPriority(connection *conn, bool is_priority); +static inline bool connIsPriority(const connection *conn) { + return conn && conn->is_priority; +} /* Get AE priority flag for a connection. */ static inline int connGetAEPriorityFlag(const connection *conn) { - if (conn == NULL) return AE_NONE; - switch (conn->priority) { - case CONN_PRIORITY_HIGH: - return AE_HIGH_PRIORITY; - case CONN_PRIORITY_NORMAL: - default: - return AE_NONE; - } + return (conn && conn->is_priority) ? AE_HIGH_PRIORITY : AE_NONE; } int RedisRegisterConnectionTypeSocket(void); int RedisRegisterConnectionTypeUnix(void); diff --git a/src/io_threads.c b/src/io_threads.c index 2b0f8ae9f61..1c23b666172 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -611,7 +611,7 @@ int trySendReadToIOThreads(client *c) { c->io_read_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); - int qidx = connGetPriority(c->conn); + int qidx = connIsPriority(c->conn); if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { c->read_flags = 0; c->io_read_state = CLIENT_IDLE; @@ -674,7 +674,7 @@ int trySendWriteToIOThreads(client *c) { connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_WRITE_CLIENT); - int qidx = connGetPriority(c->conn); + int qidx = connIsPriority(c->conn); if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], job) == false)) { c->io_write_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); @@ -1024,7 +1024,7 @@ void trySendPollJobToIOThreads(void) { } void sendToMainThread(client *c, int type) { - int qidx = connGetPriority(c->conn); + int qidx = connIsPriority(c->conn); if (unlikely(pending_io_responses[qidx])) { flushPendingList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); } @@ -1196,7 +1196,6 @@ int processIOThreadsResponses(void) { if (getPendingIOResponsesCount() == 0) return 0; int total_processed = 0; - int counter = 0; /* Loop until we consume all pending jobs */ while (1) { /* 1. Strict Priority: First, drain high-priority events (cluster bus, replication, and slot migration jobs) */ @@ -1204,12 +1203,11 @@ int processIOThreadsResponses(void) { if (processed == 0) { /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new * high-priority events arrived on qos_el before processing normal traffic. */ - aeProcessQoSEventsPreemptively(server.el, counter); + aeProcessQoSEventsPreemptively(server.el); } /* 3. Drain normal client events */ processed += processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_NORMAL]); total_processed += processed; - counter++; if (processed == 0) return total_processed; } } diff --git a/src/networking.c b/src/networking.c index fe7be2e2eca..b3440d7d207 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1925,7 +1925,7 @@ void acceptCommonHandler(connection *conn, struct ClientFlags flags, char *ip) { return; } /* Default priority is normal */ - connSetPriority(conn, CONN_PRIORITY_NORMAL); + connSetPriority(conn, false); /* Create connection and client */ if ((c = createClient(conn)) == NULL) { @@ -4542,7 +4542,7 @@ sds catClientInfoString(sds s, client *client, int hide_user_data) { if (client->flag.import_source) *p++ = 'I'; if (client->slot_migration_job && isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'i'; if (client->slot_migration_job && !isImportSlotMigrationJob(client->slot_migration_job)) *p++ = 'E'; - if (client->conn && connGetPriority(client->conn) == CONN_PRIORITY_HIGH) *p++ = 'H'; + if (connIsPriority(client->conn)) *p++ = 'H'; if (p == flags) *p++ = 'N'; *p++ = '\0'; @@ -5235,7 +5235,7 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { if (!c->slot_migration_job || isImportSlotMigrationJob(c->slot_migration_job)) return 0; break; case 'H': /* High priority connection */ - if (!c->conn || connGetPriority(c->conn) != CONN_PRIORITY_HIGH) return 0; + if (!connIsPriority(c->conn)) return 0; break; case 'N': /* Check for no flags */ if (c->flag.replica || c->flag.primary || c->flag.pubsub || @@ -5246,7 +5246,7 @@ static int clientMatchesFlagFilter(client *c, sds flag_filter) { c->flag.unix_socket || c->flag.readonly || c->flag.no_evict || c->flag.no_touch || c->flag.throttled || c->flag.import_source || c->slot_migration_job || - (c->conn && connGetPriority(c->conn) == CONN_PRIORITY_HIGH)) { + connIsPriority(c->conn)) { return 0; } break; diff --git a/src/replication.c b/src/replication.c index 98cc046d355..dcf36900cdf 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1171,12 +1171,10 @@ void syncCommand(client *c) { } serverLog(LL_NOTICE, "Replica %s asks for synchronization", replicationGetReplicaName(c)); - if (c->conn) { - /* Upgrade incoming replica connection to high priority so that replication - * command streaming and ACK heartbeats are not delayed by normal client commands. */ - if (connSetPriority(c->conn, CONN_PRIORITY_HIGH) == C_ERR) { - serverLog(LL_WARNING, "Failed to upgrade priority for replica connection %d", c->conn->fd); - } + /* Upgrade incoming replica connection to high priority so that replication + * command streaming and ACK heartbeats are not delayed by normal client commands. */ + if (connSetPriority(c->conn, true) == C_ERR) { + serverLog(LL_WARNING, "Failed to upgrade priority for replica connection %d", c->conn->fd); } /* Try a partial resynchronization if this is a PSYNC command. @@ -4394,7 +4392,7 @@ void syncWithPrimary(connection *conn) { server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); /* Tag connection as high-priority before connecting so non-blocking connect and * subsequent RDB transfer events are registered directly on qos_el. */ - connSetPriority(server.repl_rdb_transfer_s, CONN_PRIORITY_HIGH); + connSetPriority(server.repl_rdb_transfer_s, true); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { dualChannelServerLog(LL_WARNING, "Unable to connect to Primary: %s", @@ -4447,7 +4445,7 @@ int connectWithPrimary(void) { server.repl_transfer_s = connCreate(connTypeOfReplication()); /* Tag main replication connection as high-priority before connecting so handshake, * heartbeat pings, and PSYNC streaming are processed in qos_el. */ - connSetPriority(server.repl_transfer_s, CONN_PRIORITY_HIGH); + connSetPriority(server.repl_transfer_s, true); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { serverLog(LL_WARNING, "Unable to connect to PRIMARY: %s", connGetLastError(server.repl_transfer_s)); diff --git a/src/server.c b/src/server.c index 931d431dab4..600deb34413 100644 --- a/src/server.c +++ b/src/server.c @@ -3116,23 +3116,16 @@ void initServer(void) { const char *clk_msg = monotonicInit(); serverLog(LL_NOTICE, "monotonic clock: %s", clk_msg); server.el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - /* Create QoS event loop for internal connections (cluster bus, replication, and slot migration jobs) */ - aeEventLoop *qos_el = aeCreateEventLoop(server.maxclients + CONFIG_FDSET_INCR); - if (server.el == NULL || qos_el == NULL) { + if (server.el == NULL) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - /* Link QoS event loop with main event loop via nested multiplexer polling. - * If multiplexer nesting is unsupported (e.g. evport, select), gracefully - * free qos_el and fall back to main event loop event processing without QoS. */ - if (aeLinkQoSEventLoop(server.el, qos_el) == AE_ERR) { + /* Setup QoS event loop if multiplexer backend supports nested event loop polling. + * If multiplexer nesting is unsupported (e.g. evport, select), gracefully fallback to main event loop event processing without QoS. */ + if (aeActuateQoSEventLoopIfSupported(server.el, server.maxclients + CONFIG_FDSET_INCR, server.qos_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", aeGetApiName()); - aeDeleteEventLoop(qos_el); - qos_el = NULL; } - aeSetQoSStatsCallback(server.el, qosStatsCallback); - aeSetQoSPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); server.dbnum = server.cluster_enabled ? server.config_databases_cluster : server.config_databases; server.db = zcalloc(sizeof(serverDb *) * server.dbnum); @@ -4258,7 +4251,7 @@ void call(client *c, int flags) { if (server.execution_nesting == 0) { durationAddSample(EL_DURATION_TYPE_CMD, duration); /* Attribute command execution latency for high-priority client connections. */ - if (c->conn && connGetPriority(c->conn) == CONN_PRIORITY_HIGH) { + if (connIsPriority(c->conn)) { durationAddSample(EL_DURATION_TYPE_QOS_CMD, duration); } } diff --git a/src/socket.c b/src/socket.c index 6a1370196dd..a1b7f3a0ce4 100644 --- a/src/socket.c +++ b/src/socket.c @@ -291,8 +291,7 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD conn->state = CONN_STATE_CONNECTED; } - if (!conn->write_handler) - aeDeleteFileEvent(el, conn->fd, AE_WRITABLE); + if (!conn->write_handler) aeDeleteFileEvent(el, conn->fd, AE_WRITABLE); if (!callHandler(conn, conn->conn_handler)) return; conn->conn_handler = NULL; diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 29b4e8e603f..adad9338017 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -83,11 +83,11 @@ static void testQoSEventCallback(aeEventLoop *el, int fd, void *privdata, int ma static int g_qos_pipe2_write_fd = -1; static int g_normal_cb_count = 0; -static void testNormalEventCallbackMaskCheck(aeEventLoop *el, int fd, void *privdata, int mask) { +static void testNormalEventCallbackPreemptCheck(aeEventLoop *el, int fd, void *privdata, int mask) { testNormalEventCallback(el, fd, privdata, mask); g_normal_cb_count++; - /* Sleep > 2000 us on 3rd normal event (j = 2) and trigger 2nd QoS pipe so (itr_cnt & 3) == 0 check at j = 4 polls qos_el */ - if (g_normal_cb_count == 3 && g_qos_pipe2_write_fd != -1) { + /* Sleep > 2000 us on 2nd normal event and trigger 2nd QoS pipe so the next iteration immediately preempts and polls qos_el */ + if (g_normal_cb_count == 2 && g_qos_pipe2_write_fd != -1) { usleep(2500); char c = 'x'; if (write(g_qos_pipe2_write_fd, &c, 1) < 0) { @@ -133,9 +133,8 @@ class SocketPrioritizationTest : public ::testing::Test { } server.el = aeCreateEventLoop(1024); - aeEventLoop *qos_el = aeCreateEventLoop(1024); - if (server.el && qos_el) { - aeLinkQoSEventLoop(server.el, qos_el); + if (server.el) { + aeActuateQoSEventLoopIfSupported(server.el, 1024, 2000, NULL); } g_execution_count = 0; } @@ -160,20 +159,26 @@ TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { ASSERT_NE(server.el->qos_el, (aeEventLoop *)NULL); EXPECT_EQ(server.el->qos_el, server.el->qos_el); EXPECT_EQ(server.el->qos_el->qos_el, (aeEventLoop *)NULL); - EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 2000ULL); + EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 2000ULL); aeSetQoSPreemptCheckInterval(server.el, 5000ULL); - EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 5000ULL); + EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 5000ULL); aeSetQoSPreemptCheckInterval(server.el, 2000ULL); + + /* Newly created standalone loop must have preemption disabled (0) by default */ + aeEventLoop *standalone = aeCreateEventLoop(64); + ASSERT_NE(standalone, (aeEventLoop *)NULL); + EXPECT_EQ(standalone->qos_el_preempt_check_interval_us, 0ULL); + aeDeleteEventLoop(standalone); } TEST_F(SocketPrioritizationTest, DynamicPreemptionIntervalThreshold) { /* Test getter and setter */ aeSetQoSPreemptCheckInterval(server.el, 500ULL); - EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 500ULL); + EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 500ULL); /* Test disabling preemption (interval = 0) */ aeSetQoSPreemptCheckInterval(server.el, 0ULL); - EXPECT_EQ(aeGetQoSPreemptCheckInterval(server.el), 0ULL); + EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 0ULL); aeSetQoSPreemptCheckInterval(server.el, 2000ULL); } @@ -183,15 +188,15 @@ TEST_P(SocketPrioritizationConnTest, ConnectionPriorityMetadataAndHelpers) { connection *conn = connCreate(ct); ASSERT_NE(conn, (connection *)NULL); - /* Default priority should be normal */ - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + /* Default priority should be normal (false) */ + EXPECT_FALSE(connIsPriority(conn)); /* Update metadata when fd is not yet set (-1) */ - connSetPriority(conn, CONN_PRIORITY_HIGH); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + connSetPriority(conn, true); + EXPECT_TRUE(connIsPriority(conn)); - connSetPriority(conn, CONN_PRIORITY_NORMAL); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + connSetPriority(conn, false); + EXPECT_FALSE(connIsPriority(conn)); conn->state = CONN_STATE_NONE; connClose(conn); @@ -219,16 +224,16 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); /* Dynamically upgrade connection to high priority */ - EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_HIGH), C_OK); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + EXPECT_EQ(connSetPriority(conn, true), C_OK); + EXPECT_TRUE(connIsPriority(conn)); int events = aeGetFileEvents(server.el, read_fd); EXPECT_NE(events & AE_READABLE, 0); EXPECT_NE(events & AE_HIGH_PRIORITY, 0); EXPECT_NE(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); /* Dynamically downgrade connection back to normal priority */ - EXPECT_EQ(connSetPriority(conn, CONN_PRIORITY_NORMAL), C_OK); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + EXPECT_EQ(connSetPriority(conn, false), C_OK); + EXPECT_FALSE(connIsPriority(conn)); EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); @@ -299,11 +304,6 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { close(qos_pipe[1]); } -TEST_F(SocketPrioritizationTest, QoSNames) { - EXPECT_STREQ(getConnectionPriorityName(CONN_PRIORITY_NORMAL), "normal"); - EXPECT_STREQ(getConnectionPriorityName(CONN_PRIORITY_HIGH), "prioritized"); -} - static void testQoSWriteCallback(struct connection *conn) { if (g_execution_count < 1024) { g_execution_order[g_execution_count++] = 999; @@ -332,7 +332,7 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { ASSERT_NE(qos_conn, (connection *)NULL); qos_conn->state = CONN_STATE_CONNECTED; qos_conn->fd = qos_pipe[1]; - connSetPriority(qos_conn, CONN_PRIORITY_HIGH); + connSetPriority(qos_conn, true); /* Register normal read event on server.el */ ret = aeCreateFileEvent(server.el, normal_pipe[0], AE_READABLE, testNormalEventCallback, NULL); @@ -361,8 +361,8 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { close(qos_pipe[0]); } -TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { - /* Verify that preemption check services qos_el on iterations where (itr_cnt & 3) == 0 */ +TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) { + /* Verify that preemption check immediately services qos_el on the next iteration when elapsed time threshold is crossed */ int normal_pipes[5][2]; int qos_pipe[2]; int qos_pipe2[2]; @@ -385,7 +385,7 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { fcntl(qos_pipe2[1], F_SETFL, O_NONBLOCK); for (i = 0; i < 5; i++) { - ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackMaskCheck, NULL); + ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackPreemptCheck, NULL); EXPECT_EQ(ret, AE_OK); } ret = aeCreateFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE, testQoSEventCallback, NULL); @@ -411,12 +411,15 @@ TEST_F(SocketPrioritizationTest, MultiIterationPreemptionCheckMask) { ASSERT_EQ(g_execution_count, 7); /* At start (pre-loop), qos_el processed */ EXPECT_EQ(g_execution_order[0], 999); + /* 1st normal event */ EXPECT_NE(g_execution_order[1], 999); + /* 2nd normal event (sleeps >2000us and writes to qos_pipe2) */ EXPECT_NE(g_execution_order[2], 999); - EXPECT_NE(g_execution_order[3], 999); + /* Immediate preemption on the very next iteration */ + EXPECT_EQ(g_execution_order[3], 999); + /* Remaining normal events */ EXPECT_NE(g_execution_order[4], 999); - /* At iteration 4 ((j & 3) == 0 after >2000us elapsed), qos_el is checked and serviced again on later masked iteration */ - EXPECT_EQ(g_execution_order[5], 999); + EXPECT_NE(g_execution_order[5], 999); EXPECT_NE(g_execution_order[6], 999); g_qos_pipe2_write_fd = -1; @@ -593,15 +596,15 @@ TEST_F(SocketPrioritizationTest, PostponedStateDynamicPriorityUpdate) { int ret = connSetReadHandler(conn, dummyConnectionHandler); EXPECT_EQ(ret, C_OK); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_NORMAL); + EXPECT_FALSE(connIsPriority(conn)); /* Set postponed state (e.g. while offloaded to IO threads) */ conn->flags |= CONN_FLAG_POSTPONE_UPDATE_STATE; /* Upgrading priority while postponed must update priority field but defer event loop migration */ - ret = connSetPriority(conn, CONN_PRIORITY_HIGH); + ret = connSetPriority(conn, true); EXPECT_EQ(ret, C_OK); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + EXPECT_TRUE(connIsPriority(conn)); conn->flags &= ~CONN_FLAG_POSTPONE_UPDATE_STATE; conn->state = CONN_STATE_NONE; @@ -627,9 +630,9 @@ TEST_F(SocketPrioritizationTest, WriteBarrierPreservation) { EXPECT_NE(events & AE_WRITABLE, 0); /* Upgrade priority: verify migration succeeds with barrier */ - ret = connSetPriority(conn, CONN_PRIORITY_HIGH); + ret = connSetPriority(conn, true); EXPECT_EQ(ret, C_OK); - EXPECT_EQ(connGetPriority(conn), CONN_PRIORITY_HIGH); + EXPECT_TRUE(connIsPriority(conn)); conn->state = CONN_STATE_NONE; connClose(conn); @@ -896,11 +899,8 @@ static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { /* Test that QoS event loop duration callback correctly samples QoS metrics. */ TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { aeEventLoop *main_loop = aeCreateEventLoop(64); - aeEventLoop *qos_loop = aeCreateEventLoop(64); ASSERT_NE(main_loop, (aeEventLoop *)NULL); - ASSERT_NE(qos_loop, (aeEventLoop *)NULL); - ASSERT_EQ(aeLinkQoSEventLoop(main_loop, qos_loop), AE_OK); - aeSetQoSStatsCallback(main_loop, qosMetricTestCb); + ASSERT_EQ(aeActuateQoSEventLoopIfSupported(main_loop, 64, 2000, qosMetricTestCb), AE_OK); unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt; unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum; diff --git a/valkey.conf b/valkey.conf index 0670d2b5d85..0713d3a86a2 100644 --- a/valkey.conf +++ b/valkey.conf @@ -2775,7 +2775,7 @@ jemalloc-bg-thread yes # # Valkey prioritizes system-critical internal connections (such as cluster bus # heartbeats/gossip, primary-replica replication streams, and slot migration links) -# ahead of standard client traffic to maintain control-plane responsiveness and +# ahead of standard client traffic to maintain stability and # prevent false failovers under heavy command workloads. # # When the main eventloop processes large batches of normal client events (such as From 08ecf1097eca2be4d706f6b32d3d9237965c53b6 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Fri, 28 Aug 2026 18:46:36 +0000 Subject: [PATCH 07/14] Remove Nested Epoll loop Signed-off-by: Satheesha Gowda --- src/ae.c | 261 ++++++++++++------------ src/ae.h | 38 ++-- src/ae_epoll.c | 5 +- src/ae_evport.c | 4 +- src/ae_kqueue.c | 5 +- src/ae_select.c | 4 +- src/unit/test_socket_prioritization.cpp | 68 +++--- 7 files changed, 196 insertions(+), 189 deletions(-) diff --git a/src/ae.c b/src/ae.c index 3c24e066f5b..6b883a8c093 100644 --- a/src/ae.c +++ b/src/ae.c @@ -97,7 +97,9 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->aftersleep = NULL; eventLoop->custompoll = NULL; eventLoop->flags = 0; - eventLoop->qos_el = NULL; + eventLoop->qos_apidata = NULL; + eventLoop->qos_fd = -1; + eventLoop->qos_fired = NULL; eventLoop->qos_el_last_poll_us = 0; eventLoop->qos_el_preempt_check_interval_us = 0; eventLoop->qos_el_stats_callback = NULL; @@ -155,8 +157,9 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) goto err; if (aeApiResize(eventLoop->apidata, setsize) == -1) goto err; - if (eventLoop->qos_el) { - if (aeResizeSetSize(eventLoop->qos_el, setsize) == AE_ERR) goto err; + if (eventLoop->qos_apidata) { + if (aeApiResize(eventLoop->qos_apidata, setsize) == -1) goto err; + eventLoop->qos_fired = zrealloc(eventLoop->qos_fired, sizeof(aeFiredEvent) * setsize); } eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * setsize); @@ -176,10 +179,10 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { } void aeDeleteEventLoop(aeEventLoop *eventLoop) { - if (eventLoop->qos_el) { - int qos_fd = aeApiGetPollFd(eventLoop->qos_el); - if (qos_fd != -1) aeDeleteFileEvent(eventLoop, qos_fd, AE_READABLE); - aeDeleteEventLoop(eventLoop->qos_el); + if (eventLoop->qos_apidata) { + if (eventLoop->qos_fd != -1) aeDeleteFileEvent(eventLoop, eventLoop->qos_fd, AE_READABLE); + aeApiFree(eventLoop->qos_apidata); + zfree(eventLoop->qos_fired); } aeApiFree(eventLoop->apidata); zfree(eventLoop->events); @@ -201,14 +204,6 @@ void aeStop(aeEventLoop *eventLoop) { } int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc, void *clientData) { - if (mask & AE_HIGH_PRIORITY) { - if (eventLoop->qos_el) { - /* Register events on to QoS event loop */ - return aeCreateFileEvent(eventLoop->qos_el, fd, mask & ~AE_HIGH_PRIORITY, proc, clientData); - } - /* If qos_el is not initialized, strip the high priority flag */ - mask &= ~AE_HIGH_PRIORITY; - } AE_LOCK(eventLoop); int ret = AE_ERR; @@ -218,9 +213,20 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } aeFileEvent *fe = &eventLoop->events[fd]; + int is_priority = 0; + if ((mask & AE_HIGH_PRIORITY) || (fe->mask & AE_HIGH_PRIORITY)) { + if (eventLoop->qos_apidata != NULL) { + is_priority = 1; + mask |= AE_HIGH_PRIORITY; + } else { + mask &= ~AE_HIGH_PRIORITY; + } + } + aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions if (backend_add_mask) { - if (aeApiAddEvent(eventLoop->apidata, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; + if (aeApiAddEvent(target_api, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; } fe->mask |= mask; if (mask & AE_READABLE) fe->rfileProc = proc; @@ -236,14 +242,6 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { - if (eventLoop->qos_el) { - /* Check if the event exists in QoS loop first */ - int qos_mask = aeGetFileEvents(eventLoop->qos_el, fd); - if (qos_mask != AE_NONE) { - aeDeleteFileEvent(eventLoop->qos_el, fd, mask); - return; - } - } AE_LOCK(eventLoop); mask &= ~AE_HIGH_PRIORITY; if (fd >= eventLoop->setsize) goto done; @@ -251,6 +249,9 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) goto done; + int is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); + aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + /* We want to always remove AE_BARRIER if set when AE_WRITABLE * is removed. */ if (mask & AE_WRITABLE) mask |= AE_BARRIER; @@ -260,6 +261,9 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { int old_mask = fe->mask; fe->mask = fe->mask & ~mask; + if (!(fe->mask & (AE_READABLE | AE_WRITABLE))) { + fe->mask = AE_NONE; + } if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { /* Update the max fd */ int j; @@ -274,7 +278,7 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { * touching the actual events. */ int backend_del_mask = BACKEND_MASK(mask); // just the meaningful deletions if (backend_del_mask) { - aeApiDelEvent(eventLoop->apidata, fd, BACKEND_MASK(old_mask), backend_del_mask); + aeApiDelEvent(target_api, fd, BACKEND_MASK(old_mask), backend_del_mask); } done: @@ -282,12 +286,6 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { } void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { - if (eventLoop->qos_el) { - int mask = aeGetFileEvents(eventLoop->qos_el, fd); - if (mask != AE_NONE) { - return aeGetFileClientData(eventLoop->qos_el, fd); - } - } if (fd >= eventLoop->setsize) return NULL; aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) return NULL; @@ -296,12 +294,6 @@ void *aeGetFileClientData(aeEventLoop *eventLoop, int fd) { } int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { - if (eventLoop->qos_el) { - int mask = aeGetFileEvents(eventLoop->qos_el, fd); - if (mask != AE_NONE) { - return mask | AE_HIGH_PRIORITY; - } - } if (fd >= eventLoop->setsize) return 0; aeFileEvent *fe = &eventLoop->events[fd]; @@ -446,13 +438,72 @@ int aePoll(aeEventLoop *eventLoop, struct timeval *tvp) { return ret; } +/* Fire the readable and/or writable event handlers for a given file descriptor, + * respecting the AE_BARRIER flag and protecting against re-entrancy issues + * (e.g. event loop resize or event deletion within callbacks). */ +static inline void aeFireFileEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeFileEvent *fe = &eventLoop->events[fd]; + int fired = 0; /* Number of events fired for current fd. */ + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if AE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsyncing a file to disk, + * before replying to a client. */ + int invert = fe->mask & AE_BARRIER; + + /* Note the "fe->mask & mask & ..." code: maybe an already + * processed event removed an element that fired and we still + * didn't processed, so we check if the event is still valid. + * + * Fire the readable event if the call sequence is not + * inverted. */ + if (!invert && (fe->mask & mask & AE_READABLE)) { + fe->rfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + } + + /* Fire the writable event. */ + if (fe->mask & mask & AE_WRITABLE) { + if (!fired || fe->wfileProc != fe->rfileProc) { + fe->wfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + } + } + + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && (!fired || fe->wfileProc != fe->rfileProc)) { + fe->rfileProc(eventLoop, fd, fe->clientData, mask); + fired++; + } + } +} + /* Process all QoS events and invoke the stats callback * with the elapsed time in microseconds if registered. */ static int aeProcessQoSEvents(aeEventLoop *eventLoop) { int processed = 0; - if (eventLoop->qos_el != NULL) { + if (eventLoop->qos_apidata != NULL) { monotime start = getMonotonicUs(); - processed = aeProcessEvents(eventLoop->qos_el, AE_ALL_EVENTS | AE_DONT_WAIT); + struct timeval tv = {0, 0}; + int numevents = aeApiPoll(eventLoop->qos_apidata, eventLoop->qos_fired, eventLoop->events, + eventLoop->setsize, eventLoop->maxfd, &tv); + for (int j = 0; j < numevents; j++) { + int fd = eventLoop->qos_fired[j].fd; + int mask = eventLoop->qos_fired[j].mask; + aeFireFileEvent(eventLoop, fd, mask); + processed++; + } eventLoop->qos_el_last_poll_us = getMonotonicUs(); /* Update INFO stats if registered and QoS events were processed */ if (eventLoop->qos_el_stats_callback != NULL && processed > 0) { @@ -478,8 +529,8 @@ void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) /* Preemptively processes QoS events if the elapsed time since the last poll * exceeds the qos_el_preempt_check_interval_us threshold (0 disables preemption). */ int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop) { - /* Skip if the multiplexer backend doesn't support nested event loop or preemptive polling is disabled */ - if (eventLoop->qos_el == NULL || eventLoop->qos_el_preempt_check_interval_us == 0) return 0; + /* Skip if QoS is disabled or preemptive polling is disabled */ + if (eventLoop->qos_apidata == NULL || eventLoop->qos_el_preempt_check_interval_us == 0) return 0; if (elapsedUs(eventLoop->qos_el_last_poll_us) < eventLoop->qos_el_preempt_check_interval_us) return 0; return aeProcessQoSEvents(eventLoop); } @@ -548,76 +599,30 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); - /* Prioritize QoS event loop: if qos_fd fired, drain QoS events + /* Prioritize QoS events: if qos_fd fired, drain QoS events * immediately before processing normal events. */ - int qos_fd = -1; - if (eventLoop->qos_el != NULL) { - qos_fd = aeApiGetPollFd(eventLoop->qos_el); - if (qos_fd != -1) { - for (j = 0; j < numevents; j++) { - if (eventLoop->fired[j].fd == qos_fd) { - processed += aeProcessQoSEvents(eventLoop); - break; - } + if (eventLoop->qos_fd != -1) { + for (j = 0; j < numevents; j++) { + if (eventLoop->fired[j].fd == eventLoop->qos_fd) { + processed += aeProcessQoSEvents(eventLoop); + break; } } } for (j = 0; j < numevents; j++) { /* Periodic Preemptive Poll: - * If processing normal events is taking more than qos_el_preempt_check_interval_us, check for new QoS events. */ - processed += aeProcessQoSEventsPreemptively(eventLoop); + * Sample monotonic clock once every 4 iterations (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead */ + if ((j & AE_QOS_PREEMPT_CHECK_MASK) == 0 && eventLoop->qos_apidata != NULL) { + processed += aeProcessQoSEventsPreemptively(eventLoop); + } int fd = eventLoop->fired[j].fd; /* Skip processing QoS events here again as they were already processed above */ - if (fd == qos_fd) continue; - aeFileEvent *fe = &eventLoop->events[fd]; + if (fd == eventLoop->qos_fd) continue; int mask = eventLoop->fired[j].mask; - int fired = 0; /* Number of events fired for current fd. */ - - /* Normally we execute the readable event first, and the writable - * event later. This is useful as sometimes we may be able - * to serve the reply of a query immediately after processing the - * query. - * - * However if AE_BARRIER is set in the mask, our application is - * asking us to do the reverse: never fire the writable event - * after the readable. In such a case, we invert the calls. - * This is useful when, for instance, we want to do things - * in the beforeSleep() hook, like fsyncing a file to disk, - * before replying to a client. */ - int invert = fe->mask & AE_BARRIER; - - /* Note the "fe->mask & mask & ..." code: maybe an already - * processed event removed an element that fired and we still - * didn't processed, so we check if the event is still valid. - * - * Fire the readable event if the call sequence is not - * inverted. */ - if (!invert && fe->mask & mask & AE_READABLE) { - fe->rfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ - } - - /* Fire the writable event. */ - if (fe->mask & mask & AE_WRITABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { - fe->wfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - } - } - - /* If we have to invert the call, fire the readable event now - * after the writable one. */ - if (invert) { - fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ - if ((fe->mask & mask & AE_READABLE) && (!fired || fe->wfileProc != fe->rfileProc)) { - fe->rfileProc(eventLoop, fd, fe->clientData, mask); - fired++; - } - } + aeFireFileEvent(eventLoop, fd, mask); processed++; } } @@ -682,23 +687,7 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { } } -/* Register a QoS event loop file descriptor in the main event loop, - * which will awake main eventloop when the QoS event loop file descriptor is fired. */ -static int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el) { - assert(eventLoop != NULL && qos_el != NULL); - int qos_fd = aeApiGetPollFd(qos_el); - if (qos_fd == -1) return AE_ERR; - /* Register qos_fd with NULL callback: qos_fd is only used to wake up the main loop's - * multiplexer when QoS traffic arrives. aeProcessEvents() checks for qos_fd and drains - * qos_el directly before dispatching normal events, and explicitly skips callback - * dispatch for qos_fd. */ - if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, NULL, NULL) == AE_ERR) { - return AE_ERR; - } - eventLoop->qos_el = qos_el; - return AE_OK; -} -/* Actuate QoS event loop if supported: creates a nested eventloop for internal +/* Actuate QoS event loop if supported: creates a secondary multiplexer state for internal * connections (cluster bus, replication, and slot migration jobs). It installs * a preemptive polling mechanism that wakes the main event loop and processes * QoS events at regular interval. If qosPreemptPollIntervalUs is 0, standard @@ -711,9 +700,35 @@ static int aeLinkQoSEventLoop(aeEventLoop *eventLoop, aeEventLoop *qos_el) { int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { assert(eventLoop != NULL); - /* Create QoS event loop for internal connections (cluster bus, replication, and slot migration jobs) */ - aeEventLoop *qos_el = aeCreateEventLoop(setsize); - if (qos_el == NULL) return AE_ERR; + /* Create QoS polling state for internal connections (cluster bus, replication, and slot migration jobs) */ + aeApiState *qos_apidata = aeApiCreate(setsize); + if (qos_apidata == NULL) return AE_ERR; + + int qos_fd = aeApiGetPollFd(qos_apidata); + if (qos_fd == -1) { + aeApiFree(qos_apidata); + return AE_ERR; + } + + aeFiredEvent *qos_fired = zmalloc(sizeof(aeFiredEvent) * setsize); + if (qos_fired == NULL) { + aeApiFree(qos_apidata); + return AE_ERR; + } + + /* Register qos_fd with NULL callback: qos_fd is only used to wake up the main loop's + * multiplexer when QoS traffic arrives. aeProcessEvents() checks for qos_fd and drains + * qos_apidata directly before dispatching normal events, and explicitly skips callback + * dispatch for qos_fd. */ + if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, NULL, NULL) == AE_ERR) { + zfree(qos_fired); + aeApiFree(qos_apidata); + return AE_ERR; + } + + eventLoop->qos_apidata = qos_apidata; + eventLoop->qos_fd = qos_fd; + eventLoop->qos_fired = qos_fired; /* Set the QoS event preemption check interval in microseconds. */ aeSetQoSPreemptCheckInterval(eventLoop, qosPreemptPollIntervalUs); @@ -721,13 +736,5 @@ int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64 /* Sets the stats callback invoked with elapsed duration whenever QoS events are processed. */ aeSetQoSStatsCallback(eventLoop, qosStatsCallback); - /* Link QoS eventloop with main eventloop via nested multiplexer polling. - * If multiplexer nesting is unsupported (e.g. evport, select), gracefully - * free QoS eventloop and fall back to main eventloop event processing without QoS. */ - if (aeLinkQoSEventLoop(eventLoop, qos_el) == AE_ERR) { - /* gracefully free QoS eventloop and fall back to main eventloop event processing without QoS */ - aeDeleteEventLoop(qos_el); - return AE_ERR; - } return AE_OK; } diff --git a/src/ae.h b/src/ae.h index c03d08e415a..101d9f2696d 100644 --- a/src/ae.h +++ b/src/ae.h @@ -39,17 +39,18 @@ #define AE_OK 0 #define AE_ERR -1 -#define AE_NONE 0 /* No events registered. */ -#define AE_READABLE 1 /* Fire when descriptor is readable. */ -#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ -#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ - READABLE event already fired in the same event \ - loop iteration. Useful when you want to persist \ - things to disk before sending replies, and want \ - to do that in a group fashion. */ -#define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ - * the event is registered on qos_el if available. \ - * Stripped before passing to the underlying OS multiplexer. */ +#define AE_NONE 0 /* No events registered. */ +#define AE_READABLE 1 /* Fire when descriptor is readable. */ +#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ +#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the \ + READABLE event already fired in the same event \ + loop iteration. Useful when you want to persist \ + things to disk before sending replies, and want \ + to do that in a group fashion. */ +#define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ + * the event is registered on qos_apidata if available. \ + * Stripped before passing to the underlying OS multiplexer. */ +#define AE_QOS_PREEMPT_CHECK_MASK 0x03 /* Mask to check QoS preemption once every 4 iterations */ #define AE_FILE_EVENTS (1 << 0) #define AE_TIME_EVENTS (1 << 1) @@ -130,14 +131,15 @@ typedef struct aeEventLoop { int flags; /* Quality of Service (QoS): - * qos_el points to a secondary, nested event loop dedicated to critical - * internal channels (cluster bus, slot migration, replication). Sockets registered - * here are processed ahead of normal client traffic and periodically preempt - * normal event iterations. NULL if unsupported by the OS multiplexer or disabled. */ - struct aeEventLoop *qos_el; - monotime qos_el_last_poll_us; /* Timestamp (microseconds) when qos_el was last drained */ + * Sockets registered with AE_HIGH_PRIORITY are tracked in qos_apidata. + * qos_fd is registered into apidata to wake the main loop when QoS traffic arrives. + * qos_fired holds fired events when draining QoS channels. */ + aeApiState *qos_apidata; /* Dedicated QoS polling state */ + int qos_fd; /* File descriptor of QoS polling backend (-1 if disabled) */ + aeFiredEvent *qos_fired; /* Fired events buffer for QoS polling */ + monotime qos_el_last_poll_us; /* Timestamp (microseconds) when QoS was last drained */ uint64_t qos_el_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ - aeQoSStatsProc *qos_el_stats_callback; /* Callback invoked with elapsed microseconds after draining qos_el */ + aeQoSStatsProc *qos_el_stats_callback; /* Callback invoked with elapsed microseconds after draining QoS */ } aeEventLoop; /* Prototypes */ diff --git a/src/ae_epoll.c b/src/ae_epoll.c index ebe401eb48f..b0dc8a7d66b 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -135,7 +135,6 @@ static char *aeApiName(void) { return "epoll"; } -static int aeApiGetPollFd(aeEventLoop *eventLoop) { - aeApiState *state = eventLoop->apidata; - return state->epfd; +static int aeApiGetPollFd(aeApiState *state) { + return state ? state->epfd : -1; } diff --git a/src/ae_evport.c b/src/ae_evport.c index e990e141f9c..e6ab055a98d 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -290,7 +290,7 @@ static char *aeApiName(void) { return "evport"; } -static int aeApiGetPollFd(aeEventLoop *eventLoop) { - UNUSED(eventLoop); +static int aeApiGetPollFd(aeApiState *state) { + UNUSED(state); return -1; } diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 703d163c19b..d518b88c49a 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -181,7 +181,6 @@ static char *aeApiName(void) { return "kqueue"; } -static int aeApiGetPollFd(aeEventLoop *eventLoop) { - aeApiState *state = eventLoop->apidata; - return state->kqfd; +static int aeApiGetPollFd(aeApiState *state) { + return state ? state->kqfd : -1; } diff --git a/src/ae_select.c b/src/ae_select.c index 69ab3caed20..61c665b6cae 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -108,7 +108,7 @@ static char *aeApiName(void) { return "select"; } -static int aeApiGetPollFd(aeEventLoop *eventLoop) { - UNUSED(eventLoop); +static int aeApiGetPollFd(aeApiState *state) { + UNUSED(state); return -1; } diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index adad9338017..4535b4b969e 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -156,9 +156,8 @@ class SocketPrioritizationConnTest : public SocketPrioritizationTest, public ::t TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->qos_el, (aeEventLoop *)NULL); - EXPECT_EQ(server.el->qos_el, server.el->qos_el); - EXPECT_EQ(server.el->qos_el->qos_el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->qos_apidata, (aeApiState *)NULL); + EXPECT_NE(server.el->qos_fd, -1); EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 2000ULL); aeSetQoSPreemptCheckInterval(server.el, 5000ULL); EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 5000ULL); @@ -167,6 +166,8 @@ TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { /* Newly created standalone loop must have preemption disabled (0) by default */ aeEventLoop *standalone = aeCreateEventLoop(64); ASSERT_NE(standalone, (aeEventLoop *)NULL); + EXPECT_EQ(standalone->qos_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone->qos_fd, -1); EXPECT_EQ(standalone->qos_el_preempt_check_interval_us, 0ULL); aeDeleteEventLoop(standalone); } @@ -219,9 +220,9 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { ret = connSetReadHandler(conn, dummyConnectionHandler); EXPECT_EQ(ret, C_OK); - /* Event should exist in normal loop, but not in QoS loop */ + /* Event should exist in normal mode */ EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); - EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el, read_fd) & AE_HIGH_PRIORITY, 0); /* Dynamically upgrade connection to high priority */ EXPECT_EQ(connSetPriority(conn, true), C_OK); @@ -229,13 +230,12 @@ TEST_P(SocketPrioritizationConnTest, DynamicPriorityUpdateOnActiveConnection) { int events = aeGetFileEvents(server.el, read_fd); EXPECT_NE(events & AE_READABLE, 0); EXPECT_NE(events & AE_HIGH_PRIORITY, 0); - EXPECT_NE(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); /* Dynamically downgrade connection back to normal priority */ EXPECT_EQ(connSetPriority(conn, false), C_OK); EXPECT_FALSE(connIsPriority(conn)); EXPECT_NE(aeGetFileEvents(server.el, read_fd) & AE_READABLE, 0); - EXPECT_EQ(aeGetFileEvents(server.el->qos_el, read_fd) & AE_READABLE, 0); + EXPECT_EQ(aeGetFileEvents(server.el, read_fd) & AE_HIGH_PRIORITY, 0); conn->state = CONN_STATE_NONE; connClose(conn); @@ -266,8 +266,8 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { ret = aeCreateFileEvent(server.el, normal_pipes[1][0], AE_READABLE, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - /* Register QoS event on server.el->qos_el */ - ret = aeCreateFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE, testQoSEventCallback, NULL); + /* Register QoS event on server.el with AE_HIGH_PRIORITY */ + ret = aeCreateFileEvent(server.el, qos_pipe[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); /* Write 1 byte to all pipes so they fire */ @@ -279,12 +279,12 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { if (write(qos_pipe[1], &c, 1) < 0) { } - /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el->qos_el_last_poll_us */ + /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el_last_poll_us */ advanceMockTime(1000000); g_execution_count = 0; /* Process events on main event loop. Preemption check will trigger - * polling of server.el->qos_el and execute testQoSEventCallback before processing the normal events! */ + * polling of QoS channels and execute testQoSEventCallback before processing the normal events! */ aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); /* Verify that QoS event (id 999) executed first due to preemption */ @@ -294,7 +294,7 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { /* Cleanup events and file descriptors */ aeDeleteFileEvent(server.el, normal_pipes[0][0], AE_READABLE); aeDeleteFileEvent(server.el, normal_pipes[1][0], AE_READABLE); - aeDeleteFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE); + aeDeleteFileEvent(server.el, qos_pipe[0], AE_READABLE); for (i = 0; i < 2; i++) { close(normal_pipes[i][0]); @@ -388,9 +388,9 @@ TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) ret = aeCreateFileEvent(server.el, normal_pipes[i][0], AE_READABLE, testNormalEventCallbackPreemptCheck, NULL); EXPECT_EQ(ret, AE_OK); } - ret = aeCreateFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE, testQoSEventCallback, NULL); + ret = aeCreateFileEvent(server.el, qos_pipe[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - ret = aeCreateFileEvent(server.el->qos_el, qos_pipe2[0], AE_READABLE, testQoSEventCallback, NULL); + ret = aeCreateFileEvent(server.el, qos_pipe2[0], AE_READABLE | AE_HIGH_PRIORITY, testQoSEventCallback, NULL); EXPECT_EQ(ret, AE_OK); char c = 'z'; @@ -409,17 +409,18 @@ TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); ASSERT_EQ(g_execution_count, 7); - /* At start (pre-loop), qos_el processed */ + /* At start (pre-loop), initial QoS event processed */ EXPECT_EQ(g_execution_order[0], 999); /* 1st normal event */ EXPECT_NE(g_execution_order[1], 999); /* 2nd normal event (sleeps >2000us and writes to qos_pipe2) */ EXPECT_NE(g_execution_order[2], 999); - /* Immediate preemption on the very next iteration */ - EXPECT_EQ(g_execution_order[3], 999); - /* Remaining normal events */ + /* 3rd and 4th normal events execute before mask check boundary (j & 0x03) */ + EXPECT_NE(g_execution_order[3], 999); EXPECT_NE(g_execution_order[4], 999); - EXPECT_NE(g_execution_order[5], 999); + /* Preemptive polling services qos_pipe2 at mask check boundary (j = 4) */ + EXPECT_EQ(g_execution_order[5], 999); + /* 5th normal event */ EXPECT_NE(g_execution_order[6], 999); g_qos_pipe2_write_fd = -1; @@ -429,10 +430,10 @@ TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) close(normal_pipes[i][0]); close(normal_pipes[i][1]); } - aeDeleteFileEvent(server.el->qos_el, qos_pipe[0], AE_READABLE); + aeDeleteFileEvent(server.el, qos_pipe[0], AE_READABLE); close(qos_pipe[0]); close(qos_pipe[1]); - aeDeleteFileEvent(server.el->qos_el, qos_pipe2[0], AE_READABLE); + aeDeleteFileEvent(server.el, qos_pipe2[0], AE_READABLE); close(qos_pipe2[0]); close(qos_pipe2[1]); } @@ -452,7 +453,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { fcntl(pipes[i][0], F_SETFL, O_NONBLOCK); fcntl(pipes[i][1], F_SETFL, O_NONBLOCK); - ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[BYTES_PER_FD] = {'a', 'b', 'c', 'd', 'e'}; @@ -467,13 +468,13 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredEventsFairnessAndOrdering) { for (i = 0; i < NUM_FDS; i++) total_served += counts[i]; if (total_served >= NUM_FDS * BYTES_PER_FD) break; - aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); } /* Verify all FDs were served exactly BYTES_PER_FD times without any level-triggered event being lost or starved */ for (i = 0; i < NUM_FDS; i++) { EXPECT_EQ(counts[i], BYTES_PER_FD); - aeDeleteFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE); + aeDeleteFileEvent(server.el, pipes[i][0], AE_READABLE); close(pipes[i][0]); close(pipes[i][1]); } @@ -496,14 +497,14 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve /* 1. Register Head FDs (0..6) with 3 bytes (unprocessed at the head) and Tail FDs (14..19) with 5 bytes */ for (i = 0; i <= 6; i++) { - ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[3] = {'h', 'e', 'a'}; if (write(pipes[i][1], data, 3) < 0) { } } for (i = 14; i <= 19; i++) { - ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[5] = {'t', 'a', 'i', 'l', 's'}; if (write(pipes[i][1], data, 5) < 0) { @@ -511,13 +512,13 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve } /* Process batch 1: Head and Tail FDs each fire once (reading 1 byte due to level trigger) */ - aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); for (i = 0; i <= 6; i++) EXPECT_GE(counts[i], 1); for (i = 14; i <= 19; i++) EXPECT_GE(counts[i], 1); /* 2. Register Middle FDs (7..13) with 3 bytes (new events in the middle while head/tail have remaining level-triggered data) */ for (i = 7; i <= 13; i++) { - ret = aeCreateFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE, testLevelTriggeredCallback, &counts[i]); + ret = aeCreateFileEvent(server.el, pipes[i][0], AE_READABLE | AE_HIGH_PRIORITY, testLevelTriggeredCallback, &counts[i]); EXPECT_EQ(ret, AE_OK); char data[3] = {'m', 'i', 'd'}; if (write(pipes[i][1], data, 3) < 0) { @@ -531,7 +532,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve /* 7 head FDs x 3 + 7 middle FDs x 3 + 6 tail FDs x 5 = 21 + 21 + 30 = 72 total bytes */ if (total_served >= 72) break; - aeProcessEvents(server.el->qos_el, AE_FILE_EVENTS | AE_DONT_WAIT); + aeProcessEvents(server.el, AE_FILE_EVENTS | AE_DONT_WAIT); } /* Verify every single FD across head (unprocessed), middle (new), and tail (level-triggered remaining data) was fully served */ @@ -540,7 +541,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve for (i = 14; i <= 19; i++) EXPECT_EQ(counts[i], 5); for (i = 0; i < NUM_FDS; i++) { - aeDeleteFileEvent(server.el->qos_el, pipes[i][0], AE_READABLE); + aeDeleteFileEvent(server.el, pipes[i][0], AE_READABLE); close(pipes[i][0]); close(pipes[i][1]); } @@ -548,20 +549,19 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve TEST_F(SocketPrioritizationTest, DualEventLoopResize) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->qos_el, (aeEventLoop *)NULL); + ASSERT_NE(server.el->qos_apidata, (aeApiState *)NULL); EXPECT_EQ(aeGetSetSize(server.el), 1024); - EXPECT_EQ(aeGetSetSize(server.el->qos_el), 1024); int ret = aeResizeSetSize(server.el, 2048); EXPECT_EQ(ret, AE_OK); EXPECT_EQ(aeGetSetSize(server.el), 2048); - EXPECT_EQ(aeGetSetSize(server.el->qos_el), 2048); } TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { aeEventLoop *standalone_el = aeCreateEventLoop(64); ASSERT_NE(standalone_el, (aeEventLoop *)NULL); - EXPECT_EQ(standalone_el->qos_el, (aeEventLoop *)NULL); + EXPECT_EQ(standalone_el->qos_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone_el->qos_fd, -1); int fds[2]; ASSERT_EQ(pipe(fds), 0); From 26128669713517fc8d6d366fb5b61e1953468208 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Sat, 29 Aug 2026 04:18:54 +0000 Subject: [PATCH 08/14] Address Review comments from Jim Signed-off-by: Satheesha Gowda --- src/ae.c | 55 +++++++++---------------- src/ae.h | 4 +- src/cluster_legacy.c | 4 +- src/connection.h | 8 ++-- src/io_threads.c | 4 +- src/replication.c | 4 +- src/server.c | 6 +-- src/unit/test_socket_prioritization.cpp | 23 +++++------ 8 files changed, 46 insertions(+), 62 deletions(-) diff --git a/src/ae.c b/src/ae.c index 6b883a8c093..37dac57278c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -100,7 +100,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->qos_apidata = NULL; eventLoop->qos_fd = -1; eventLoop->qos_fired = NULL; - eventLoop->qos_el_last_poll_us = 0; + eventLoop->qos_el_last_poll = 0; eventLoop->qos_el_preempt_check_interval_us = 0; eventLoop->qos_el_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ @@ -180,7 +180,6 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { void aeDeleteEventLoop(aeEventLoop *eventLoop) { if (eventLoop->qos_apidata) { - if (eventLoop->qos_fd != -1) aeDeleteFileEvent(eventLoop, eventLoop->qos_fd, AE_READABLE); aeApiFree(eventLoop->qos_apidata); zfree(eventLoop->qos_fired); } @@ -213,15 +212,8 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } aeFileEvent *fe = &eventLoop->events[fd]; - int is_priority = 0; - if ((mask & AE_HIGH_PRIORITY) || (fe->mask & AE_HIGH_PRIORITY)) { - if (eventLoop->qos_apidata != NULL) { - is_priority = 1; - mask |= AE_HIGH_PRIORITY; - } else { - mask &= ~AE_HIGH_PRIORITY; - } - } + bool is_priority = (eventLoop->qos_apidata != NULL) && ((mask | fe->mask) & AE_HIGH_PRIORITY); + if (!is_priority) mask &= ~AE_HIGH_PRIORITY; aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions @@ -243,13 +235,13 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { AE_LOCK(eventLoop); - mask &= ~AE_HIGH_PRIORITY; + mask &= ~AE_HIGH_PRIORITY; // Priority flag is removed implicitly when read & write have been removed if (fd >= eventLoop->setsize) goto done; aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) goto done; - int is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); + bool is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; /* We want to always remove AE_BARRIER if set when AE_WRITABLE @@ -504,20 +496,15 @@ static int aeProcessQoSEvents(aeEventLoop *eventLoop) { aeFireFileEvent(eventLoop, fd, mask); processed++; } - eventLoop->qos_el_last_poll_us = getMonotonicUs(); + eventLoop->qos_el_last_poll = getMonotonicUs(); /* Update INFO stats if registered and QoS events were processed */ if (eventLoop->qos_el_stats_callback != NULL && processed > 0) { - eventLoop->qos_el_stats_callback(eventLoop, eventLoop->qos_el_last_poll_us - start); + eventLoop->qos_el_stats_callback(eventLoop, eventLoop->qos_el_last_poll - start); } } return processed; } -/* Set callback to receive elapsed duration of QoS event processing */ -static void aeSetQoSStatsCallback(aeEventLoop *eventLoop, aeQoSStatsProc *cb) { - eventLoop->qos_el_stats_callback = cb; -} - /* Set the preemptive poll interval in microseconds for QoS events. * QoS events are checked periodically during normal event processing * when processing batches of normal events exceeds this interval. @@ -531,7 +518,7 @@ void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop) { /* Skip if QoS is disabled or preemptive polling is disabled */ if (eventLoop->qos_apidata == NULL || eventLoop->qos_el_preempt_check_interval_us == 0) return 0; - if (elapsedUs(eventLoop->qos_el_last_poll_us) < eventLoop->qos_el_preempt_check_interval_us) return 0; + if (elapsedUs(eventLoop->qos_el_last_poll) < eventLoop->qos_el_preempt_check_interval_us) return 0; return aeProcessQoSEvents(eventLoop); } @@ -611,12 +598,6 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { } for (j = 0; j < numevents; j++) { - /* Periodic Preemptive Poll: - * Sample monotonic clock once every 4 iterations (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead */ - if ((j & AE_QOS_PREEMPT_CHECK_MASK) == 0 && eventLoop->qos_apidata != NULL) { - processed += aeProcessQoSEventsPreemptively(eventLoop); - } - int fd = eventLoop->fired[j].fd; /* Skip processing QoS events here again as they were already processed above */ if (fd == eventLoop->qos_fd) continue; @@ -624,6 +605,12 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { aeFireFileEvent(eventLoop, fd, mask); processed++; + + /* Periodic Preemptive Poll: + * Sample monotonic clock once every 4 iterations (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead */ + if ((j & AE_QOS_PREEMPT_CHECK_MASK) == 0) { + processed += aeProcessQoSEventsPreemptively(eventLoop); + } } } /* Check time events */ @@ -697,11 +684,11 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { * Returns AE_OK on success, or AE_ERR if QoS eventloop actuation fails * (e.g., unsupported platform, memory allocation failure, or I/O error). */ -int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { assert(eventLoop != NULL); /* Create QoS polling state for internal connections (cluster bus, replication, and slot migration jobs) */ - aeApiState *qos_apidata = aeApiCreate(setsize); + aeApiState *qos_apidata = aeApiCreate(eventLoop->setsize); if (qos_apidata == NULL) return AE_ERR; int qos_fd = aeApiGetPollFd(qos_apidata); @@ -710,7 +697,7 @@ int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64 return AE_ERR; } - aeFiredEvent *qos_fired = zmalloc(sizeof(aeFiredEvent) * setsize); + aeFiredEvent *qos_fired = zmalloc(sizeof(aeFiredEvent) * eventLoop->setsize); if (qos_fired == NULL) { aeApiFree(qos_apidata); return AE_ERR; @@ -729,12 +716,8 @@ int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64 eventLoop->qos_apidata = qos_apidata; eventLoop->qos_fd = qos_fd; eventLoop->qos_fired = qos_fired; - - /* Set the QoS event preemption check interval in microseconds. */ - aeSetQoSPreemptCheckInterval(eventLoop, qosPreemptPollIntervalUs); - - /* Sets the stats callback invoked with elapsed duration whenever QoS events are processed. */ - aeSetQoSStatsCallback(eventLoop, qosStatsCallback); + eventLoop->qos_el_stats_callback = qosStatsCallback; + eventLoop->qos_el_preempt_check_interval_us = qosPreemptPollIntervalUs; return AE_OK; } diff --git a/src/ae.h b/src/ae.h index 101d9f2696d..a2a6fd4a746 100644 --- a/src/ae.h +++ b/src/ae.h @@ -137,7 +137,7 @@ typedef struct aeEventLoop { aeApiState *qos_apidata; /* Dedicated QoS polling state */ int qos_fd; /* File descriptor of QoS polling backend (-1 if disabled) */ aeFiredEvent *qos_fired; /* Fired events buffer for QoS polling */ - monotime qos_el_last_poll_us; /* Timestamp (microseconds) when QoS was last drained */ + monotime qos_el_last_poll; /* Timestamp when QoS was last drained */ uint64_t qos_el_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ aeQoSStatsProc *qos_el_stats_callback; /* Callback invoked with elapsed microseconds after draining QoS */ } aeEventLoop; @@ -170,7 +170,7 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); /* QoS event loop prototypes */ -int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, int setsize, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback); +int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback); int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop); void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index 2a0f08e14ab..f6721a27e19 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -2053,7 +2053,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth); /* Tag inbound cluster bus link as high-priority so cluster gossip and heartbeats - * are processed in qos_el ahead of normal client traffic. */ + * are processed via QoS ahead of normal client traffic. */ connSetPriority(conn, true); /* Mark as cluster-owned before any TLS accept retries so generic * accept offload routing can safely avoid client assumptions. */ @@ -6681,7 +6681,7 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t now, long clusterLink *link = createClusterLink(node); link->conn = connCreate(connTypeOfCluster()); /* Tag outbound cluster bus link as high-priority so node reconnects, gossip ping/pong, - * and failure detection heartbeats operate within qos_el. */ + * and failure detection heartbeats operate with QoS priority. */ connSetPriority(link->conn, true); connSetPrivateData(link->conn, link); connSetOwnerKind(link->conn, CONN_OWNER_CLUSTER_LINK); diff --git a/src/connection.h b/src/connection.h index 19b6c364708..f231e625ea7 100644 --- a/src/connection.h +++ b/src/connection.h @@ -179,11 +179,13 @@ typedef struct ConnectionType { /* Connection scheduling priority levels. * * High-priority connections receive preferential treatment across two subsystems: - * 1. QoS Event loop (ae): Connections that are registered with qos_el are - * processed ahead of normal priority events in the event polling loop. + * 1. Event loop QoS (ae): Sockets registered with AE_HIGH_PRIORITY are tracked + * in a dedicated QoS multiplexer state (qos_apidata) and processed ahead of + * normal priority events in the event loop, with periodic preemption during + * normal event batches to service newly arrived QoS events. * 2. I/O threads: Read and write tasks are dispatched to dedicated * high-priority queues (io_shared_inbox[CONN_PRIORITY_HIGH] and io_shared_outbox[CONN_PRIORITY_HIGH]), - * are processed ahead of normal priority queues (io_shared_inbox[CONN_PRIORITY_NORMAL] + * and are processed ahead of normal priority queues (io_shared_inbox[CONN_PRIORITY_NORMAL] * and io_shared_outbox[CONN_PRIORITY_NORMAL]), when IO threads are enabled. */ typedef enum { diff --git a/src/io_threads.c b/src/io_threads.c index 1c23b666172..c4dfe87745f 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -1188,7 +1188,7 @@ static int processOutboxBatch(mpscQueue *outbox) { /* Process completed IO jobs from worker threads back onto the main thread. * Drains the high-priority outbox first to guarantee control-plane responsiveness, - * and performs periodic preemptive polling of qos_el while consuming normal jobs. */ + * and performs periodic preemptive polling of QoS events while consuming normal jobs. */ int processIOThreadsResponses(void) { /* We don't check for threads number since some threads may return jobs then deactivate/shut-down */ @@ -1202,7 +1202,7 @@ int processIOThreadsResponses(void) { int processed = processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_HIGH]); if (processed == 0) { /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new - * high-priority events arrived on qos_el before processing normal traffic. */ + * high-priority events arrived on QoS channels before processing normal traffic. */ aeProcessQoSEventsPreemptively(server.el); } /* 3. Drain normal client events */ diff --git a/src/replication.c b/src/replication.c index dcf36900cdf..8c8036c0d63 100644 --- a/src/replication.c +++ b/src/replication.c @@ -4391,7 +4391,7 @@ void syncWithPrimary(connection *conn) { /* Create RDB connection */ server.repl_rdb_transfer_s = connCreate(connTypeOfReplication()); /* Tag connection as high-priority before connecting so non-blocking connect and - * subsequent RDB transfer events are registered directly on qos_el. */ + * subsequent RDB transfer events are registered with QoS priority. */ connSetPriority(server.repl_rdb_transfer_s, true); if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, dualChannelFullSyncWithPrimary) == C_ERR) { @@ -4444,7 +4444,7 @@ void syncWithPrimary(connection *conn) { int connectWithPrimary(void) { server.repl_transfer_s = connCreate(connTypeOfReplication()); /* Tag main replication connection as high-priority before connecting so handshake, - * heartbeat pings, and PSYNC streaming are processed in qos_el. */ + * heartbeat pings, and PSYNC streaming are processed with QoS priority. */ connSetPriority(server.repl_transfer_s, true); if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, server.repl_mptcp, syncWithPrimary) == C_ERR) { diff --git a/src/server.c b/src/server.c index 600deb34413..e3aad5a2137 100644 --- a/src/server.c +++ b/src/server.c @@ -3120,9 +3120,9 @@ void initServer(void) { serverLog(LL_WARNING, "Failed creating the event loop. Error message: '%s'", strerror(errno)); exit(1); } - /* Setup QoS event loop if multiplexer backend supports nested event loop polling. - * If multiplexer nesting is unsupported (e.g. evport, select), gracefully fallback to main event loop event processing without QoS. */ - if (aeActuateQoSEventLoopIfSupported(server.el, server.maxclients + CONFIG_FDSET_INCR, server.qos_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { + /* Setup QoS event loop if multiplexer backend supports secondary polling. + * If secondary polling is unsupported (e.g. evport, select), gracefully fallback to standard event processing without QoS. */ + if (aeActuateQoSEventLoopIfSupported(server.el, server.qos_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", aeGetApiName()); } diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index 4535b4b969e..fe219d6c4e9 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -86,7 +86,7 @@ static int g_normal_cb_count = 0; static void testNormalEventCallbackPreemptCheck(aeEventLoop *el, int fd, void *privdata, int mask) { testNormalEventCallback(el, fd, privdata, mask); g_normal_cb_count++; - /* Sleep > 2000 us on 2nd normal event and trigger 2nd QoS pipe so the next iteration immediately preempts and polls qos_el */ + /* Sleep > 2000 us on 2nd normal event and trigger 2nd QoS pipe so the next iteration immediately preempts and polls QoS events */ if (g_normal_cb_count == 2 && g_qos_pipe2_write_fd != -1) { usleep(2500); char c = 'x'; @@ -134,7 +134,7 @@ class SocketPrioritizationTest : public ::testing::Test { server.el = aeCreateEventLoop(1024); if (server.el) { - aeActuateQoSEventLoopIfSupported(server.el, 1024, 2000, NULL); + aeActuateQoSEventLoopIfSupported(server.el, 2000, NULL); } g_execution_count = 0; } @@ -279,7 +279,7 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { if (write(qos_pipe[1], &c, 1) < 0) { } - /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el_last_poll_us */ + /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el_last_poll */ advanceMockTime(1000000); g_execution_count = 0; @@ -338,7 +338,7 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { ret = aeCreateFileEvent(server.el, normal_pipe[0], AE_READABLE, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); - /* Register QoS write event on server.el->qos_el via connection wrapper */ + /* Register QoS write event on QoS multiplexer via connection wrapper */ ret = connSetWriteHandler(qos_conn, testQoSWriteCallback); EXPECT_EQ(ret, C_OK); @@ -362,7 +362,7 @@ TEST_P(SocketPrioritizationConnTest, WriteHandlerPriorityPreemption) { } TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) { - /* Verify that preemption check immediately services qos_el on the next iteration when elapsed time threshold is crossed */ + /* Verify that preemption check immediately services QoS events on the next iteration when elapsed time threshold is crossed */ int normal_pipes[5][2]; int qos_pipe[2]; int qos_pipe2[2]; @@ -415,13 +415,12 @@ TEST_F(SocketPrioritizationTest, ImmediatePreemptionDuringNormalEventProcessing) EXPECT_NE(g_execution_order[1], 999); /* 2nd normal event (sleeps >2000us and writes to qos_pipe2) */ EXPECT_NE(g_execution_order[2], 999); - /* 3rd and 4th normal events execute before mask check boundary (j & 0x03) */ + /* 3rd, 4th, and 5th normal events execute before mask check boundary at end of iteration (j = 4) */ EXPECT_NE(g_execution_order[3], 999); EXPECT_NE(g_execution_order[4], 999); - /* Preemptive polling services qos_pipe2 at mask check boundary (j = 4) */ - EXPECT_EQ(g_execution_order[5], 999); - /* 5th normal event */ - EXPECT_NE(g_execution_order[6], 999); + EXPECT_NE(g_execution_order[5], 999); + /* Preemptive polling services qos_pipe2 at mask check boundary (end of j = 4 iteration) */ + EXPECT_EQ(g_execution_order[6], 999); g_qos_pipe2_write_fd = -1; @@ -566,7 +565,7 @@ TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { int fds[2]; ASSERT_EQ(pipe(fds), 0); - /* Register with AE_HIGH_PRIORITY on standalone event loop with no qos_el */ + /* Register with AE_HIGH_PRIORITY on standalone event loop with no QoS state */ int ret = aeCreateFileEvent(standalone_el, fds[0], AE_READABLE | AE_HIGH_PRIORITY, testNormalEventCallback, NULL); EXPECT_EQ(ret, AE_OK); @@ -900,7 +899,7 @@ static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { aeEventLoop *main_loop = aeCreateEventLoop(64); ASSERT_NE(main_loop, (aeEventLoop *)NULL); - ASSERT_EQ(aeActuateQoSEventLoopIfSupported(main_loop, 64, 2000, qosMetricTestCb), AE_OK); + ASSERT_EQ(aeActuateQoSEventLoopIfSupported(main_loop, 2000, qosMetricTestCb), AE_OK); unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt; unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum; From 9e5f14d41d58e8b22a02d9221d33cc2d0d2b227c Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Sun, 30 Aug 2026 03:09:16 +0000 Subject: [PATCH 09/14] Address Review comments from Jim Signed-off-by: Satheesha Gowda --- src/ae.c | 52 +++++++++++++++++++++++++------------- src/cluster_migrateslots.c | 2 +- src/connection.c | 3 --- src/replication.c | 2 +- src/tls.c | 10 +++++--- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/ae.c b/src/ae.c index 37dac57278c..5a4498c6f26 100644 --- a/src/ae.c +++ b/src/ae.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -212,13 +213,32 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } aeFileEvent *fe = &eventLoop->events[fd]; - bool is_priority = (eventLoop->qos_apidata != NULL) && ((mask | fe->mask) & AE_HIGH_PRIORITY); - if (!is_priority) mask &= ~AE_HIGH_PRIORITY; - aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + bool old_is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); + bool new_is_priority = (mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); + + /* Handle dynamic cross-multiplexer migration if the priority of an active socket is changed */ + if (eventLoop->qos_apidata != NULL && old_is_priority != new_is_priority && fe->mask != AE_NONE) { + aeApiState *old_api = old_is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + aeApiState *new_api = new_is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + + /* 1. Remove active event mask from the old multiplexer */ + aeApiDelEvent(old_api, fd, BACKEND_MASK(fe->mask), BACKEND_MASK(fe->mask)); + + /* 2. Add combined event mask into the new multiplexer */ + int combined_backend_mask = BACKEND_MASK(fe->mask | mask); + if (combined_backend_mask) { + if (aeApiAddEvent(new_api, fd, 0, combined_backend_mask) == -1) goto done; + } + fe->mask = (fe->mask & ~AE_HIGH_PRIORITY) | (new_is_priority ? AE_HIGH_PRIORITY : 0); + } else { + bool is_priority = new_is_priority || old_is_priority; + if (!is_priority) mask &= ~AE_HIGH_PRIORITY; + aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; - int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions - if (backend_add_mask) { - if (aeApiAddEvent(target_api, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; + int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions + if (backend_add_mask) { + if (aeApiAddEvent(target_api, fd, BACKEND_MASK(fe->mask), backend_add_mask) == -1) goto done; + } } fe->mask |= mask; if (mask & AE_READABLE) fe->rfileProc = proc; @@ -586,28 +606,24 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); - /* Prioritize QoS events: if qos_fd fired, drain QoS events + /* Prioritize QoS events: Always drain priority events * immediately before processing normal events. */ - if (eventLoop->qos_fd != -1) { - for (j = 0; j < numevents; j++) { - if (eventLoop->fired[j].fd == eventLoop->qos_fd) { - processed += aeProcessQoSEvents(eventLoop); - break; - } - } - } + processed += aeProcessQoSEvents(eventLoop); + /* Process normal file events */ for (j = 0; j < numevents; j++) { int fd = eventLoop->fired[j].fd; - /* Skip processing QoS events here again as they were already processed above */ + /* Skip processing QoS FD events again as they were already processed above */ if (fd == eventLoop->qos_fd) continue; int mask = eventLoop->fired[j].mask; aeFireFileEvent(eventLoop, fd, mask); processed++; - /* Periodic Preemptive Poll: - * Sample monotonic clock once every 4 iterations (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead */ + /* Periodic Preemptive Polling of high priority events: This is to + * ensure that high priority events are processed in a timely manner + * even when there are long running normal events. */ + /* Sample monotonic clock once every (AE_QOS_PREEMPT_CHECK_MASK) to amortize vDSO overhead. */ if ((j & AE_QOS_PREEMPT_CHECK_MASK) == 0) { processed += aeProcessQoSEventsPreemptively(eventLoop); } diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index 18785e16c3d..db4e65cfc05 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -1749,7 +1749,7 @@ int slotExportJobBeginSnapshotToTargetSocket(slotMigrationJob *job) { serverLog(LL_NOTICE, "Started child process %ld for slot migration %s", (long)childpid, job->description); close(slot_migration_pipe_write); /* close write in parent so that it can detect the close on the child. */ - if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE, slotMigrationPipeReadHandler, NULL) == AE_ERR) { + if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE | AE_HIGH_PRIORITY, slotMigrationPipeReadHandler, NULL) == AE_ERR) { serverPanic("Unrecoverable error creating server.slot_migration_pipe_read file event."); } close(safe_to_exit_pipe); diff --git a/src/connection.c b/src/connection.c index 055cb10d2c6..2c714f14455 100644 --- a/src/connection.c +++ b/src/connection.c @@ -200,9 +200,6 @@ int connSetPriority(connection *conn, bool is_priority) { bool old_priority = conn->is_priority; conn->is_priority = is_priority; - /* Delete existing registration (AE routes delete to the correct loop) */ - aeDeleteFileEvent(server.el, conn->fd, AE_READABLE | AE_WRITABLE); - /* If transport has custom state updater (e.g. TLS), delegate to it */ if (conn->type && conn->type->update_state) { conn->type->update_state(conn); diff --git a/src/replication.c b/src/replication.c index 8c8036c0d63..5ec718b6f0e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1963,7 +1963,7 @@ void slotMigrationPipeWriteHandler(struct connection *conn) { /* Remove the write handler and set up the pipe read handler. */ connSetWriteHandler(conn, NULL); target->repl_data->repl_last_partial_write = 0; - if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE, slotMigrationPipeReadHandler, NULL) == AE_ERR) { + if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE | AE_HIGH_PRIORITY, slotMigrationPipeReadHandler, NULL) == AE_ERR) { serverPanic("Unrecoverable error creating server.slot_migration_pipe_read file event."); } } diff --git a/src/tls.c b/src/tls.c index 490dff52534..84c1e03e409 100644 --- a/src/tls.c +++ b/src/tls.c @@ -1200,13 +1200,14 @@ static int updateStateAfterSSLIO(tls_connection *conn, int ret_value, int update static void registerSSLEvent(tls_connection *conn) { int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); + bool priority_changed = ((mask & AE_HIGH_PRIORITY) != (priority_flag & AE_HIGH_PRIORITY)); if (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ) { if (mask & AE_WRITABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); - if (!(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); + if (!(mask & AE_READABLE) || priority_changed) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); } else if (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE) { if (mask & AE_READABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (!(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); + if (!(mask & AE_WRITABLE) || priority_changed) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); } else { serverAssert(0); } @@ -1250,13 +1251,14 @@ static void updateSSLEvent(tls_connection *conn) { int priority_flag = connGetAEPriorityFlag(&conn->c); int mask = aeGetFileEvents(server.el, conn->c.fd); + bool priority_changed = ((mask & AE_HIGH_PRIORITY) != (priority_flag & AE_HIGH_PRIORITY)); int need_read = conn->c.read_handler || (conn->c.write_handler && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)); int need_write = conn->c.write_handler || (conn->c.read_handler && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)); - if (need_read && !(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); + if (need_read && (!(mask & AE_READABLE) || priority_changed)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE | priority_flag, tlsEventHandler, conn); if (!need_read && (mask & AE_READABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); - if (need_write && !(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); + if (need_write && (!(mask & AE_WRITABLE) || priority_changed)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE | priority_flag, tlsEventHandler, conn); if (!need_write && (mask & AE_WRITABLE)) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); } From f316e1bd86119f4817acd00106a965ceb49d08f7 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Mon, 31 Aug 2026 17:14:04 +0000 Subject: [PATCH 10/14] Address Review comments from Jim Signed-off-by: Satheesha Gowda --- src/cluster_migrateslots.c | 2 +- src/connection.h | 22 ---------------------- src/io_threads.c | 15 +++++++++++++++ src/replication.c | 2 +- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/cluster_migrateslots.c b/src/cluster_migrateslots.c index db4e65cfc05..18785e16c3d 100644 --- a/src/cluster_migrateslots.c +++ b/src/cluster_migrateslots.c @@ -1749,7 +1749,7 @@ int slotExportJobBeginSnapshotToTargetSocket(slotMigrationJob *job) { serverLog(LL_NOTICE, "Started child process %ld for slot migration %s", (long)childpid, job->description); close(slot_migration_pipe_write); /* close write in parent so that it can detect the close on the child. */ - if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE | AE_HIGH_PRIORITY, slotMigrationPipeReadHandler, NULL) == AE_ERR) { + if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE, slotMigrationPipeReadHandler, NULL) == AE_ERR) { serverPanic("Unrecoverable error creating server.slot_migration_pipe_read file event."); } close(safe_to_exit_pipe); diff --git a/src/connection.h b/src/connection.h index f231e625ea7..6226b1c876e 100644 --- a/src/connection.h +++ b/src/connection.h @@ -176,28 +176,6 @@ typedef struct ConnectionType { int (*is_closing)(connection *conn); // return 1 if connection is closed } ConnectionType; -/* Connection scheduling priority levels. - * - * High-priority connections receive preferential treatment across two subsystems: - * 1. Event loop QoS (ae): Sockets registered with AE_HIGH_PRIORITY are tracked - * in a dedicated QoS multiplexer state (qos_apidata) and processed ahead of - * normal priority events in the event loop, with periodic preemption during - * normal event batches to service newly arrived QoS events. - * 2. I/O threads: Read and write tasks are dispatched to dedicated - * high-priority queues (io_shared_inbox[CONN_PRIORITY_HIGH] and io_shared_outbox[CONN_PRIORITY_HIGH]), - * and are processed ahead of normal priority queues (io_shared_inbox[CONN_PRIORITY_NORMAL] - * and io_shared_outbox[CONN_PRIORITY_NORMAL]), when IO threads are enabled. - */ -typedef enum { - /* Normal priority connections - used for normal client connections */ - CONN_PRIORITY_NORMAL = 0, - /* High priority connections - used for critical internal communication such as - * cluster bus messages, slot migration, and replication streams */ - CONN_PRIORITY_HIGH, - /* Number of priority levels */ - CONN_PRIORITY_COUNT -} connPriority; - struct connection { ConnectionType *type; ConnectionState state; diff --git a/src/io_threads.c b/src/io_threads.c index c4dfe87745f..a9fc8d3017c 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -19,6 +19,21 @@ #define IO_SPMC_QUEUE_SIZE 4096 #define IO_SPSC_QUEUE_SIZE 4096 +/* QoS Swim Lanes for I/O threads + * High priority queues: reserved for critical internal communication such as + * cluster bus messages, slot migration, and replication streams + * Normal priority queues: used for normal client connections + */ +typedef enum { + /* Normal priority connections - used for normal client connections */ + CONN_PRIORITY_NORMAL = 0, + /* High priority connections - used for critical internal communication such as + * cluster bus messages, slot migration, and replication streams */ + CONN_PRIORITY_HIGH, + /* Number of priority levels */ + CONN_PRIORITY_COUNT +} connPriority; + static _Thread_local int thread_id = 0; static _Thread_local mpscTicket io_thread_ticket[CONN_PRIORITY_COUNT] = {0}; /* Backlog of responses when io_shared_outbox is full. Should be rare. */ diff --git a/src/replication.c b/src/replication.c index 5ec718b6f0e..8c8036c0d63 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1963,7 +1963,7 @@ void slotMigrationPipeWriteHandler(struct connection *conn) { /* Remove the write handler and set up the pipe read handler. */ connSetWriteHandler(conn, NULL); target->repl_data->repl_last_partial_write = 0; - if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE | AE_HIGH_PRIORITY, slotMigrationPipeReadHandler, NULL) == AE_ERR) { + if (aeCreateFileEvent(server.el, server.slot_migration_pipe_read, AE_READABLE, slotMigrationPipeReadHandler, NULL) == AE_ERR) { serverPanic("Unrecoverable error creating server.slot_migration_pipe_read file event."); } } From c6420e3f882aec7899e98531c26ccb0bd175a9ad Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Mon, 7 Sep 2026 04:46:47 +0000 Subject: [PATCH 11/14] Incorporate review suggestions from Viktor Signed-off-by: Satheesha Gowda --- src/cluster_legacy.c | 5 +- src/commands.def | 4 +- src/commands/client-kill.json | 2 +- src/commands/client-list.json | 2 +- src/commands/cluster-links.json | 4 - src/io_threads.c | 80 ++++++++++++-------- src/io_threads.h | 2 +- tests/unit/cluster/links.tcl | 3 +- tests/unit/cluster/socket-prioritization.tcl | 8 -- 9 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index f6721a27e19..2a039086435 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -7689,7 +7689,7 @@ sds clusterGenNodesDescription(client *c, int filter, int tls_primary) { /* Add to the output buffer of the given client the description of the given cluster link. * The description is a map with each entry being an attribute of the link. */ void addReplyClusterLinkDescription(client *c, clusterLink *link) { - addReplyMapLen(c, 7); + addReplyMapLen(c, 6); addReplyBulkCString(c, "direction"); addReplyBulkCString(c, link->inbound ? "from" : "to"); @@ -7721,9 +7721,6 @@ void addReplyClusterLinkDescription(client *c, clusterLink *link) { addReplyBulkCString(c, "send-buffer-used"); addReplyLongLong(c, link->send_msg_queue_mem); - - addReplyBulkCString(c, "qos"); - addReplyBulkCString(c, connIsPriority(link->conn) ? "prioritized" : "normal"); } /* Add to the output buffer of the given client an array of cluster link descriptions, diff --git a/src/commands.def b/src/commands.def index 3606e9f1161..4835e39ca76 100644 --- a/src/commands.def +++ b/src/commands.def @@ -1470,7 +1470,7 @@ commandHistory CLIENT_KILL_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","`ID` option accepts multiple IDs."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, -{"10.0.0","Added `H` flag to indicate high priority clients."}, +{"9.2.0","Added `H` flag to indicate high priority clients."}, }; #endif @@ -1566,7 +1566,7 @@ commandHistory CLIENT_LIST_History[] = { {"8.0.0","Replaced `master` `TYPE` with `primary`. `master` still supported for backward compatibility."}, {"8.1.0","Added filters USER, ADDR, LADDR, SKIPME, and MAXAGE."}, {"9.0.0","Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP."}, -{"10.0.0","Added `H` flag to indicate high priority clients."}, +{"9.2.0","Added `H` flag to indicate high priority clients."}, }; #endif diff --git a/src/commands/client-kill.json b/src/commands/client-kill.json index 1ccd5ccb224..529de31a5c3 100644 --- a/src/commands/client-kill.json +++ b/src/commands/client-kill.json @@ -45,7 +45,7 @@ "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." ], [ - "10.0.0", + "9.2.0", "Added `H` flag to indicate high priority clients." ] ], diff --git a/src/commands/client-list.json b/src/commands/client-list.json index 2b15e25fd9e..c934935e441 100644 --- a/src/commands/client-list.json +++ b/src/commands/client-list.json @@ -45,7 +45,7 @@ "Added filters NAME, IDLE, FLAGS, LIB-NAME, LIB-VER, DB, CAPA, and IP. And negative filters NOT-ID, NOT-TYPE, NOT-ADDR, NOT-LADDR, NOT-USER, NOT-FLAGS, NOT-NAME, NOT-LIB-NAME, NOT-LIB-VER, NOT-DB, NOT-CAPA, NOT-IP." ], [ - "10.0.0", + "9.2.0", "Added `H` flag to indicate high priority clients." ] ], diff --git a/src/commands/cluster-links.json b/src/commands/cluster-links.json index 865289ff2f6..eef363367ca 100644 --- a/src/commands/cluster-links.json +++ b/src/commands/cluster-links.json @@ -51,10 +51,6 @@ "send-buffer-used": { "description": "Size of the portion of the link's send buffer that is currently holding data(messages).", "type": "integer" - }, - "qos": { - "description": "Quality of service priority level of the link connection.", - "type": "string" } }, "additionalProperties": false diff --git a/src/io_threads.c b/src/io_threads.c index a9fc8d3017c..d4fde906129 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -25,26 +25,26 @@ * Normal priority queues: used for normal client connections */ typedef enum { - /* Normal priority connections - used for normal client connections */ - CONN_PRIORITY_NORMAL = 0, - /* High priority connections - used for critical internal communication such as + /* Normal priority jobs - used for normal client connections */ + JOB_PRIORITY_NORMAL = 0, + /* High priority jobs - used for critical internal communication such as * cluster bus messages, slot migration, and replication streams */ - CONN_PRIORITY_HIGH, + JOB_PRIORITY_HIGH, /* Number of priority levels */ - CONN_PRIORITY_COUNT -} connPriority; + JOB_PRIORITY_COUNT +} jobPriority; static _Thread_local int thread_id = 0; -static _Thread_local mpscTicket io_thread_ticket[CONN_PRIORITY_COUNT] = {0}; +static _Thread_local mpscTicket io_thread_ticket[JOB_PRIORITY_COUNT] = {0}; /* Backlog of responses when io_shared_outbox is full. Should be rare. */ -static _Thread_local list *pending_io_responses[CONN_PRIORITY_COUNT] = {NULL, NULL}; +static _Thread_local list *pending_io_responses[JOB_PRIORITY_COUNT] = {NULL, NULL}; static pthread_t io_threads[IO_THREADS_MAX_NUM] = {0}; static pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; static int cur_epoll_thread = 0; // Main -> IO: Shared Queue (Single Producer Multi Consumer) where all IO threads pull jobs from -static spmcQueue io_shared_inbox[CONN_PRIORITY_COUNT] = {0}; +static spmcQueue io_shared_inbox[JOB_PRIORITY_COUNT] = {0}; // IO -> Main: Response Channel (Multi Producer Single Consumer) used by IO threads to send results back to main-thread -static mpscQueue io_shared_outbox[CONN_PRIORITY_COUNT] = {0}; +static mpscQueue io_shared_outbox[JOB_PRIORITY_COUNT] = {0}; // Main -> IO (Thread-Specific) for tasks that must run on specific IO thread where IO threads check their private inbox before the shared queue static spscQueue io_private_inbox[IO_THREADS_MAX_NUM] = {0}; static size_t io_jobs_submitted; @@ -59,6 +59,10 @@ _Atomic long long used_active_time_io_thread[IO_THREADS_MAX_NUM] = {0}; #define JOB_TAG_MASK 0x7 #define JOB_PTR_MASK (~(uintptr_t)JOB_TAG_MASK) +static inline jobPriority getJobPriority(const client *c) { + return (c && connIsPriority(c->conn)) ? JOB_PRIORITY_HIGH : JOB_PRIORITY_NORMAL; +} + static inline void *tagJob(void *ptr, int type) { return (void *)((uintptr_t)ptr | type); } @@ -209,8 +213,8 @@ void IOThreadsAfterSleep(int numevents) { if (now - last_sample_time < IO_SAMPLE_RATE_MS) return; last_sample_time = now; - size_t q_size = spmcSize(&io_shared_inbox[CONN_PRIORITY_NORMAL]) + spmcSize(&io_shared_inbox[CONN_PRIORITY_HIGH]); - spmc_size_sum += q_size; + spmc_size_sum += spmcSize(&io_shared_inbox[JOB_PRIORITY_NORMAL]); + spmc_size_sum += spmcSize(&io_shared_inbox[JOB_PRIORITY_HIGH]); sample_count++; trackInstantaneousMetric(STATS_METRIC_IO_WAIT, spmc_size_sum, sample_count, 1); @@ -245,7 +249,7 @@ void IOThreadsAfterSleep(int numevents) { /* Don't suspend if work remains in the specific thread's queue... */ if (!spscIsEmpty(&io_private_inbox[tid])) return; /* ...or if we are dropping to 1 thread but the global queue still has work */ - if (target == 1 && (!spmcIsEmpty(&io_shared_inbox[CONN_PRIORITY_NORMAL]) || !spmcIsEmpty(&io_shared_inbox[CONN_PRIORITY_HIGH]))) return; + if (target == 1 && (!spmcIsEmpty(&io_shared_inbox[JOB_PRIORITY_NORMAL]) || !spmcIsEmpty(&io_shared_inbox[JOB_PRIORITY_HIGH]))) return; pthread_mutex_lock(&io_threads_mutex[tid]); server.active_io_threads_num--; @@ -262,7 +266,7 @@ void ioThreadPoll(aeEventLoop *el) { atomic_store_explicit(&server.io_poll_state, AE_IO_STATE_DONE, memory_order_release); } -static void flushPendingList(list **pending_list, mpscQueue *outbox, mpscTicket *ticket, int blocking) { +static void flushPendingIOResponsesList(list **pending_list, mpscQueue *outbox, mpscTicket *ticket, int blocking) { if (*pending_list == NULL) return; listIter li; listNode *ln; @@ -292,8 +296,8 @@ static void flushPendingList(list **pending_list, mpscQueue *outbox, mpscTicket } static void flushPendingIOResponses(int blocking) { - flushPendingList(&pending_io_responses[CONN_PRIORITY_HIGH], &io_shared_outbox[CONN_PRIORITY_HIGH], &io_thread_ticket[CONN_PRIORITY_HIGH], blocking); - flushPendingList(&pending_io_responses[CONN_PRIORITY_NORMAL], &io_shared_outbox[CONN_PRIORITY_NORMAL], &io_thread_ticket[CONN_PRIORITY_NORMAL], blocking); + flushPendingIOResponsesList(&pending_io_responses[JOB_PRIORITY_HIGH], &io_shared_outbox[JOB_PRIORITY_HIGH], &io_thread_ticket[JOB_PRIORITY_HIGH], blocking); + flushPendingIOResponsesList(&pending_io_responses[JOB_PRIORITY_NORMAL], &io_shared_outbox[JOB_PRIORITY_NORMAL], &io_thread_ticket[JOB_PRIORITY_NORMAL], blocking); } /* Define a cleanup function that will clean all thread resources */ @@ -393,12 +397,12 @@ static void *IOThreadMain(void *myid) { /* PRIORITY 2: Shared Global Queue (SPMC) * Only checked after SPSC is drained. */ void *tagged_job; - if ((tagged_job = spmcDequeue(&io_shared_inbox[CONN_PRIORITY_HIGH])) != NULL) { + if ((tagged_job = spmcDequeue(&io_shared_inbox[JOB_PRIORITY_HIGH])) != NULL) { processTaggedSPMCJob(tagged_job); processed++; } - if ((tagged_job = spmcDequeue(&io_shared_inbox[CONN_PRIORITY_NORMAL])) != NULL) { + if ((tagged_job = spmcDequeue(&io_shared_inbox[JOB_PRIORITY_NORMAL])) != NULL) { processTaggedSPMCJob(tagged_job); processed++; } @@ -410,7 +414,7 @@ static void *IOThreadMain(void *myid) { /* If both queues were empty (no processing done), wait for signal. */ if (processed == 0) { int has_pending = 0; - for (int p = 0; p < CONN_PRIORITY_COUNT; p++) { + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { if (pending_io_responses[p]) has_pending = 1; } if (unlikely(has_pending)) { @@ -500,7 +504,12 @@ int updateIOThreads(const char **err) { * in that state, we will deadlock (Main thread waits for worker, Worker waits for queue space). */ size_t pending = getPendingIOResponsesCount(); - if (pending > io_shared_outbox[CONN_PRIORITY_NORMAL].queue_size || pending > io_shared_outbox[CONN_PRIORITY_HIGH].queue_size) { + /* Since pending is the sum of all in-flight read/write jobs, in the worst-case scenario where + * 100% of the traffic happens to be on one priority lane, that outbox will receive at most pending + * responses. If pending fits within each queue's capacity, neither queue can ever overflow or cause + * workers to block while draining*/ + if (pending > io_shared_outbox[JOB_PRIORITY_NORMAL].queue_size || + pending > io_shared_outbox[JOB_PRIORITY_HIGH].queue_size) { if (err) *err = "Can't update IO threads under load, try again later"; return 0; } @@ -539,7 +548,7 @@ void initIOThreads(int prev_threads_num) { server.active_io_threads_num = 1; /* We start with threads not active. */ server.io_poll_state = AE_IO_STATE_NONE; server.io_ae_fired_events = 0; - for (int p = 0; p < CONN_PRIORITY_COUNT; p++) { + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { spmcInit(&io_shared_inbox[p], IO_SPMC_QUEUE_SIZE); mpscInit(&io_shared_outbox[p], IO_MPSC_QUEUE_SIZE); } @@ -626,7 +635,7 @@ int trySendReadToIOThreads(client *c) { c->io_read_state = CLIENT_PENDING_IO; connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); - int qidx = connIsPriority(c->conn); + jobPriority qidx = getJobPriority(c); if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], tagJob(c, JOB_REQ_READ_CLIENT)) == false)) { c->read_flags = 0; c->io_read_state = CLIENT_IDLE; @@ -689,7 +698,7 @@ int trySendWriteToIOThreads(client *c) { connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_WRITE_CLIENT); - int qidx = connIsPriority(c->conn); + jobPriority qidx = getJobPriority(c); if (unlikely(spmcEnqueue(&io_shared_inbox[qidx], job) == false)) { c->io_write_state = CLIENT_IDLE; connSetPostponeUpdateState(c->conn, 0); @@ -972,7 +981,7 @@ int tryOffloadFreeObjToIOThreads(robj *obj) { if (obj->encoding != OBJ_ENCODING_RAW || obj->type != OBJ_STRING) return C_ERR; void *job = tagJob(obj, JOB_REQ_FREE_OBJ); - if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], job) == false)) return C_ERR; + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], job) == false)) return C_ERR; io_jobs_submitted++; server.stat_io_freed_objects++; return C_OK; @@ -1019,7 +1028,7 @@ void trySendPollJobToIOThreads(void) { /* Use SPMC to minimize polling overhead. At high thread counts, use private SPSC queues for lower latency. */ if (server.active_io_threads_num <= 9) { - if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], tagJob(server.el, JOB_REQ_POLL)) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], tagJob(server.el, JOB_REQ_POLL)) == false)) { server.io_poll_state = AE_IO_STATE_NONE; aeSetPollProtect(server.el, 0); return; @@ -1038,12 +1047,16 @@ void trySendPollJobToIOThreads(void) { io_jobs_submitted++; } -void sendToMainThread(client *c, int type) { - int qidx = connIsPriority(c->conn); +void sendToMainThread(void *data, int type) { + jobPriority qidx = JOB_PRIORITY_NORMAL; + if (type == JOB_RES_READ_CLIENT || type == JOB_RES_WRITE_CLIENT) { + client *c = (client *)data; + qidx = getJobPriority(c); + } if (unlikely(pending_io_responses[qidx])) { - flushPendingList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); + flushPendingIOResponsesList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); } - void *job = tagJob(c, type); + void *job = tagJob(data, type); if (unlikely(pending_io_responses[qidx] || !mpscEnqueue(&io_shared_outbox[qidx], job, &io_thread_ticket[qidx]))) { if (pending_io_responses[qidx] == NULL) { pending_io_responses[qidx] = listCreate(); @@ -1100,7 +1113,7 @@ int trySendAcceptToIOThreads(connection *conn) { connSetPostponeUpdateState(c->conn, clientConnPostponeMaskFromIOState(c)); void *job = tagJob(c, JOB_REQ_ACCEPT); - if (unlikely(spmcEnqueue(&io_shared_inbox[CONN_PRIORITY_NORMAL], job) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_NORMAL], job) == false)) { c->io_read_state = CLIENT_IDLE; c->flag.pending_read = 0; connSetPostponeUpdateState(c->conn, 0); @@ -1183,11 +1196,12 @@ static int processOutboxBatch(mpscQueue *outbox) { void *data; int job_type; untagJob(jobs[i], &data, &job_type); - client *c = (client *)data; if (job_type == JOB_RES_READ_CLIENT) { + client *c = (client *)data; serverAssert(c->io_read_state == CLIENT_COMPLETED_IO); read_jobs[read_count++] = c; } else if (job_type == JOB_RES_WRITE_CLIENT) { + client *c = (client *)data; serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); write_jobs[write_count++] = c; } else { @@ -1214,14 +1228,14 @@ int processIOThreadsResponses(void) { /* Loop until we consume all pending jobs */ while (1) { /* 1. Strict Priority: First, drain high-priority events (cluster bus, replication, and slot migration jobs) */ - int processed = processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_HIGH]); + int processed = processOutboxBatch(&io_shared_outbox[JOB_PRIORITY_HIGH]); if (processed == 0) { /* 2. Preemptive Poll: When high-priority outbox is empty, check if any new * high-priority events arrived on QoS channels before processing normal traffic. */ aeProcessQoSEventsPreemptively(server.el); } /* 3. Drain normal client events */ - processed += processOutboxBatch(&io_shared_outbox[CONN_PRIORITY_NORMAL]); + processed += processOutboxBatch(&io_shared_outbox[JOB_PRIORITY_NORMAL]); total_processed += processed; if (processed == 0) return total_processed; } diff --git a/src/io_threads.h b/src/io_threads.h index 88e440d29c2..06d4cf889cb 100644 --- a/src/io_threads.h +++ b/src/io_threads.h @@ -66,6 +66,6 @@ long long getIOThreadActiveTimeMicroseconds(int id); int clientHasPendingIO(struct client *c); int processIOThreadsResponses(void); int getCurTid(void); -void sendToMainThread(client *c, int type); +void sendToMainThread(void *data, int type); #endif /* IO_THREADS_H */ diff --git a/tests/unit/cluster/links.tcl b/tests/unit/cluster/links.tcl index 2f54551f7fa..9cf61d9f741 100644 --- a/tests/unit/cluster/links.tcl +++ b/tests/unit/cluster/links.tcl @@ -186,14 +186,13 @@ start_cluster 3 0 {tags {external:skip cluster network}} { set lines [R 0 cluster links] foreach l $lines { if {$l eq {}} continue - assert_equal [llength $l] 14 + assert_equal [llength $l] 12 assert_equal 1 [dict exists $l "direction"] assert_equal 1 [dict exists $l "node"] assert_equal 1 [dict exists $l "create-time"] assert_equal 1 [dict exists $l "events"] assert_equal 1 [dict exists $l "send-buffer-allocated"] assert_equal 1 [dict exists $l "send-buffer-used"] - assert_equal 1 [dict exists $l "qos"] } } diff --git a/tests/unit/cluster/socket-prioritization.tcl b/tests/unit/cluster/socket-prioritization.tcl index 7614c21cd01..9fb6b5bba40 100644 --- a/tests/unit/cluster/socket-prioritization.tcl +++ b/tests/unit/cluster/socket-prioritization.tcl @@ -119,12 +119,4 @@ start_cluster 2 2 {tags {socket-prioritization external:skip cluster}} { fail "Slot 0 was not migrated to R1" } } - - test "Verify cluster nodes connections are prioritized from beginning" { - set links [R 0 CLUSTER LINKS] - assert {[llength $links] > 0} - foreach link $links { - assert_equal "prioritized" [dict get $link qos] - } - } } From 23b1ed3ce38c548d9479a24fd18ff94b1526a6af Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Tue, 8 Sep 2026 21:18:06 +0000 Subject: [PATCH 12/14] Incorporate TSC suggestion for metrics naming convention Signed-off-by: Satheesha Gowda --- src/latency.h | 14 ++++----- src/server.c | 40 +++++++++++-------------- src/server.h | 8 ++--- src/unit/test_socket_prioritization.cpp | 10 +++---- tests/unit/info.tcl | 16 +++++----- tests/unit/socket-prioritization.tcl | 18 +++++------ 6 files changed, 49 insertions(+), 57 deletions(-) diff --git a/src/latency.h b/src/latency.h index af0c0285579..eed2f14f169 100644 --- a/src/latency.h +++ b/src/latency.h @@ -103,13 +103,13 @@ typedef struct durationStats { } durationStats; typedef enum { - EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop - EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands - EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop - EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and - // AOF) - EL_DURATION_TYPE_QOS_EL, // cumulative time duration metric of QoS eventloop - EL_DURATION_TYPE_QOS_CMD, // cumulative time duration metric of QoS handler execution + EL_DURATION_TYPE_EL = 0, // cumulative time duration metric of the whole eventloop + EL_DURATION_TYPE_CMD, // cumulative time duration metric of executing commands + EL_DURATION_TYPE_AOF, // cumulative time duration metric of flushing AOF in eventloop + EL_DURATION_TYPE_CRON, // cumulative time duration metric of cron (serverCron and beforeSleep, but excluding IO and + // AOF) + EL_DURATION_TYPE_PRIORITY_EL, // cumulative time duration metric of priority eventloop + EL_DURATION_TYPE_PRIORITY_CMD, // cumulative time duration metric of priority commands execution EL_DURATION_TYPE_NUM } DurationType; diff --git a/src/server.c b/src/server.c index e3aad5a2137..8a7b40de2b9 100644 --- a/src/server.c +++ b/src/server.c @@ -1630,10 +1630,6 @@ long long serverCron(struct aeEventLoop *eventLoop, long long id, void *clientDa factor); trackInstantaneousMetric(STATS_METRIC_EL_DURATION, server.duration_stats[EL_DURATION_TYPE_EL].sum, server.duration_stats[EL_DURATION_TYPE_EL].cnt, 1); - trackInstantaneousMetric(STATS_METRIC_QOS_EL_CYCLE, server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, current_time, - factor); - trackInstantaneousMetric(STATS_METRIC_QOS_EL_DURATION, server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, - server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, 1); } cronUpdateMemoryStats(); @@ -2167,18 +2163,18 @@ void afterSleep(struct aeEventLoop *eventLoop, int numevents) { IOThreadsAfterSleep(numevents); } -/* Callback invoked by the event loop after draining QoS events. - * Records QoS eventloop duration and updates peak commands executed per QoS cycle. */ +/* Callback invoked by the event loop after draining priority events. + * Records priority eventloop duration and updates peak commands executed per priority cycle. */ static void qosStatsCallback(struct aeEventLoop *el, uint64_t duration_us) { UNUSED(el); - durationAddSample(EL_DURATION_TYPE_QOS_EL, duration_us); - unsigned long long qos_cmds = server.duration_stats[EL_DURATION_TYPE_QOS_CMD].cnt; - if (qos_cmds > (unsigned long long)server.qos_el_cmd_cnt_prev) { - long long el_cmd_cnt = qos_cmds - server.qos_el_cmd_cnt_prev; - if (el_cmd_cnt > server.qos_el_cmd_cnt_max) { - server.qos_el_cmd_cnt_max = el_cmd_cnt; + durationAddSample(EL_DURATION_TYPE_PRIORITY_EL, duration_us); + unsigned long long priority_cmds = server.duration_stats[EL_DURATION_TYPE_PRIORITY_CMD].cnt; + if (priority_cmds > (unsigned long long)server.priority_el_cmd_cnt_prev) { + long long el_cmd_cnt = priority_cmds - server.priority_el_cmd_cnt_prev; + if (el_cmd_cnt > server.priority_el_cmd_cnt_max) { + server.priority_el_cmd_cnt_max = el_cmd_cnt; } - server.qos_el_cmd_cnt_prev = qos_cmds; + server.priority_el_cmd_cnt_prev = priority_cmds; } } @@ -2965,8 +2961,8 @@ void resetServerStats(void) { server.stat_reply_buffer_expands = 0; memset(server.duration_stats, 0, sizeof(durationStats) * EL_DURATION_TYPE_NUM); server.el_cmd_cnt_max = 0; - server.qos_el_cmd_cnt_max = 0; - server.qos_el_cmd_cnt_prev = 0; + server.priority_el_cmd_cnt_max = 0; + server.priority_el_cmd_cnt_prev = 0; server.stat_active_time = 0; server.el_iteration_active = false; server.stat_total_prefetch_batches = 0; @@ -4252,7 +4248,7 @@ void call(client *c, int flags) { durationAddSample(EL_DURATION_TYPE_CMD, duration); /* Attribute command execution latency for high-priority client connections. */ if (connIsPriority(c->conn)) { - durationAddSample(EL_DURATION_TYPE_QOS_CMD, duration); + durationAddSample(EL_DURATION_TYPE_PRIORITY_CMD, duration); } } } @@ -6829,11 +6825,9 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CMD].sum, "instantaneous_eventloop_cycles_per_sec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_CYCLE), "instantaneous_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_EL_DURATION), - "qos_eventloop_cycles:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, - "qos_eventloop_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, - "qos_eventloop_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_CMD].sum, - "instantaneous_qos_eventloop_cycles_per_sec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_QOS_EL_CYCLE), - "instantaneous_qos_eventloop_duration_usec:%llu\r\n", getInstantaneousMetric(STATS_METRIC_QOS_EL_DURATION))); + "eventloop_priority_cycles:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt, + "eventloop_priority_duration_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum, + "eventloop_priority_duration_cmd_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_CMD].sum)); info = genValkeyInfoStringACLStats(info); } @@ -7120,8 +7114,8 @@ sds genValkeyInfoString(dict *section_dict, int all_sections, int everything) { "eventloop_duration_cron_sum:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_CRON].sum, "eventloop_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_EL].max, "eventloop_cmd_per_cycle_max:%lld\r\n", server.el_cmd_cnt_max, - "qos_eventloop_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_QOS_EL].max, - "qos_eventloop_cmd_per_cycle_max:%lld\r\n", server.qos_el_cmd_cnt_max, + "eventloop_priority_duration_max:%llu\r\n", server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].max, + "eventloop_priority_cmd_per_cycle_max:%lld\r\n", server.priority_el_cmd_cnt_max, "io_threaded_reads_pending:%lld\r\n", server.stat_io_reads_pending, "io_threaded_writes_pending:%lld\r\n", server.stat_io_writes_pending)); diff --git a/src/server.h b/src/server.h index 445fab6ab83..ef4d5209307 100644 --- a/src/server.h +++ b/src/server.h @@ -217,8 +217,6 @@ typedef enum { STATS_METRIC_EL_DURATION, /* Eventloop duration. */ STATS_METRIC_IO_WAIT, /* IO queue size */ STATS_METRIC_MAIN_THREAD_ACTIVE_TIME, /* Main-thread active time */ - STATS_METRIC_QOS_EL_CYCLE, /* Number of QoS eventloop cycles per sec. */ - STATS_METRIC_QOS_EL_DURATION, /* QoS eventloop duration in microseconds. */ STATS_METRIC_COUNT /* Total count */ } instantaneous_metric_type; @@ -2013,9 +2011,9 @@ struct valkeyServer { * Note that commands in transactions are also counted. */ long long el_cmd_cnt_start; long long el_cmd_cnt_max; - /* Record the previous baseline and peak number of QoS commands executed in one QoS cycle. */ - long long qos_el_cmd_cnt_prev; - long long qos_el_cmd_cnt_max; + /* Record the previous baseline and peak number of priority commands executed in one priority cycle. */ + long long priority_el_cmd_cnt_prev; + long long priority_el_cmd_cnt_max; /* The sum of active-expire, active-defrag and all other tasks done by cron and beforeSleep, but excluding read, write and AOF, which are counted by other sets of metrics. */ monotime el_cron_duration; diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index fe219d6c4e9..e824962a8fa 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -882,7 +882,7 @@ TEST(KqueueFairnessTest, KernelBehavior) { static void qosMetricTestCb(struct aeEventLoop *el, uint64_t duration_us) { (void)el; - durationAddSample(EL_DURATION_TYPE_QOS_EL, duration_us); + durationAddSample(EL_DURATION_TYPE_PRIORITY_EL, duration_us); } static void qosTestFileProc(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -901,8 +901,8 @@ TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { ASSERT_NE(main_loop, (aeEventLoop *)NULL); ASSERT_EQ(aeActuateQoSEventLoopIfSupported(main_loop, 2000, qosMetricTestCb), AE_OK); - unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt; - unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum; + unsigned long long orig_cnt = server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt; + unsigned long long orig_sum = server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum; int fds[2]; ASSERT_EQ(pipe(fds), 0); @@ -912,8 +912,8 @@ TEST_F(SocketPrioritizationTest, QoSEventLoopStatsMetrics) { ASSERT_EQ(write(fds[1], &b, 1), 1); aeProcessEvents(main_loop, AE_DONT_WAIT | AE_ALL_EVENTS); - EXPECT_GT(server.duration_stats[EL_DURATION_TYPE_QOS_EL].cnt, orig_cnt); - EXPECT_GE(server.duration_stats[EL_DURATION_TYPE_QOS_EL].sum, orig_sum); + EXPECT_GT(server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].cnt, orig_cnt); + EXPECT_GE(server.duration_stats[EL_DURATION_TYPE_PRIORITY_EL].sum, orig_sum); close(fds[0]); close(fds[1]); diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index caf96423797..0343a97ab7e 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -356,12 +356,12 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { # make sure debug info is hidden set info [r info] assert_equal [getInfoProperty $info eventloop_duration_aof_sum] {} - assert_equal [getInfoProperty $info qos_eventloop_duration_max] {} - assert_equal [getInfoProperty $info qos_eventloop_cmd_per_cycle_max] {} + assert_equal [getInfoProperty $info eventloop_priority_duration_max] {} + assert_equal [getInfoProperty $info eventloop_priority_cmd_per_cycle_max] {} set info_all [r info all] assert_equal [getInfoProperty $info_all eventloop_duration_aof_sum] {} - assert_equal [getInfoProperty $info_all qos_eventloop_duration_max] {} - assert_equal [getInfoProperty $info_all qos_eventloop_cmd_per_cycle_max] {} + assert_equal [getInfoProperty $info_all eventloop_priority_duration_max] {} + assert_equal [getInfoProperty $info_all eventloop_priority_cmd_per_cycle_max] {} set info1 [r info debug] @@ -373,10 +373,10 @@ start_server {tags {"info" "external:skip" "debug_defrag:skip"}} { assert {$cycle_max1 > 0} set duration_max1 [getInfoProperty $info1 eventloop_duration_max] assert {$duration_max1 > 0} - set qos_duration_max1 [getInfoProperty $info1 qos_eventloop_duration_max] - assert {$qos_duration_max1 >= 0} - set qos_cycle_max1 [getInfoProperty $info1 qos_eventloop_cmd_per_cycle_max] - assert {$qos_cycle_max1 >= 0} + set priority_duration_max1 [getInfoProperty $info1 eventloop_priority_duration_max] + assert {$priority_duration_max1 >= 0} + set priority_cycle_max1 [getInfoProperty $info1 eventloop_priority_cmd_per_cycle_max] + assert {$priority_cycle_max1 >= 0} after 110 ;# hz is 10, wait for a cron tick. set info2 [r info debug] diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl index 690fc553204..a1a51b44499 100644 --- a/tests/unit/socket-prioritization.tcl +++ b/tests/unit/socket-prioritization.tcl @@ -166,19 +166,19 @@ start_server {tags {"socket-prioritization external:skip"}} { set info_stats [$primary info stats] set info_debug [$primary info debug] - assert_morethan [getInfoProperty $info_stats qos_eventloop_cycles] 0 - assert_morethan [getInfoProperty $info_stats qos_eventloop_duration_sum] 0 - assert_morethan [getInfoProperty $info_stats qos_eventloop_duration_cmd_sum] 0 - assert {[getInfoProperty $info_debug qos_eventloop_duration_max] >= 0} - assert {[getInfoProperty $info_debug qos_eventloop_cmd_per_cycle_max] >= 0} + assert_morethan [getInfoProperty $info_stats eventloop_priority_cycles] 0 + assert_morethan [getInfoProperty $info_stats eventloop_priority_duration_sum] 0 + assert_morethan [getInfoProperty $info_stats eventloop_priority_duration_cmd_sum] 0 + assert {[getInfoProperty $info_debug eventloop_priority_duration_max] >= 0} + assert {[getInfoProperty $info_debug eventloop_priority_cmd_per_cycle_max] >= 0} # Reset stats and verify $primary config resetstat set info_stats_reset [$primary info stats] - assert_equal [getInfoProperty $info_stats_reset qos_eventloop_cycles] 0 - assert_equal [getInfoProperty $info_stats_reset qos_eventloop_duration_sum] 0 - assert_equal [getInfoProperty $info_stats_reset qos_eventloop_duration_cmd_sum] 0 - assert_equal [getInfoProperty [$primary info debug] qos_eventloop_cmd_per_cycle_max] 0 + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_cycles] 0 + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_duration_sum] 0 + assert_equal [getInfoProperty $info_stats_reset eventloop_priority_duration_cmd_sum] 0 + assert_equal [getInfoProperty [$primary info debug] eventloop_priority_cmd_per_cycle_max] 0 } } } From b9ce363a47ce4192d8eb0e8112c10f5edfbc9fb6 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Thu, 10 Sep 2026 20:50:41 +0000 Subject: [PATCH 13/14] Resolve conflicts with cluster bus IO offload Signed-off-by: Satheesha Gowda --- src/io_threads.c | 66 ++++++++++++++++++---------- src/unit/test_cluster_io_offload.cpp | 5 +++ 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/io_threads.c b/src/io_threads.c index d4fde906129..f1fc518dc81 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -566,38 +566,44 @@ void initIOThreads(int prev_threads_num) { } void testOnlyInitIOThreadQueues(void) { - if (io_shared_inbox.buffer) spmcFree(&io_shared_inbox); - if (io_shared_outbox.buffer) mpscFree(&io_shared_outbox); - if (pending_io_responses) { - listRelease(pending_io_responses); - pending_io_responses = NULL; - } - spmcInit(&io_shared_inbox, IO_SPMC_QUEUE_SIZE); - mpscInit(&io_shared_outbox, IO_MPSC_QUEUE_SIZE); + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + if (io_shared_inbox[p].buffer) spmcFree(&io_shared_inbox[p]); + if (io_shared_outbox[p].buffer) mpscFree(&io_shared_outbox[p]); + if (pending_io_responses[p]) { + listRelease(pending_io_responses[p]); + pending_io_responses[p] = NULL; + } + spmcInit(&io_shared_inbox[p], IO_SPMC_QUEUE_SIZE); + mpscInit(&io_shared_outbox[p], IO_MPSC_QUEUE_SIZE); + io_thread_ticket[p] = (mpscTicket){0}; + } io_jobs_submitted = 0; atomic_store_explicit(&io_jobs_finished, 0, memory_order_relaxed); cluster_io_pending_responses = 0; - io_thread_ticket = (mpscTicket){0}; } void testOnlyFreeIOThreadQueues(void) { - if (pending_io_responses) { - listRelease(pending_io_responses); - pending_io_responses = NULL; + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + if (pending_io_responses[p]) { + listRelease(pending_io_responses[p]); + pending_io_responses[p] = NULL; + } + spmcFree(&io_shared_inbox[p]); + mpscFree(&io_shared_outbox[p]); + io_thread_ticket[p] = (mpscTicket){0}; } - spmcFree(&io_shared_inbox); - mpscFree(&io_shared_outbox); io_jobs_submitted = 0; atomic_store_explicit(&io_jobs_finished, 0, memory_order_relaxed); cluster_io_pending_responses = 0; - io_thread_ticket = (mpscTicket){0}; } /* Fill the shared inbox so the next dispatch has to take its enqueue-failure * path. The queue is file-static, so tests cannot do this themselves. */ void testOnlyFillIOThreadInbox(void) { - while (spmcEnqueue(&io_shared_inbox, (void *)-1)) { - /* Keep going until the queue rejects the push. */ + for (int p = 0; p < JOB_PRIORITY_COUNT; p++) { + while (spmcEnqueue(&io_shared_inbox[p], (void *)-1)) { + /* Keep going until the queue rejects the push. */ + } } } @@ -769,7 +775,7 @@ int trySendClusterReadToIOThreads(struct clusterLink *link) { link->rcvbuf_alloc_at_dispatch = link->rcvbuf_alloc; /* Enqueue the read job. */ - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(link, JOB_REQ_CLUSTER_READ)) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(link, JOB_REQ_CLUSTER_READ)) == false)) { /* Rollback on enqueue failure. */ link->io_read_state = CLUSTER_LINK_IO_IDLE; link->io_refs--; @@ -846,7 +852,7 @@ int trySendClusterWriteToIOThreads(struct clusterLink *link) { link->io_refs++; /* Enqueue the write job. */ - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(link, JOB_REQ_CLUSTER_WRITE)) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(link, JOB_REQ_CLUSTER_WRITE)) == false)) { link->io_write_state = CLUSTER_LINK_IO_IDLE; link->io_refs--; link->io_last_send_block = NULL; @@ -878,7 +884,7 @@ int trySendClusterAcceptToIOThreads(connection *conn) { connSetPostponeUpdateState(conn, 1); connIncrRefs(conn); - if (unlikely(spmcEnqueue(&io_shared_inbox, tagJob(conn, JOB_REQ_CLUSTER_ACCEPT)) == false)) { + if (unlikely(spmcEnqueue(&io_shared_inbox[JOB_PRIORITY_HIGH], tagJob(conn, JOB_REQ_CLUSTER_ACCEPT)) == false)) { connDecrRefs(conn); connSetPostponeUpdateState(conn, 0); conn->flags &= ~CONN_FLAG_ACCEPT_OFFLOAD_PENDING; @@ -1052,6 +1058,8 @@ void sendToMainThread(void *data, int type) { if (type == JOB_RES_READ_CLIENT || type == JOB_RES_WRITE_CLIENT) { client *c = (client *)data; qidx = getJobPriority(c); + } else if (type == JOB_RES_CLUSTER_READ || type == JOB_RES_CLUSTER_WRITE || type == JOB_RES_CLUSTER_ACCEPT) { + qidx = JOB_PRIORITY_HIGH; } if (unlikely(pending_io_responses[qidx])) { flushPendingIOResponsesList(&pending_io_responses[qidx], &io_shared_outbox[qidx], &io_thread_ticket[qidx], 0); @@ -1184,13 +1192,12 @@ static int processOutboxBatch(mpscQueue *outbox) { /* Try to dequeue JOB_BATCH_SIZE */ while (received_responses < JOB_BATCH_SIZE) { - dequeued_count = mpscDequeueBatch(&io_shared_outbox, jobs, JOB_BATCH_SIZE - received_responses); + int dequeued_count = mpscDequeueBatch(outbox, jobs, JOB_BATCH_SIZE - received_responses); /* Stop if we can't get more jobs from the queue. */ if (dequeued_count == 0) break; received_responses += dequeued_count; - total_processed += dequeued_count; for (int i = 0; i < dequeued_count; i++) { void *data; @@ -1204,6 +1211,21 @@ static int processOutboxBatch(mpscQueue *outbox) { client *c = (client *)data; serverAssert(c->io_write_state == CLIENT_COMPLETED_IO); write_jobs[write_count++] = c; + } else if (job_type == JOB_RES_CLUSTER_READ) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_reads_processed++; + clusterHandleReadCompletion((struct clusterLink *)data); + } else if (job_type == JOB_RES_CLUSTER_WRITE) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_writes_processed++; + clusterHandleWriteCompletion((struct clusterLink *)data); + } else if (job_type == JOB_RES_CLUSTER_ACCEPT) { + serverAssert(cluster_io_pending_responses > 0); + cluster_io_pending_responses--; + server.stat_cluster_threaded_accepts_processed++; + clusterHandleAcceptCompletion((connection *)data); } else { serverPanic("Unknown job type %d", job_type); } diff --git a/src/unit/test_cluster_io_offload.cpp b/src/unit/test_cluster_io_offload.cpp index 802498821a2..40c129ae9a9 100644 --- a/src/unit/test_cluster_io_offload.cpp +++ b/src/unit/test_cluster_io_offload.cpp @@ -47,6 +47,7 @@ class ClusterIOOffloadTest : public ::testing::Test { owned_conns_count = 0; owned_links_count = 0; memset(&server, 0, sizeof(server)); + server.el = aeCreateEventLoop(1024); server.io_threads_num = 2; server.active_io_threads_num = 2; testOnlyInitIOThreadQueues(); @@ -74,6 +75,10 @@ class ClusterIOOffloadTest : public ::testing::Test { * forever and stall processIOThreadsResponses(). */ EXPECT_EQ(testOnlyGetClusterIOPendingResponses(), 0u) << "leaked a cluster I/O pending response"; testOnlyFreeIOThreadQueues(); + if (server.el) { + aeDeleteEventLoop(server.el); + server.el = NULL; + } } /* A connection in the state cluster code expects post-accept: established, From 089e2c0eb43403488bb5c75c50deecf499355708 Mon Sep 17 00:00:00 2001 From: Satheesha Gowda Date: Thu, 10 Sep 2026 22:05:00 +0000 Subject: [PATCH 14/14] Refactored config and variables have prefix 'priority_' instead of 'qos_' Signed-off-by: Satheesha Gowda --- src/ae.c | 122 ++++++++++++------------ src/ae.h | 28 +++--- src/config.c | 8 +- src/server.c | 2 +- src/server.h | 2 +- src/unit/test_socket_prioritization.cpp | 26 ++--- tests/unit/socket-prioritization.tcl | 20 ++-- valkey.conf | 4 +- 8 files changed, 106 insertions(+), 106 deletions(-) diff --git a/src/ae.c b/src/ae.c index 5a4498c6f26..edf6cad167c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -98,12 +98,12 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->aftersleep = NULL; eventLoop->custompoll = NULL; eventLoop->flags = 0; - eventLoop->qos_apidata = NULL; - eventLoop->qos_fd = -1; - eventLoop->qos_fired = NULL; - eventLoop->qos_el_last_poll = 0; - eventLoop->qos_el_preempt_check_interval_us = 0; - eventLoop->qos_el_stats_callback = NULL; + eventLoop->priority_apidata = NULL; + eventLoop->priority_fd = -1; + eventLoop->priority_fired = NULL; + eventLoop->priority_events_last_poll = 0; + eventLoop->priority_events_preempt_check_interval_us = 0; + eventLoop->priority_events_stats_callback = NULL; /* Initialize the eventloop mutex with PTHREAD_MUTEX_ERRORCHECK type */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); @@ -158,9 +158,9 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) goto err; if (aeApiResize(eventLoop->apidata, setsize) == -1) goto err; - if (eventLoop->qos_apidata) { - if (aeApiResize(eventLoop->qos_apidata, setsize) == -1) goto err; - eventLoop->qos_fired = zrealloc(eventLoop->qos_fired, sizeof(aeFiredEvent) * setsize); + if (eventLoop->priority_apidata) { + if (aeApiResize(eventLoop->priority_apidata, setsize) == -1) goto err; + eventLoop->priority_fired = zrealloc(eventLoop->priority_fired, sizeof(aeFiredEvent) * setsize); } eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * setsize); @@ -180,9 +180,9 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { } void aeDeleteEventLoop(aeEventLoop *eventLoop) { - if (eventLoop->qos_apidata) { - aeApiFree(eventLoop->qos_apidata); - zfree(eventLoop->qos_fired); + if (eventLoop->priority_apidata) { + aeApiFree(eventLoop->priority_apidata); + zfree(eventLoop->priority_fired); } aeApiFree(eventLoop->apidata); zfree(eventLoop->events); @@ -213,13 +213,13 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } aeFileEvent *fe = &eventLoop->events[fd]; - bool old_is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); - bool new_is_priority = (mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); + bool old_is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); + bool new_is_priority = (mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); /* Handle dynamic cross-multiplexer migration if the priority of an active socket is changed */ - if (eventLoop->qos_apidata != NULL && old_is_priority != new_is_priority && fe->mask != AE_NONE) { - aeApiState *old_api = old_is_priority ? eventLoop->qos_apidata : eventLoop->apidata; - aeApiState *new_api = new_is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + if (eventLoop->priority_apidata != NULL && old_is_priority != new_is_priority && fe->mask != AE_NONE) { + aeApiState *old_api = old_is_priority ? eventLoop->priority_apidata : eventLoop->apidata; + aeApiState *new_api = new_is_priority ? eventLoop->priority_apidata : eventLoop->apidata; /* 1. Remove active event mask from the old multiplexer */ aeApiDelEvent(old_api, fd, BACKEND_MASK(fe->mask), BACKEND_MASK(fe->mask)); @@ -233,7 +233,7 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc } else { bool is_priority = new_is_priority || old_is_priority; if (!is_priority) mask &= ~AE_HIGH_PRIORITY; - aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + aeApiState *target_api = is_priority ? eventLoop->priority_apidata : eventLoop->apidata; int backend_add_mask = BACKEND_MASK(mask) & ~fe->mask; // just the meaningful additions if (backend_add_mask) { @@ -261,8 +261,8 @@ void aeDeleteFileEvent(aeEventLoop *eventLoop, int fd, int mask) { aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) goto done; - bool is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->qos_apidata != NULL); - aeApiState *target_api = is_priority ? eventLoop->qos_apidata : eventLoop->apidata; + bool is_priority = (fe->mask & AE_HIGH_PRIORITY) && (eventLoop->priority_apidata != NULL); + aeApiState *target_api = is_priority ? eventLoop->priority_apidata : eventLoop->apidata; /* We want to always remove AE_BARRIER if set when AE_WRITABLE * is removed. */ @@ -501,44 +501,44 @@ static inline void aeFireFileEvent(aeEventLoop *eventLoop, int fd, int mask) { } } -/* Process all QoS events and invoke the stats callback +/* Process all high-priority events and invoke the stats callback * with the elapsed time in microseconds if registered. */ static int aeProcessQoSEvents(aeEventLoop *eventLoop) { int processed = 0; - if (eventLoop->qos_apidata != NULL) { + if (eventLoop->priority_apidata != NULL) { monotime start = getMonotonicUs(); struct timeval tv = {0, 0}; - int numevents = aeApiPoll(eventLoop->qos_apidata, eventLoop->qos_fired, eventLoop->events, + int numevents = aeApiPoll(eventLoop->priority_apidata, eventLoop->priority_fired, eventLoop->events, eventLoop->setsize, eventLoop->maxfd, &tv); for (int j = 0; j < numevents; j++) { - int fd = eventLoop->qos_fired[j].fd; - int mask = eventLoop->qos_fired[j].mask; + int fd = eventLoop->priority_fired[j].fd; + int mask = eventLoop->priority_fired[j].mask; aeFireFileEvent(eventLoop, fd, mask); processed++; } - eventLoop->qos_el_last_poll = getMonotonicUs(); - /* Update INFO stats if registered and QoS events were processed */ - if (eventLoop->qos_el_stats_callback != NULL && processed > 0) { - eventLoop->qos_el_stats_callback(eventLoop, eventLoop->qos_el_last_poll - start); + eventLoop->priority_events_last_poll = getMonotonicUs(); + /* Update INFO stats if registered and high-priority events were processed */ + if (eventLoop->priority_events_stats_callback != NULL && processed > 0) { + eventLoop->priority_events_stats_callback(eventLoop, eventLoop->priority_events_last_poll - start); } } return processed; } -/* Set the preemptive poll interval in microseconds for QoS events. - * QoS events are checked periodically during normal event processing +/* Set the preemptive poll interval in microseconds for high-priority events. + * High-priority events are checked periodically during normal event processing * when processing batches of normal events exceeds this interval. * Setting this interval to 0 disables preemptive polling. */ void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us) { - eventLoop->qos_el_preempt_check_interval_us = interval_us; + eventLoop->priority_events_preempt_check_interval_us = interval_us; } -/* Preemptively processes QoS events if the elapsed time since the last poll - * exceeds the qos_el_preempt_check_interval_us threshold (0 disables preemption). */ +/* Preemptively processes high-priority events if the elapsed time since the last poll + * exceeds the priority_events_preempt_check_interval_us threshold (0 disables preemption). */ int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop) { - /* Skip if QoS is disabled or preemptive polling is disabled */ - if (eventLoop->qos_apidata == NULL || eventLoop->qos_el_preempt_check_interval_us == 0) return 0; - if (elapsedUs(eventLoop->qos_el_last_poll) < eventLoop->qos_el_preempt_check_interval_us) return 0; + /* Skip if high-priority processing is disabled or preemptive polling is disabled */ + if (eventLoop->priority_apidata == NULL || eventLoop->priority_events_preempt_check_interval_us == 0) return 0; + if (elapsedUs(eventLoop->priority_events_last_poll) < eventLoop->priority_events_preempt_check_interval_us) return 0; return aeProcessQoSEvents(eventLoop); } @@ -606,15 +606,15 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) { /* After sleep callback. */ if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) eventLoop->aftersleep(eventLoop, numevents); - /* Prioritize QoS events: Always drain priority events + /* Prioritize high-priority events: Always drain priority events * immediately before processing normal events. */ processed += aeProcessQoSEvents(eventLoop); /* Process normal file events */ for (j = 0; j < numevents; j++) { int fd = eventLoop->fired[j].fd; - /* Skip processing QoS FD events again as they were already processed above */ - if (fd == eventLoop->qos_fd) continue; + /* Skip processing high-priority FD events again as they were already processed above */ + if (fd == eventLoop->priority_fd) continue; int mask = eventLoop->fired[j].mask; aeFireFileEvent(eventLoop, fd, mask); @@ -703,37 +703,37 @@ void aeSetPollProtect(aeEventLoop *eventLoop, int protect) { int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback) { assert(eventLoop != NULL); - /* Create QoS polling state for internal connections (cluster bus, replication, and slot migration jobs) */ - aeApiState *qos_apidata = aeApiCreate(eventLoop->setsize); - if (qos_apidata == NULL) return AE_ERR; + /* Create high-priority polling state for internal connections (cluster bus, replication, and slot migration jobs) */ + aeApiState *priority_apidata = aeApiCreate(eventLoop->setsize); + if (priority_apidata == NULL) return AE_ERR; - int qos_fd = aeApiGetPollFd(qos_apidata); - if (qos_fd == -1) { - aeApiFree(qos_apidata); + int priority_fd = aeApiGetPollFd(priority_apidata); + if (priority_fd == -1) { + aeApiFree(priority_apidata); return AE_ERR; } - aeFiredEvent *qos_fired = zmalloc(sizeof(aeFiredEvent) * eventLoop->setsize); - if (qos_fired == NULL) { - aeApiFree(qos_apidata); + aeFiredEvent *priority_fired = zmalloc(sizeof(aeFiredEvent) * eventLoop->setsize); + if (priority_fired == NULL) { + aeApiFree(priority_apidata); return AE_ERR; } - /* Register qos_fd with NULL callback: qos_fd is only used to wake up the main loop's - * multiplexer when QoS traffic arrives. aeProcessEvents() checks for qos_fd and drains - * qos_apidata directly before dispatching normal events, and explicitly skips callback - * dispatch for qos_fd. */ - if (aeCreateFileEvent(eventLoop, qos_fd, AE_READABLE, NULL, NULL) == AE_ERR) { - zfree(qos_fired); - aeApiFree(qos_apidata); + /* Register priority_fd with NULL callback: priority_fd is only used to wake up the main loop's + * multiplexer when high-priority traffic arrives. aeProcessEvents() checks for priority_fd and drains + * priority_apidata directly before dispatching normal events, and explicitly skips callback + * dispatch for priority_fd. */ + if (aeCreateFileEvent(eventLoop, priority_fd, AE_READABLE, NULL, NULL) == AE_ERR) { + zfree(priority_fired); + aeApiFree(priority_apidata); return AE_ERR; } - eventLoop->qos_apidata = qos_apidata; - eventLoop->qos_fd = qos_fd; - eventLoop->qos_fired = qos_fired; - eventLoop->qos_el_stats_callback = qosStatsCallback; - eventLoop->qos_el_preempt_check_interval_us = qosPreemptPollIntervalUs; + eventLoop->priority_apidata = priority_apidata; + eventLoop->priority_fd = priority_fd; + eventLoop->priority_fired = priority_fired; + eventLoop->priority_events_stats_callback = qosStatsCallback; + eventLoop->priority_events_preempt_check_interval_us = qosPreemptPollIntervalUs; return AE_OK; } diff --git a/src/ae.h b/src/ae.h index a2a6fd4a746..bba180fd52b 100644 --- a/src/ae.h +++ b/src/ae.h @@ -48,9 +48,9 @@ things to disk before sending replies, and want \ to do that in a group fashion. */ #define AE_HIGH_PRIORITY 8 /* Virtual routing mask flag: when set in aeCreateFileEvent(), \ - * the event is registered on qos_apidata if available. \ + * the event is registered on priority_apidata if available. \ * Stripped before passing to the underlying OS multiplexer. */ -#define AE_QOS_PREEMPT_CHECK_MASK 0x03 /* Mask to check QoS preemption once every 4 iterations */ +#define AE_QOS_PREEMPT_CHECK_MASK 0x03 /* Mask to check high-priority preemption once every 4 iterations */ #define AE_FILE_EVENTS (1 << 0) #define AE_TIME_EVENTS (1 << 1) @@ -84,7 +84,7 @@ typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientDat typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop); typedef void aeAfterSleepProc(struct aeEventLoop *eventLoop, int numevents); typedef int aeCustomPollProc(struct aeEventLoop *eventLoop); -/* Callback invoked with elapsed microseconds after QoS events are processed. */ +/* Callback invoked with elapsed microseconds after high-priority events are processed. */ typedef void aeQoSStatsProc(struct aeEventLoop *eventLoop, uint64_t duration_us); /* File event structure */ @@ -130,16 +130,16 @@ typedef struct aeEventLoop { pthread_mutex_t poll_mutex; int flags; - /* Quality of Service (QoS): - * Sockets registered with AE_HIGH_PRIORITY are tracked in qos_apidata. - * qos_fd is registered into apidata to wake the main loop when QoS traffic arrives. - * qos_fired holds fired events when draining QoS channels. */ - aeApiState *qos_apidata; /* Dedicated QoS polling state */ - int qos_fd; /* File descriptor of QoS polling backend (-1 if disabled) */ - aeFiredEvent *qos_fired; /* Fired events buffer for QoS polling */ - monotime qos_el_last_poll; /* Timestamp when QoS was last drained */ - uint64_t qos_el_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ - aeQoSStatsProc *qos_el_stats_callback; /* Callback invoked with elapsed microseconds after draining QoS */ + /* High-priority event processing: + * Sockets registered with AE_HIGH_PRIORITY are tracked in priority_apidata. + * priority_fd is registered into apidata to wake the main loop when high-priority traffic arrives. + * priority_fired holds fired events when draining high-priority channels. */ + aeApiState *priority_apidata; /* Dedicated high-priority polling state */ + int priority_fd; /* File descriptor of high-priority polling backend (-1 if disabled) */ + aeFiredEvent *priority_fired; /* Fired events buffer for high-priority polling */ + monotime priority_events_last_poll; /* Timestamp when high-priority events were last drained */ + uint64_t priority_events_preempt_check_interval_us; /* Preemptive check interval in microseconds (0 = disabled) */ + aeQoSStatsProc *priority_events_stats_callback; /* Callback invoked with elapsed microseconds after draining high-priority events */ } aeEventLoop; /* Prototypes */ @@ -169,7 +169,7 @@ int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); void aeSetDontWait(aeEventLoop *eventLoop, int noWait); -/* QoS event loop prototypes */ +/* High-priority event loop prototypes */ int aeActuateQoSEventLoopIfSupported(aeEventLoop *eventLoop, uint64_t qosPreemptPollIntervalUs, aeQoSStatsProc *qosStatsCallback); int aeProcessQoSEventsPreemptively(aeEventLoop *eventLoop); void aeSetQoSPreemptCheckInterval(aeEventLoop *eventLoop, uint64_t interval_us); diff --git a/src/config.c b/src/config.c index 12555a68852..1d2876b312f 100644 --- a/src/config.c +++ b/src/config.c @@ -2677,12 +2677,12 @@ static int updateDefragConfiguration(const char **err) { return 1; } -/* Dynamic configuration apply callback for qos-preemptive-poll-interval-us. +/* Dynamic configuration apply callback for priority-preemptive-poll-interval-us. * Updates the preemption threshold on the active event loop. */ -static int updateQoSPreemptivePollInterval(const char **err) { +static int updatePriorityPreemptivePollInterval(const char **err) { UNUSED(err); if (server.el) { - aeSetQoSPreemptCheckInterval(server.el, server.qos_preemptive_poll_interval_us); + aeSetQoSPreemptCheckInterval(server.el, server.priority_preemptive_poll_interval_us); } return 1; } @@ -3518,7 +3518,7 @@ standardConfig static_configs[] = { createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, updateDefragConfiguration), /* Default: maximum defrag force at 100% fragmentation */ createIntConfig("active-defrag-cycle-us", NULL, MODIFIABLE_CONFIG, 0, 100000, server.active_defrag_cycle_us, 500, INTEGER_CONFIG, NULL, updateDefragConfiguration), - createIntConfig("qos-preemptive-poll-interval-us", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.qos_preemptive_poll_interval_us, 2000, INTEGER_CONFIG, NULL, updateQoSPreemptivePollInterval), + createIntConfig("priority-preemptive-poll-interval-us", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.priority_preemptive_poll_interval_us, 2000, INTEGER_CONFIG, NULL, updatePriorityPreemptivePollInterval), createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_log_factor, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, lfu_config_decay_time, 1, INTEGER_CONFIG, NULL, NULL), createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.replica_priority, 100, INTEGER_CONFIG, NULL, NULL), diff --git a/src/server.c b/src/server.c index 8a7b40de2b9..ea477e0d5ba 100644 --- a/src/server.c +++ b/src/server.c @@ -3118,7 +3118,7 @@ void initServer(void) { } /* Setup QoS event loop if multiplexer backend supports secondary polling. * If secondary polling is unsupported (e.g. evport, select), gracefully fallback to standard event processing without QoS. */ - if (aeActuateQoSEventLoopIfSupported(server.el, server.qos_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { + if (aeActuateQoSEventLoopIfSupported(server.el, server.priority_preemptive_poll_interval_us, qosStatsCallback) == AE_ERR) { serverLog(LL_NOTICE, "QoS event prioritization not supported on %s multiplexer, falling back to standard event processing", aeGetApiName()); } diff --git a/src/server.h b/src/server.h index ef4d5209307..a5a4e329968 100644 --- a/src/server.h +++ b/src/server.h @@ -2060,7 +2060,7 @@ struct valkeyServer { invocation of the event loop. */ unsigned int max_new_conns_per_cycle; /* The maximum number of tcp connections that will be accepted during each invocation of the event loop. */ - int qos_preemptive_poll_interval_us; /* QoS event loop preemptive poll interval in microseconds */ + int priority_preemptive_poll_interval_us; /* Priority event loop preemptive poll interval in microseconds */ /* AOF persistence */ int aof_enabled; /* AOF configuration */ int aof_state; /* AOF_(ON|OFF|WAIT_REWRITE) */ diff --git a/src/unit/test_socket_prioritization.cpp b/src/unit/test_socket_prioritization.cpp index e824962a8fa..cdd8b600658 100644 --- a/src/unit/test_socket_prioritization.cpp +++ b/src/unit/test_socket_prioritization.cpp @@ -156,30 +156,30 @@ class SocketPrioritizationConnTest : public SocketPrioritizationTest, public ::t TEST_F(SocketPrioritizationTest, EventLoopDualInitialization) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->qos_apidata, (aeApiState *)NULL); - EXPECT_NE(server.el->qos_fd, -1); - EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 2000ULL); + ASSERT_NE(server.el->priority_apidata, (aeApiState *)NULL); + EXPECT_NE(server.el->priority_fd, -1); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 2000ULL); aeSetQoSPreemptCheckInterval(server.el, 5000ULL); - EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 5000ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 5000ULL); aeSetQoSPreemptCheckInterval(server.el, 2000ULL); /* Newly created standalone loop must have preemption disabled (0) by default */ aeEventLoop *standalone = aeCreateEventLoop(64); ASSERT_NE(standalone, (aeEventLoop *)NULL); - EXPECT_EQ(standalone->qos_apidata, (aeApiState *)NULL); - EXPECT_EQ(standalone->qos_fd, -1); - EXPECT_EQ(standalone->qos_el_preempt_check_interval_us, 0ULL); + EXPECT_EQ(standalone->priority_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone->priority_fd, -1); + EXPECT_EQ(standalone->priority_events_preempt_check_interval_us, 0ULL); aeDeleteEventLoop(standalone); } TEST_F(SocketPrioritizationTest, DynamicPreemptionIntervalThreshold) { /* Test getter and setter */ aeSetQoSPreemptCheckInterval(server.el, 500ULL); - EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 500ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 500ULL); /* Test disabling preemption (interval = 0) */ aeSetQoSPreemptCheckInterval(server.el, 0ULL); - EXPECT_EQ(server.el->qos_el_preempt_check_interval_us, 0ULL); + EXPECT_EQ(server.el->priority_events_preempt_check_interval_us, 0ULL); aeSetQoSPreemptCheckInterval(server.el, 2000ULL); } @@ -279,7 +279,7 @@ TEST_F(SocketPrioritizationTest, PreemptionOfNormalEventsByQoSLoop) { if (write(qos_pipe[1], &c, 1) < 0) { } - /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for qos_el_last_poll */ + /* Ensure more than AE_QOS_DEFAULT_PREEMPT_CHECK_INTERVAL_US (2000 us) has elapsed for priority_events_last_poll */ advanceMockTime(1000000); g_execution_count = 0; @@ -548,7 +548,7 @@ TEST_F(SocketPrioritizationTest, LevelTriggeredBatchProcessingAndStarvationPreve TEST_F(SocketPrioritizationTest, DualEventLoopResize) { ASSERT_NE(server.el, (aeEventLoop *)NULL); - ASSERT_NE(server.el->qos_apidata, (aeApiState *)NULL); + ASSERT_NE(server.el->priority_apidata, (aeApiState *)NULL); EXPECT_EQ(aeGetSetSize(server.el), 1024); int ret = aeResizeSetSize(server.el, 2048); @@ -559,8 +559,8 @@ TEST_F(SocketPrioritizationTest, DualEventLoopResize) { TEST_F(SocketPrioritizationTest, FallbackMaskSanitization) { aeEventLoop *standalone_el = aeCreateEventLoop(64); ASSERT_NE(standalone_el, (aeEventLoop *)NULL); - EXPECT_EQ(standalone_el->qos_apidata, (aeApiState *)NULL); - EXPECT_EQ(standalone_el->qos_fd, -1); + EXPECT_EQ(standalone_el->priority_apidata, (aeApiState *)NULL); + EXPECT_EQ(standalone_el->priority_fd, -1); int fds[2]; ASSERT_EQ(pipe(fds), 0); diff --git a/tests/unit/socket-prioritization.tcl b/tests/unit/socket-prioritization.tcl index a1a51b44499..700156d3090 100644 --- a/tests/unit/socket-prioritization.tcl +++ b/tests/unit/socket-prioritization.tcl @@ -150,16 +150,16 @@ start_server {tags {"socket-prioritization external:skip"}} { } } - test {Dynamic configuration of qos-preemptive-poll-interval-us} { - assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 2000 - r config set qos-preemptive-poll-interval-us 500 - assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 500 - r config set qos-preemptive-poll-interval-us 0 - assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 0 - assert_error "*argument must be between*" {r config set qos-preemptive-poll-interval-us -1} - assert_error "*argument couldn't be parsed into an integer*" {r config set qos-preemptive-poll-interval-us invalid} - r config set qos-preemptive-poll-interval-us 2000 - assert_equal [lindex [r config get qos-preemptive-poll-interval-us] 1] 2000 + test {Dynamic configuration of priority-preemptive-poll-interval-us} { + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 2000 + r config set priority-preemptive-poll-interval-us 500 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 500 + r config set priority-preemptive-poll-interval-us 0 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 0 + assert_error "*argument must be between*" {r config set priority-preemptive-poll-interval-us -1} + assert_error "*argument couldn't be parsed into an integer*" {r config set priority-preemptive-poll-interval-us invalid} + r config set priority-preemptive-poll-interval-us 2000 + assert_equal [lindex [r config get priority-preemptive-poll-interval-us] 1] 2000 } test {Socket Prioritization: INFO stats and debug QoS metrics} { diff --git a/valkey.conf b/valkey.conf index 0713d3a86a2..419dfb40bfe 100644 --- a/valkey.conf +++ b/valkey.conf @@ -2784,7 +2784,7 @@ jemalloc-bg-thread yes # internal events are not starved, Valkey periodically interrupts normal events # processing to check and drain pending high-priority events. # -# The `qos-preemptive-poll-interval-us` configuration defines the maximum elapsed +# The `priority-preemptive-poll-interval-us` configuration defines the maximum elapsed # time (in microseconds) between preemptive poll of the high-priority events # while iterating through normal client events. # @@ -2796,7 +2796,7 @@ jemalloc-bg-thread yes # events will only be processed at the start of each event loop cycle. # # Default: 2000 microseconds (2 milliseconds) -# qos-preemptive-poll-interval-us 2000 +# priority-preemptive-poll-interval-us 2000 ########################### HOT KEY DETECTION #################################