From beb4e984ad3e989a46507968666ce340ad56ae6e Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 8 Apr 2026 10:47:29 +0200 Subject: [PATCH 1/5] CUDA: use a ring-buffer for cuda graphs --- ggml/src/ggml-cuda/common.cuh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 66ed02d29238..66dad054ebb7 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -28,6 +28,7 @@ #include #include #include +#include #if defined(GGML_USE_HIP) #include "vendors/hip.h" @@ -151,6 +152,7 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses #define GGML_CUDA_MAX_STREAMS 8 +#define GGML_CUDA_MAX_GRAPHS 64 [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); @@ -1367,11 +1369,17 @@ struct ggml_backend_cuda_context { // Map from first_node_ptr to cuda_graph - allows multiple graphs per context // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map> cuda_graphs; + std::deque graph_roots; ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { auto it = cuda_graphs.find(first_node_ptr); if (it == cuda_graphs.end()) { + if (graph_roots.size() >= GGML_CUDA_MAX_GRAPHS) { + cuda_graphs.erase(graph_roots.front()); + graph_roots.pop_front(); + } cuda_graphs[first_node_ptr] = std::make_unique(); + graph_roots.push_back(first_node_ptr); return cuda_graphs[first_node_ptr].get(); } return it->second.get(); From 193b81ea41dfb79352dc64c21a67e4f52583f98c Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Thu, 16 Apr 2026 19:38:23 +0800 Subject: [PATCH 2/5] bump limit to 128 --- ggml/src/ggml-cuda/common.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 66dad054ebb7..015efe6ce6fd 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -152,7 +152,7 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses #define GGML_CUDA_MAX_STREAMS 8 -#define GGML_CUDA_MAX_GRAPHS 64 +#define GGML_CUDA_MAX_GRAPHS 128 [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); From bd942c4f242f4cabd0ed389b3751d4163f8b46ca Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Fri, 17 Apr 2026 01:11:18 +0800 Subject: [PATCH 3/5] use LRU eviction --- ggml/src/ggml-cuda/common.cuh | 42 ++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 015efe6ce6fd..46aab2ef92c0 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -28,7 +28,7 @@ #include #include #include -#include +#include #if defined(GGML_USE_HIP) #include "vendors/hip.h" @@ -152,7 +152,6 @@ static int ggml_cuda_highest_compiled_arch(const int arch) { #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses #define GGML_CUDA_MAX_STREAMS 8 -#define GGML_CUDA_MAX_GRAPHS 128 [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg); @@ -1369,18 +1368,41 @@ struct ggml_backend_cuda_context { // Map from first_node_ptr to cuda_graph - allows multiple graphs per context // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map> cuda_graphs; - std::deque graph_roots; + + // pair of timestamp and node ptr + std::priority_queue< + std::pair, + std::vector>, + std::greater<> + > graph_roots; + + std::unordered_map graph_last_used_time; ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { + int64_t time_now = ggml_time_us(); + + // delete all graph elements older than 10 seconds + while (!graph_roots.empty() && time_now - graph_roots.top().first >= 10'000'000) { + const auto & [ts, node_ptr] = graph_roots.top(); + graph_roots.pop(); + + // lazy delete + if (ts == graph_last_used_time.at(node_ptr)) { + cuda_graphs.erase(node_ptr); + graph_last_used_time.erase(node_ptr); + } + } + auto it = cuda_graphs.find(first_node_ptr); if (it == cuda_graphs.end()) { - if (graph_roots.size() >= GGML_CUDA_MAX_GRAPHS) { - cuda_graphs.erase(graph_roots.front()); - graph_roots.pop_front(); - } - cuda_graphs[first_node_ptr] = std::make_unique(); - graph_roots.push_back(first_node_ptr); - return cuda_graphs[first_node_ptr].get(); + it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; + } + + // throttle re-pushes into the lru queue by 1s per entry + auto last_it = graph_last_used_time.find(first_node_ptr); + if (last_it == graph_last_used_time.end() || time_now - last_it->second >= 1'000'000) { + graph_last_used_time[first_node_ptr] = time_now; + graph_roots.emplace(time_now, first_node_ptr); } return it->second.get(); } From 958fd9f82cb4bce55119818548307488f4aeb1a5 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Fri, 17 Apr 2026 01:24:58 +0800 Subject: [PATCH 4/5] better naming --- ggml/src/ggml-cuda/common.cuh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 46aab2ef92c0..8789dd224fd2 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1374,20 +1374,21 @@ struct ggml_backend_cuda_context { std::pair, std::vector>, std::greater<> - > graph_roots; + > graph_lru_cache; std::unordered_map graph_last_used_time; ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { int64_t time_now = ggml_time_us(); - // delete all graph elements older than 10 seconds - while (!graph_roots.empty() && time_now - graph_roots.top().first >= 10'000'000) { - const auto & [ts, node_ptr] = graph_roots.top(); - graph_roots.pop(); + // delete all cuda graphs older than 10 seconds + while (!graph_lru_cache.empty() && time_now - graph_lru_cache.top().first >= 10'000'000) { + const auto & [ts, node_ptr] = graph_lru_cache.top(); + graph_lru_cache.pop(); // lazy delete - if (ts == graph_last_used_time.at(node_ptr)) { + auto it = graph_last_used_time.find(node_ptr); + if (it != graph_last_used_time.end() && ts == it->second) { cuda_graphs.erase(node_ptr); graph_last_used_time.erase(node_ptr); } @@ -1402,7 +1403,7 @@ struct ggml_backend_cuda_context { auto last_it = graph_last_used_time.find(first_node_ptr); if (last_it == graph_last_used_time.end() || time_now - last_it->second >= 1'000'000) { graph_last_used_time[first_node_ptr] = time_now; - graph_roots.emplace(time_now, first_node_ptr); + graph_lru_cache.emplace(time_now, first_node_ptr); } return it->second.get(); } From 579972b7fbef828cc17f15ebf4d6f23fcec0241e Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Fri, 17 Apr 2026 17:05:00 +0800 Subject: [PATCH 5/5] do periodic clean-up --- ggml/src/ggml-cuda/common.cuh | 42 ++++++++++++----------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 8789dd224fd2..ddf50baf4956 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -28,7 +28,6 @@ #include #include #include -#include #if defined(GGML_USE_HIP) #include "vendors/hip.h" @@ -1188,6 +1187,7 @@ struct ggml_cuda_graph { bool disable_due_to_gpu_arch = false; bool warmup_complete = false; uint64_t uid = 0; + int64_t last_used_time = 0; struct node_properties { ggml_tensor node; void * node_src_data_ptrs[GGML_MAX_SRC]; @@ -1369,28 +1369,20 @@ struct ggml_backend_cuda_context { // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map> cuda_graphs; - // pair of timestamp and node ptr - std::priority_queue< - std::pair, - std::vector>, - std::greater<> - > graph_lru_cache; - - std::unordered_map graph_last_used_time; + int64_t last_graph_eviction_sweep = 0; ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { - int64_t time_now = ggml_time_us(); - - // delete all cuda graphs older than 10 seconds - while (!graph_lru_cache.empty() && time_now - graph_lru_cache.top().first >= 10'000'000) { - const auto & [ts, node_ptr] = graph_lru_cache.top(); - graph_lru_cache.pop(); - - // lazy delete - auto it = graph_last_used_time.find(node_ptr); - if (it != graph_last_used_time.end() && ts == it->second) { - cuda_graphs.erase(node_ptr); - graph_last_used_time.erase(node_ptr); + const int64_t time_now = ggml_time_us(); + + // sweep every 5s, evicting cuda graphs unused for >=10s + if (time_now - last_graph_eviction_sweep >= 5'000'000) { + last_graph_eviction_sweep = time_now; + for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { + if (time_now - it->second->last_used_time >= 10'000'000) { + it = cuda_graphs.erase(it); + } else { + ++it; + } } } @@ -1398,13 +1390,7 @@ struct ggml_backend_cuda_context { if (it == cuda_graphs.end()) { it = cuda_graphs.emplace(first_node_ptr, std::make_unique()).first; } - - // throttle re-pushes into the lru queue by 1s per entry - auto last_it = graph_last_used_time.find(first_node_ptr); - if (last_it == graph_last_used_time.end() || time_now - last_it->second >= 1'000'000) { - graph_last_used_time[first_node_ptr] = time_now; - graph_lru_cache.emplace(time_now, first_node_ptr); - } + it->second->last_used_time = time_now; return it->second.get(); }