Eval bug: Failed to prepare attention ubatches -> crash

[ZisIsNotZis]

Name and Version

version: 2696 (549b9d8)
built with GNU 13.3.0 for Linux x86_64

Operating systems

Linux

GGML backends

CUDA

Hardware

i7-6850K + 4090

Models

unsloth/Qwen3.6-35B-A3B-UD-IQ4_XS

Problem description & steps to reproduce

I'm running with llama-server -m ~/hf/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf -c 262144 -ctk q5_1 -ctv q5_1 -ctkd q5_1 -ctvd q5_1 --temp .6 --top-p .8 --top-k 20 --presence-penalty 0 --min-p 0 --spec-type draft-mtp --spec-draft-n-max 1 --chat-template-kwargs '{"enable_thinking":false}'. After long time running, it crashes with failed to prepare ubatch. This is probably hard to reproduce?

First Bad Commit

Maybe has something to do with kv-quantization or MTP. No crash happened previously when I was using neither of them

Relevant log output

Logs

1482.46.827.375 I slot      release: id  0 | task 232083 | stop processing: n_tokens = 30946, truncated = 0
1482.46.938.685 I slot print_timing: id  3 | task 232040 | prompt eval time =     897.72 ms /   467 tokens (    1.92 ms per token,   520.21 tokens per second)
1482.46.938.692 I slot print_timing: id  3 | task 232040 |        eval time =   24572.14 ms /   189 tokens (  130.01 ms per token,     7.69 tokens per second)
1482.46.938.693 I slot print_timing: id  3 | task 232040 |       total time =   25469.85 ms /   656 tokens
1482.46.938.694 I slot print_timing: id  3 | task 232040 |    graphs reused =     224999
1482.46.938.696 I slot print_timing: id  3 | task 232040 | draft acceptance = 0.09942 (   17 accepted /   171 generated)
1482.46.938.719 I statistics        draft-mtp: #calls(b,g,a) =  587 227521 229853, #gen drafts = 229853, #acc drafts = 15459, #gen tokens = 229853, #acc tokens = 15459, dur(b,g,a) = 0.541, 475880.177, 224.089 ms
1482.46.940.473 I slot      release: id  3 | task 232040 | stop processing: n_tokens = 28639, truncated = 0
1482.47.635.662 I srv  params_from_: Chat format: peg-native
1482.47.700.063 I slot get_availabl: id  0 | task -1 | selected slot by LCP similarity, sim_best = 0.996 (> 0.100 thold), f_keep = 1.000
1482.47.702.471 I reasoning-budget: activated, budget=2147483647 tokens
1482.47.702.475 I reasoning-budget: deactivated (natural end)
1482.47.702.606 I slot launch_slot_: id  0 | task 232227 | processing task, is_child = 0
1482.47.702.608 I slot slot_save_an: id  3 | task -1 | saving idle slot to prompt cache
1482.47.705.745 W srv   prompt_save:  - saving prompt with length 28639, total state size = 294.640 MiB (draft: 21.522 MiB)
1482.47.746.913 I srv  params_from_: Chat format: peg-native
1482.48.190.373 I slot prompt_clear: id  3 | task -1 | clearing prompt with 28639 tokens
1482.48.199.226 I srv        update:  - cache state: 2 prompts, 1152.091 MiB (limits: 8192.000 MiB, 262144 tokens, 290750 est)
1482.48.199.230 I srv        update:    - prompt 0x570dee03f930:   12251 tokens, checkpoints:  3,   374.292 MiB
1482.48.199.231 I srv        update:    - prompt 0x570def5c9810:   28639 tokens, checkpoints:  6,   777.800 MiB
1482.48.199.238 I slot get_availabl: id  3 | task -1 | selected slot by LRU, t_last = 285083052679
1482.48.199.239 I srv  get_availabl: updating prompt cache
1482.48.199.241 I srv          load:  - looking for better prompt, base f_keep = -1.000, sim = 0.000
1482.48.199.265 I srv          load:  - found better prompt with f_keep = 1.000, sim = 0.455
1482.51.174.710 I srv        update:  - cache state: 1 prompts, 374.292 MiB (limits: 8192.000 MiB, 262144 tokens, 268133 est)
1482.51.174.714 I srv        update:    - prompt 0x570dee03f930:   12251 tokens, checkpoints:  3,   374.292 MiB
1482.51.174.716 I srv  get_availabl: prompt cache update took 2975.48 ms
1482.51.176.866 I reasoning-budget: activated, budget=2147483647 tokens
1482.51.176.868 I reasoning-budget: deactivated (natural end)
1482.51.176.991 I slot launch_slot_: id  3 | task 232229 | processing task, is_child = 0
1482.51.182.122 W slot update_slots: id  0 | task 232227 | n_past = 30946, slot.prompt.tokens.size() = 30946, seq_id = 0, pos_min = 30945, n_swa = 0
1482.51.182.126 I slot update_slots: id  0 | task 232227 | Checking checkpoint with [30803, 30803] against 30945...
1482.51.489.586 W slot update_slots: id  0 | task 232227 | restored context checkpoint (pos_min = 30803, pos_max = 30803, n_tokens = 30804, n_past = 30804, size = 85.962 MiB)
1482.51.490.670 W slot update_slots: id  3 | task 232229 | n_past = 28639, slot.prompt.tokens.size() = 28639, seq_id = 3, pos_min = 28638, n_swa = 0
1482.51.490.672 I slot update_slots: id  3 | task 232229 | Checking checkpoint with [28446, 28446] against 28638...
1482.51.780.399 W slot update_slots: id  3 | task 232229 | restored context checkpoint (pos_min = 28446, pos_max = 28446, n_tokens = 28447, n_past = 28447, size = 84.191 MiB)
1482.53.305.403 I slot print_timing: id  1 | task 231735 | n_decoded =    511, tg =   6.19 t/s
1482.53.347.697 I slot create_check: id  0 | task 232227 | created context checkpoint 8 of 32 (pos_min = 31067, pos_max = 31067, n_tokens = 31068, size = 86.161 MiB)
1482.54.782.492 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =   3820, progress = 0.51, t =   3.29 s / 1160.45 tokens per second
1482.56.230.309 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =   5862, progress = 0.54, t =   4.74 s / 1236.80 tokens per second
1482.57.681.309 I slot print_timing: id  1 | task 231735 | n_decoded =    514, tg =   5.91 t/s
1482.57.692.369 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =   7904, progress = 0.58, t =   6.20 s / 1274.49 tokens per second
1482.59.149.018 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =   9946, progress = 0.61, t =   7.66 s / 1298.71 tokens per second
1482.59.149.906 I slot update_slots: id  3 | task 232229 | 8192 tokens since last checkpoint at 28447, creating new checkpoint during processing at position 40435
1482.59.186.028 I slot create_check: id  3 | task 232229 | created context checkpoint 7 of 32 (pos_min = 38392, pos_max = 38392, n_tokens = 38393, size = 91.665 MiB)
1483.00.665.714 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =  11988, progress = 0.64, t =   9.18 s / 1306.59 tokens per second
1483.02.144.956 I slot print_timing: id  1 | task 231735 | n_decoded =    517, tg =   5.66 t/s
1483.02.155.904 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =  14030, progress = 0.67, t =  10.67 s / 1315.49 tokens per second
1483.03.660.862 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =  16072, progress = 0.71, t =  12.17 s / 1320.60 tokens per second
1483.05.173.565 I slot print_timing: id  1 | task 231735 | n_decoded =    519, tg =   5.50 t/s
1483.05.186.901 I slot print_timing: id  3 | task 232229 | prompt processing, n_tokens =  18114, progress = 0.74, t =  13.70 s / 1322.55 tokens per second
1483.05.191.289 E init_batch: failed to prepare attention ubatches
1483.05.191.308 W decode: failed to find a memory slot for batch of size 2048
1483.05.191.311 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 1024, ret = 1
1483.05.194.031 E init_batch: failed to prepare attention ubatches
1483.05.194.043 W decode: failed to find a memory slot for batch of size 1024
1483.05.194.045 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 512, ret = 1
1483.05.196.518 E init_batch: failed to prepare attention ubatches
1483.05.196.526 W decode: failed to find a memory slot for batch of size 512
1483.05.196.528 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 256, ret = 1
1483.05.198.950 E init_batch: failed to prepare attention ubatches
1483.05.198.960 W decode: failed to find a memory slot for batch of size 256
1483.05.198.962 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 128, ret = 1
1483.05.201.299 E init_batch: failed to prepare attention ubatches
1483.05.201.306 W decode: failed to find a memory slot for batch of size 128
1483.05.201.308 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 64, ret = 1
1483.05.203.609 E init_batch: failed to prepare attention ubatches
1483.05.203.617 W decode: failed to find a memory slot for batch of size 64
1483.05.203.619 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 32, ret = 1
1483.05.206.751 E init_batch: failed to prepare attention ubatches
1483.05.206.765 W decode: failed to find a memory slot for batch of size 32
1483.05.206.768 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 16, ret = 1
1483.05.209.109 E init_batch: failed to prepare attention ubatches
1483.05.209.118 W decode: failed to find a memory slot for batch of size 16
1483.05.209.119 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 8, ret = 1
1483.05.211.503 E init_batch: failed to prepare attention ubatches
1483.05.211.523 W decode: failed to find a memory slot for batch of size 8
1483.05.211.525 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 4, ret = 1
1483.05.214.941 E init_batch: failed to prepare attention ubatches
1483.05.214.953 W decode: failed to find a memory slot for batch of size 4
1483.05.214.956 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 2, ret = 1
/home/z/llama.cpp/common/sampling.cpp:154: GGML_ASSERT(logits != nullptr) failed
1483.05.247.093 E get_logits_ith: invalid logits id 2, reason: batch.logits[2] != true
[New LWP 1685059]
[New LWP 1685058]
[New LWP 1685057]
[New LWP 1685056]
[New LWP 1685055]
[New LWP 1685054]
[New LWP 1685053]
[New LWP 1685052]
[New LWP 1685051]
[New LWP 1685050]
[New LWP 1685049]
[New LWP 1685048]
[New LWP 1685047]
[New LWP 1685046]
[New LWP 1685045]
[New LWP 1685041]
[New LWP 1685040]

This GDB supports auto-downloading debuginfo from the following URLs:
  <https://debuginfod.ubuntu.com>
Enable debuginfod for this session? (y or [n]) [answered N; input not from terminal]
Debuginfod has been disabled.
To make this setting permanent, add 'set debuginfod enabled off' to .gdbinit.
warning: Could not load shared library symbols for 3 libraries, e.g. /usr/local/cuda-13.2/lib64/libcudart.so.13.
Use the "info sharedlibrary" command to see the complete listing.
Do you need "set solib-search-path" or "set sysroot"?
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x000079c9e0310813 in __GI___wait4 (pid=1949300, stat_loc=0x0, options=0, usage=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
warning: 30	../sysdeps/unix/sysv/linux/wait4.c: No such file or directory
#0  0x000079c9e0310813 in __GI___wait4 (pid=1949300, stat_loc=0x0, options=0, usage=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
30	in ../sysdeps/unix/sysv/linux/wait4.c
#1  0x000079c9e055a073 in ggml_print_backtrace () from /home/z/llama.cpp/build/bin/libggml-base.so.0
#2  0x000079c9e055a223 in ggml_abort () from /home/z/llama.cpp/build/bin/libggml-base.so.0
#3  0x000079c9dfeb60bf in common_sampler_sample(common_sampler*, llama_context*, int, bool) () from /home/z/llama.cpp/build/bin/libllama-common.so.0
#4  0x000079c9dfeb62b0 in common_sampler_sample_and_accept_n(common_sampler*, llama_context*, std::vector<int, std::allocator<int> > const&, std::vector<int, std::allocator<int> > const&, bool) () from /home/z/llama.cpp/build/bin/libllama-common.so.0
#5  0x000079c9e0b72031 in server_context_impl::update_slots() () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#6  0x000079c9e0c03091 in server_queue::start_loop(long) () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#7  0x000079c9e0acc08b in llama_server(int, char**) () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#8  0x000079c9e022a1ca in __libc_start_call_main (main=main@entry=0x570db0407270, argc=argc@entry=29, argv=argv@entry=0x7ffe53142128) at ../sysdeps/nptl/libc_start_call_main.h:58
warning: 58	../sysdeps/nptl/libc_start_call_main.h: No such file or directory
#9  0x000079c9e022a28b in __libc_start_main_impl (main=0x570db0407270, argc=29, argv=0x7ffe53142128, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe53142118) at ../csu/libc-start.c:360
warning: 360	../csu/libc-start.c: No such file or directory
#10 0x0000570db04072a5 in ?? ()
[Inferior 1 (process 1685038) detached]
Aborted (core dumped)

[ZisIsNotZis]

Encountered again:

Details

4045.13.348.425 I slot print_timing: id  1 | task 124924 | prompt eval time =   40065.55 ms / 64515 tokens (    0.62 ms per token,  1610.24 tokens per second)
4045.13.348.431 I slot print_timing: id  1 | task 124924 |        eval time =  124552.84 ms /  1042 tokens (  119.53 ms per token,     8.37 tokens per second)
4045.13.348.432 I slot print_timing: id  1 | task 124924 |       total time =  164618.39 ms / 65557 tokens
4045.13.348.433 I slot print_timing: id  1 | task 124924 |    graphs reused =      99013
4045.13.348.434 I slot print_timing: id  1 | task 124924 | draft acceptance = 0.00483 (    5 accepted /  1036 generated)
4045.13.348.453 I statistics        draft-mtp: #calls(b,g,a) =  272 123858 186746, #gen drafts = 186747, #acc drafts =  6830, #gen tokens = 186747, #acc tokens =  6830, dur(b,g,a) = 0.416, 315597.723, 239.181 ms
4045.13.351.087 I slot      release: id  1 | task 124924 | stop processing: n_tokens = 65556, truncated = 0
4045.14.394.963 I srv  params_from_: Chat format: peg-native
4045.14.518.229 I slot get_availabl: id  1 | task -1 | selected slot by LCP similarity, sim_best = 0.997 (> 0.100 thold), f_keep = 1.000
4045.14.521.002 I reasoning-budget: activated, budget=2147483647 tokens
4045.14.521.005 I reasoning-budget: deactivated (natural end)
4045.14.521.226 I slot launch_slot_: id  1 | task 126080 | processing task, is_child = 0
4045.14.526.217 I slot update_slots: id  1 | task 126080 | Checking checkpoint with [64510, 64510] against 65555...
4045.15.148.860 W slot update_slots: id  1 | task 126080 | restored context checkpoint (pos_min = 64510, pos_max = 64510, n_tokens = 64511, n_past = 64511, size = 111.293 MiB)
4045.15.775.791 I slot print_timing: id  0 | task 125450 | n_decoded =    634, tg =  11.67 t/s
4045.15.779.629 I slot print_timing: id  2 | task 124990 | n_decoded =   1124, tg =  12.23 t/s
4045.16.180.976 I slot create_check: id  1 | task 126080 | created context checkpoint 3 of 32 (pos_min = 65557, pos_max = 65557, n_tokens = 65558, size = 112.080 MiB)
4045.18.833.461 I slot print_timing: id  0 | task 125450 | n_decoded =    665, tg =  11.59 t/s
4045.18.841.030 I slot print_timing: id  2 | task 124990 | n_decoded =   1157, tg =  12.18 t/s
4045.21.883.799 I slot print_timing: id  0 | task 125450 | n_decoded =    704, tg =  11.65 t/s
4045.21.899.202 I slot print_timing: id  2 | task 124990 | n_decoded =   1197, tg =  12.21 t/s
4045.24.722.527 I slot print_timing: id  1 | task 126080 | n_decoded =    100, tg =  12.26 t/s
4045.24.892.000 I slot print_timing: id  0 | task 125450 | n_decoded =    739, tg =  11.65 t/s
4045.24.900.359 I slot print_timing: id  2 | task 124990 | n_decoded =   1236, tg =  12.23 t/s
4045.27.777.463 I slot print_timing: id  1 | task 126080 | n_decoded =    136, tg =  12.13 t/s
4045.27.941.350 I slot print_timing: id  0 | task 125450 | n_decoded =    775, tg =  11.65 t/s
4045.27.949.814 I slot print_timing: id  2 | task 124990 | n_decoded =   1276, tg =  12.26 t/s
4045.29.237.364 E init_batch: failed to prepare attention ubatches
4045.29.237.372 W decode: failed to find a memory slot for batch of size 6
4045.29.237.374 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 1024, ret = 1
4045.29.239.515 E init_batch: failed to prepare attention ubatches
4045.29.239.519 W decode: failed to find a memory slot for batch of size 6
4045.29.239.520 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 512, ret = 1
4045.29.241.605 E init_batch: failed to prepare attention ubatches
4045.29.241.610 W decode: failed to find a memory slot for batch of size 6
4045.29.241.612 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 256, ret = 1
4045.29.243.595 E init_batch: failed to prepare attention ubatches
4045.29.243.599 W decode: failed to find a memory slot for batch of size 6
4045.29.243.600 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 128, ret = 1
4045.29.245.606 E init_batch: failed to prepare attention ubatches
4045.29.245.612 W decode: failed to find a memory slot for batch of size 6
4045.29.245.614 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 64, ret = 1
4045.29.247.697 E init_batch: failed to prepare attention ubatches
4045.29.247.702 W decode: failed to find a memory slot for batch of size 6
4045.29.247.703 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 32, ret = 1
4045.29.249.823 E init_batch: failed to prepare attention ubatches
4045.29.249.829 W decode: failed to find a memory slot for batch of size 6
4045.29.249.830 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 16, ret = 1
4045.29.251.959 E init_batch: failed to prepare attention ubatches
4045.29.251.965 W decode: failed to find a memory slot for batch of size 6
4045.29.251.966 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 8, ret = 1
4045.29.254.060 E init_batch: failed to prepare attention ubatches
4045.29.254.064 W decode: failed to find a memory slot for batch of size 6
4045.29.254.080 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 4, ret = 1
/home/z/llama.cpp/common/sampling.cpp:154: GGML_ASSERT(logits != nullptr) failed
4045.29.310.955 E get_logits_ith: invalid logits id 4, reason: batch.logits[4] != true
[New LWP 2258263]
[New LWP 2258262]
[New LWP 2258261]
[New LWP 2258260]
[New LWP 2258259]
[New LWP 2258258]
[New LWP 2258257]
[New LWP 2258256]
[New LWP 2258255]
[New LWP 2258254]
[New LWP 2258253]
[New LWP 2258252]
[New LWP 2258251]
[New LWP 2258250]
[New LWP 2258249]
[New LWP 2258245]
[New LWP 2258244]

This GDB supports auto-downloading debuginfo from the following URLs:
  <https://debuginfod.ubuntu.com>
Enable debuginfod for this session? (y or [n]) [answered N; input not from terminal]
Debuginfod has been disabled.
To make this setting permanent, add 'set debuginfod enabled off' to .gdbinit.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x000076dd3e510813 in __GI___wait4 (pid=2625383, stat_loc=0x0, options=0, usage=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
warning: 30	../sysdeps/unix/sysv/linux/wait4.c: No such file or directory
#0  0x000076dd3e510813 in __GI___wait4 (pid=2625383, stat_loc=0x0, options=0, usage=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
30	in ../sysdeps/unix/sysv/linux/wait4.c
#1  0x000076dd3eb4a0a3 in ggml_print_backtrace () from /home/z/llama.cpp/build/bin/libggml-base.so.0
#2  0x000076dd3eb4a253 in ggml_abort () from /home/z/llama.cpp/build/bin/libggml-base.so.0
#3  0x000076dd3e0c7acf in common_sampler_sample(common_sampler*, llama_context*, int, bool) () from /home/z/llama.cpp/build/bin/libllama-common.so.0
#4  0x000076dd3e0c7cc0 in common_sampler_sample_and_accept_n(common_sampler*, llama_context*, std::vector<int, std::allocator<int> > const&, std::vector<int, std::allocator<int> > const&, bool) () from /home/z/llama.cpp/build/bin/libllama-common.so.0
#5  0x000076dd3ed7f4c1 in server_context_impl::update_slots() () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#6  0x000076dd3ee107f1 in server_queue::start_loop(long) () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#7  0x000076dd3ecd73eb in llama_server(int, char**) () from /home/z/llama.cpp/build/bin/libllama-server-impl.so
#8  0x000076dd3e42a1ca in __libc_start_call_main (main=main@entry=0x5a7f7f918270, argc=argc@entry=29, argv=argv@entry=0x7ffd7fdc1aa8) at ../sysdeps/nptl/libc_start_call_main.h:58
warning: 58	../sysdeps/nptl/libc_start_call_main.h: No such file or directory
#9  0x000076dd3e42a28b in __libc_start_main_impl (main=0x5a7f7f918270, argc=29, argv=0x7ffd7fdc1aa8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffd7fdc1a98) at ../csu/libc-start.c:360
warning: 360	../csu/libc-start.c: No such file or directory
#10 0x00005a7f7f9182a5 in ?? ()
[Inferior 1 (process 2258242) detached]
Aborted (core dumped)

[whoreson]

I'm also seeing this, with 3.6-27b + ROCM, even with only 127k cache filled out of total 150k.

Confirmed workaround: I gave up.

[de-flandres]

I guess I also got it today:

logs

215.46.971.607 I slot print_timing: id  0 | task 17934 | n_decoded =    102, tg =  10.51 t/s
215.50.220.831 I slot print_timing: id  0 | task 17934 | n_decoded =    140, tg =  10.81 t/s
215.53.465.230 I slot print_timing: id  0 | task 17934 | n_decoded =    177, tg =  10.93 t/s
215.56.747.831 I slot print_timing: id  0 | task 17934 | n_decoded =    217, tg =  11.14 t/s
215.59.713.383 I slot print_timing: id  0 | task 17934 | prompt eval time =    7212.92 ms /   456 tokens (   15.82 ms per token,    63.22 tokens per second)
215.59.713.387 I slot print_timing: id  0 | task 17934 |        eval time =   22443.06 ms /   250 tokens (   89.77 ms per token,    11.14 tokens per second)
215.59.713.388 I slot print_timing: id  0 | task 17934 |       total time =   29655.98 ms /   706 tokens
215.59.713.390 I slot print_timing: id  0 | task 17934 |    graphs reused =      16621
215.59.713.391 I slot print_timing: id  0 | task 17934 | draft acceptance = 0.87923 (  182 accepted /   207 generated)
215.59.713.413 I statistics        draft-mtp: #calls(b,g,a) =  157  16942  16942, #gen drafts =  16942, #acc drafts = 14489, #gen tokens =  50826, #acc tokens = 31615, dur(b,g,a) = 0.603, 664688.072, 39.759 ms
215.59.717.035 I slot      release: id  0 | task 17934 | stop processing: n_tokens = 78554, truncated = 0
215.59.717.857 I srv  update_slots: all slots are idle
216.00.518.432 I srv  params_from_: Chat format: peg-native
216.00.639.462 I slot get_availabl: id  0 | task -1 | selected slot by LCP similarity, sim_best = 0.997 (> 0.100 thold), f_keep = 1.000
216.00.640.253 I reasoning-budget: activated, budget=2048 tokens
216.00.640.407 I slot launch_slot_: id  0 | task 18007 | processing task, is_child = 0
216.00.640.709 W slot create_check: id  0 | task 18007 | erasing old context checkpoint (pos_min = 68281, pos_max = 68281, n_tokens = 76796, size = 385.271 MiB)
216.00.755.895 I slot create_check: id  0 | task 18007 | created context checkpoint 4 of 4 (pos_min = 70039, pos_max = 70039, n_tokens = 78554, size = 388.952 MiB)
216.04.207.378 I slot print_timing: id  0 | task 18007 | prompt processing, n_tokens =    249, progress = 1.00, t =   3.57 s / 69.81 tokens per second
216.06.234.175 I reasoning-budget: deactivated (natural end)
216.13.326.423 I slot print_timing: id  0 | task 18007 | n_decoded =    101, tg =  11.58 t/s
216.16.553.710 I slot print_timing: id  0 | task 18007 | n_decoded =    141, tg =  11.80 t/s
216.19.800.427 I slot print_timing: id  0 | task 18007 | n_decoded =    181, tg =  11.91 t/s
216.20.163.442 E init_batch: failed to prepare attention ubatches
216.20.163.449 W decode: failed to find a memory slot for batch of size 4
216.20.163.454 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 1024, ret = 1
216.20.165.091 E init_batch: failed to prepare attention ubatches
216.20.165.093 W decode: failed to find a memory slot for batch of size 4
216.20.165.094 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 512, ret = 1
216.20.166.743 E init_batch: failed to prepare attention ubatches
216.20.166.745 W decode: failed to find a memory slot for batch of size 4
216.20.166.746 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 256, ret = 1
216.20.168.391 E init_batch: failed to prepare attention ubatches
216.20.168.393 W decode: failed to find a memory slot for batch of size 4
216.20.168.394 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 128, ret = 1
216.20.170.030 E init_batch: failed to prepare attention ubatches
216.20.170.033 W decode: failed to find a memory slot for batch of size 4
216.20.170.033 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 64, ret = 1
216.20.171.687 E init_batch: failed to prepare attention ubatches
216.20.171.690 W decode: failed to find a memory slot for batch of size 4
216.20.171.691 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 32, ret = 1
216.20.173.470 E init_batch: failed to prepare attention ubatches
216.20.173.472 W decode: failed to find a memory slot for batch of size 4
216.20.173.473 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 16, ret = 1
216.20.175.108 E init_batch: failed to prepare attention ubatches
216.20.175.110 W decode: failed to find a memory slot for batch of size 4
216.20.175.111 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 8, ret = 1
216.20.176.757 E init_batch: failed to prepare attention ubatches
216.20.176.759 W decode: failed to find a memory slot for batch of size 4
216.20.176.760 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 4, ret = 1
216.20.178.412 E init_batch: failed to prepare attention ubatches
216.20.178.414 W decode: failed to find a memory slot for batch of size 4
216.20.178.415 W srv  update_slots: failed to find free space in the KV cache, retrying with smaller batch size, i = 0, n_batch = 2, ret = 1
216.20.416.898 E get_logits_ith: invalid logits id 2, reason: batch.logits[2] != true
/opt/llama.cpp/common/sampling.cpp:154: GGML_ASSERT(logits != nullptr) failed
/lib64/libggml-base.so.0(+0x47e5) [0x7fe23626b7e5]
/lib64/libggml-base.so.0(ggml_print_backtrace+0x1eb) [0x7fe23626bbbb]
/lib64/libggml-base.so.0(ggml_abort+0x11f) [0x7fe23626bd3f]
/lib64/libllama-common.so.0(_Z21common_sampler_sampleP14common_samplerP13llama_contextib+0x11e6) [0x7fe23b21c536]
/lib64/libllama-common.so.0(_Z34common_sampler_sample_and_accept_nP14common_samplerP13llama_contextRKSt6vectorIiSaIiEES7_b+0xde) [0x7fe23b21c65e]
/lib64/libllama-server-impl.so(_ZN19server_context_impl12update_slotsEv+0x4882) [0x7fe23b568902]
/lib64/libllama-server-impl.so(_ZN12server_queue10start_loopEl+0x1f8a) [0x7fe23b5fce5a]
/lib64/libllama-server-impl.so(_Z12llama_serveriPPc+0x3661) [0x7fe23b4bc8c1]
/lib64/libc.so.6(+0x35b5) [0x7fe2356b15b5]
/lib64/libc.so.6(__libc_start_main+0x88) [0x7fe2356b1668]
/usr/bin/llama-server() [0x4003b5]

Used PiehSoft/Qwen3.6-40B-Deckard-MTP with this model for vision

docker compose

services:
  llamacpp-vulkan-radv-deckard-mmproj:
    image: llm-vulkan-radv:latest
    container_name: deckard-40b-vulkan-mmproj
    ports:
      - "11434:8080"
    devices:
      - /dev/kfd:/dev/kfd
      - /dev/dri:/dev/dri
    security_opt:
      - seccomp:unconfined
    environment:
      - LLAMA_ARG_HOST=0.0.0.0
      - LLAMA_ARG_PORT=8080
    volumes:
      - ${MODEL_PATH}:/model
      - ~/llm/models/mmproj:/mmproj
      - ~/llm/chat_template_new.jinja:/chat-template.jinja
    command: >
      -m /model/${MODEL_FILE}
      --mmproj /mmproj/mmproj-Qwen3.6-27B-f16.gguf
      --alias deckard-40b-q5k-mtp-vision
      --device Vulkan0
      --threads-http 1
      --spec-type draft-mtp
      --spec-draft-p-min 0.0
      --spec-draft-p-split 0.10
      --spec-draft-n-max 3
      --spec-draft-n-min 0
      --spec-draft-ngl all
      --n-gpu-layers 999
      --spec-draft-device Vulkan0
      --timeout 3600
      --n-predict 32000
      --ctx-size 80000
      --threads 16
      --threads-batch 16
      --parallel 1
      --cache-type-k q5_1
      --cache-type-v q8_0
      --spec-draft-type-k q8_0
      --spec-draft-type-v q8_0
      --batch-size 2048
      --ubatch-size 256
      --ctx-checkpoints 4
      --checkpoint-min-step 256
      --cache-reuse 256
      --cache-ram 4096
      --log-verbosity 3
      --log-timestamps
      --log-prefix
      --log-colors on
      --no-ui
      --image-min-tokens 1024
      --temp 0.6
      --top-p 0.95
      --top-k 20
      --min-p 0.0
      --presence-penalty 0.0
      --repeat-penalty 1.05
      --repeat-last-n 512
      --chat-template-file /chat-template.jinja
      --reasoning-format deepseek
      --reasoning on
      --reasoning-budget 2048
      --reasoning-budget-message ". Provide concise answer now."
      --chat-template-kwargs '{"preserve_thinking":true, "auto_disable_thinking_with_tools":false}'
      --poll 1
      --poll-batch 1
      --spec-draft-poll 1
      --spec-draft-poll-batch 1
      --jinja
      --flash-attn on
      --keep 20480
      --metrics

Dockerfile

# build stage
FROM registry.fedoraproject.org/fedora:43 AS builder

# deps
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
  git vim \
  make gcc cmake ninja-build lld clang clang-devel compiler-rt libcurl-devel \
  vulkan-loader-devel vulkaninfo mesa-vulkan-drivers \
  spirv-headers-devel radeontop glslc patch \
  && dnf clean all && rm -rf /var/cache/dnf/*

# llama.cpp
WORKDIR /opt/llama.cpp
ARG REPO=https://github.com/ggerganov/llama.cpp.git
ARG BRANCH=master
RUN git clone -b ${BRANCH} --single-branch --recursive ${REPO} .

COPY llama-grammar.patch /tmp/llama-grammar.patch
COPY fix-seq-rm-rescue.patch /tmp/fix-seq-rm-rescue.patch

# build
RUN git clean -xdf \
  && git submodule update --recursive \
  && patch -p1 < /tmp/llama-grammar.patch \
  && patch -p1 < /tmp/fix-seq-rm-rescue.patch \
  && cmake -S . -B build -G Ninja \
  -DGGML_VULKAN=ON \
  -DCMAKE_BUILD_TYPE=Release \
  -DGGML_RPC=ON \
  -DCMAKE_INSTALL_PREFIX=/usr \
  -DLLAMA_BUILD_TESTS=OFF \
  -DLLAMA_BUILD_EXAMPLES=ON \
  -DLLAMA_BUILD_SERVER=ON \
  -DLLAMA_BUILD_UI=OFF \
  && cmake --build build --config Release -- -j$(nproc) \
  && cmake --install build --config Release

# libs
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
  && ldconfig

# helper
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py

# runtime stage
FROM registry.fedoraproject.org/fedora-minimal:43

# runtime deps
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
  bash ca-certificates libatomic libstdc++ libgcc \
  vulkan-loader vulkan-loader-devel vulkaninfo mesa-vulkan-drivers radeontop procps-ng \
  && microdnf clean all && rm -rf /var/cache/dnf/*

# copy
COPY --from=builder /usr/ /usr/
COPY --from=builder /usr/local/ /usr/local/
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/

# ld
RUN echo "/usr/local/lib"  > /etc/ld.so.conf.d/local.conf \
  && echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
  && ldconfig \
  && cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
  && ldconfig

# helper
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py

ENTRYPOINT ["/usr/bin/llama-server"]

fix-seq-rm-rescue.patch

fix-seq-rm-rescue: rescue multi-turn KV cache on hybrid recurrent models
See: https://github.com/ggml-org/llama.cpp/issues/22746
Upstream: 57f81cb13849de50a39e509e712d7c68e4f3e4b7

Adds backstep-and-restore path that prevents update_slots from wiping
entire slot on every multi-turn request to QWEN35MOE / QWEN3NEXT.
Post-LCP cut rounded to nearest turn boundary, checkpoint infrastructure
used to restore state when recurrent backend rejects partial seq_rm,
checkpoint forced on every boundary token.

From 57f81cb13849de50a39e509e712d7c68e4f3e4b7 Mon Sep 17 00:00:00 2001
From: Reed Mayhew <46308944+reedmayhew18@users.noreply.github.com>
Date: Tue, 26 May 2026 23:28:04 -0400
Subject: [PATCH] server: rescue multi-turn KV cache on hybrid recurrent
 models

---
 tools/server/server-context.cpp | 142 +++++++++++++++++++++++++++++++-
 1 file changed, 138 insertions(+), 4 deletions(-)

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index ae9e0bf60d8d..1a2a5a6e6671 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -622,6 +622,34 @@ struct server_metrics {
 // server_context_impl (private implementation)
 //
 
+static llama_pos find_safe_boundary_backstep(
+        const server_tokens & cached,
+        llama_pos             n_past,
+        const std::vector<std::vector<llama_token>> & boundaries) {
+    if (boundaries.empty() || n_past <= 0) {
+        return 0;
+    }
+    for (llama_pos i = n_past; i > 0; --i) {
+        for (const auto & bt : boundaries) {
+            const llama_pos blen = (llama_pos) bt.size();
+            if (blen == 0 || blen > i) {
+                continue;
+            }
+            bool match = true;
+            for (llama_pos k = 0; k < blen; ++k) {
+                if (cached[i - blen + k] != bt[k]) {
+                    match = false;
+                    break;
+                }
+            }
+            if (match) {
+                return i;
+            }
+        }
+    }
+    return 0;
+}
+
 struct server_context_impl {
     friend struct server_context;
 
@@ -671,6 +699,9 @@ struct server_context_impl {
     common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
     common_context_seq_rm_type ctx_dft_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
 
+    std::vector<std::string>              turn_boundary_strings;
+    std::vector<std::vector<llama_token>> turn_boundary_tokens;
+
     common_speculative_ptr spec;
 
     bool add_bos_token = true;
@@ -1027,6 +1058,22 @@ struct server_context_impl {
             SRV_WRN("%s", "speculative decoding will use checkpoints\n");
         }
 
+        if (ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {
+            turn_boundary_strings = { "<|im_end|>\n", "<|im_end|>" };
+            turn_boundary_tokens.reserve(turn_boundary_strings.size());
+            for (const auto & s : turn_boundary_strings) {
+                auto toks = common_tokenize(vocab, s, false, true);
+                if (toks.empty()) {
+                    SRV_WRN("turn-boundary string '%s' did not tokenize; skipping\n", s.c_str());
+                    continue;
+                }
+                turn_boundary_tokens.push_back(std::move(toks));
+            }
+            if (!turn_boundary_tokens.empty()) {
+                SRV_INF("turn-boundary backstep enabled (%zu boundary string(s))\n", turn_boundary_tokens.size());
+            }
+        }
+
         // initialize slots
         for (int i = 0; i < params_base.n_parallel; i++) {
             slots.emplace_back();
@@ -2768,7 +2815,7 @@ struct server_context_impl {
                                     SLT_WRN(slot, "%s\n", st1.str().c_str());
                                 }
 
-                                if (pos_min >= pos_min_thold) {
+                                if (n_swa > 0 && pos_min >= pos_min_thold) {
                                     // search for a context checkpoint
                                     const auto it = std::find_if(
                                         slot.prompt.checkpoints.rbegin(),
@@ -2823,6 +2870,22 @@ struct server_context_impl {
                             SLT_WRN(slot, "n_past was set to %d\n", n_past);
                         }
 
+                        if (n_past > 0 &&
+                            (size_t) n_past < slot.prompt.tokens.size() &&
+                            !slot.prompt.tokens.has_mtmd &&
+                            !turn_boundary_tokens.empty() &&
+                            ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {
+                            const llama_pos n_past_safe = find_safe_boundary_backstep(
+                                slot.prompt.tokens, (llama_pos) n_past, turn_boundary_tokens);
+                            if (n_past_safe > 0 && n_past_safe < (llama_pos) n_past) {
+                                SLT_INF(slot, "LCP backstep: n_past %d -> %d (turn-boundary aligned)\n",
+                                        (int) n_past, (int) n_past_safe);
+                                n_past = (size_t) n_past_safe;
+                            } else if (n_past_safe == 0) {
+                                SLT_INF(slot, "%s", "LCP backstep: no turn boundary in cached prefix; existing fallback will handle if seq_rm fails\n");
+                            }
+                        }
+
                         slot.n_prompt_tokens_cache = n_past;
                         slot.n_prompt_tokens_processed = 0;
 
@@ -2851,9 +2914,45 @@ struct server_context_impl {
 
                     SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0);
 
-                    common_context_seq_rm(ctx_tgt, slot.id, p0, -1);
+                    bool ctx_tgt_restored_from_checkpoint = false;
+                    bool ctx_tgt_cleared_from_seq_rm_failure = false;
+
+                    if (!llama_memory_seq_rm(llama_get_memory(ctx_tgt), slot.id, p0, -1)) {
+                        const llama_pos p0_checkpoint_max = p0 > 0 ? p0 - 1 : -1;
+
+                        auto it = std::find_if(
+                            slot.prompt.checkpoints.rbegin(),
+                            slot.prompt.checkpoints.rend(),
+                            [&](const auto & cur) { return cur.pos_max <= p0_checkpoint_max; }
+                        );
+                        if (it != slot.prompt.checkpoints.rend()) {
+                            it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                            if (ctx_dft) {
+                                it->load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                            }
+                            const llama_pos pos_restored = std::max(it->pos_min + 1, it->pos_max);
+                            const size_t    n_past_new   = std::min(slot.prompt.tokens.size_up_to_pos(pos_restored), (size_t) it->n_tokens);
+                            slot.n_prompt_tokens_cache = n_past_new;
+                            slot.prompt.tokens.keep_first(n_past_new);
+                            ctx_tgt_restored_from_checkpoint = true;
+                            SLT_INF(slot, "rescue: seq_rm at p0=%d failed; restored checkpoint at pos=%d (n_past %d -> %zu, size = %.3f MiB)\n",
+                                    (int) p0, (int) pos_restored, (int) p0, n_past_new, (float) it->size() / 1024 / 1024);
+                        } else {
+                            SLT_WRN(slot, "partial seq_rm at p0=%d failed (recurrent backend cannot roll into cache); no checkpoint <= p0-1, clearing slot and re-evaluating from scratch\n", p0);
+                            slot.n_prompt_tokens_cache = 0;
+                            slot.prompt.tokens.keep_first(0);
+                            common_context_seq_rm(ctx_tgt, slot.id, 0, -1);
+                            ctx_tgt_cleared_from_seq_rm_failure = true;
+                        }
+                    }
                     if (ctx_dft) {
-                        common_context_seq_rm(ctx_dft.get(), slot.id, p0, -1);
+                        if (ctx_tgt_cleared_from_seq_rm_failure) {
+                            common_context_seq_rm(ctx_dft.get(), slot.id, 0, -1);
+                        } else if (!ctx_tgt_restored_from_checkpoint) {
+                            if (!llama_memory_seq_rm(llama_get_memory(ctx_dft.get()), slot.id, p0, -1)) {
+                                common_context_seq_rm(ctx_dft.get(), slot.id, 0, -1);
+                            }
+                        }
                     }
 
                     // If using an alora, there may be uncached tokens that come
@@ -2983,6 +3082,37 @@ struct server_context_impl {
                     // the number of tokens added to the batch for the current slot
                     const auto n_tokens_cur = batch.n_tokens - n_tokens_prev;
 
+                    bool boundary_just_processed = false;
+                    if (!turn_boundary_tokens.empty() && n_tokens_cur > 0) {
+                        const size_t total       = slot.prompt.tokens.size();
+                        const size_t batch_start = total > (size_t) n_tokens_cur ? total - (size_t) n_tokens_cur : 0;
+
+                        size_t max_boundary = 0;
+                        for (const auto & bt : turn_boundary_tokens) {
+                            max_boundary = std::max(max_boundary, bt.size());
+                        }
+
+                        const size_t scan_start = batch_start > max_boundary ? batch_start - max_boundary + 1 : 0;
+                        for (size_t i = scan_start; i < total && !boundary_just_processed; ++i) {
+                            for (const auto & bt : turn_boundary_tokens) {
+                                if (bt.empty() || i + bt.size() > total || i + bt.size() <= batch_start) {
+                                    continue;
+                                }
+                                bool match = true;
+                                for (size_t k = 0; k < bt.size(); ++k) {
+                                    if (slot.prompt.tokens[i + k] != bt[k]) {
+                                        match = false;
+                                        break;
+                                    }
+                                }
+                                if (match) {
+                                    boundary_just_processed = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+
                     const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;
 
                     // entire prompt has been processed
@@ -3000,8 +3130,11 @@ struct server_context_impl {
                         slot.init_sampler();
                     } else {
                         // skip ordinary mid-prompt checkpoints
-                        if (!n_before_user_known && !near_prompt_end) {
+                        if (!n_before_user_known && !near_prompt_end && !boundary_just_processed) {
                             do_checkpoint = false;
+                        } else if (boundary_just_processed) {
+                            do_checkpoint = do_checkpoint && true;
+                            SLT_DBG(slot, "%s", "boundary token in batch, forcing checkpoint candidate\n");
                         }
                     }
 
@@ -3022,6 +3155,7 @@ struct server_context_impl {
                             n_tokens_start > n_before_user;
 
                         const bool is_allowed =
+                            boundary_just_processed ||
                             !n_before_user_known ||
                             is_on_user ||
                             (is_after_user && near_prompt_end);

llama-grammar.patch

--- a/src/llama-grammar.cpp
+++ b/src/llama-grammar.cpp
@@ -10,7 +10,7 @@
 #include <set>
 #include <stdexcept>
 
-#define MAX_REPETITION_THRESHOLD 2000
+#define MAX_REPETITION_THRESHOLD 100000
 //
 // helpers
 //