From ad0bfb8146db14e15e0f57d5c47e9679972b1198 Mon Sep 17 00:00:00 2001 From: poprhythm Date: Sun, 13 Sep 2026 15:30:36 +0000 Subject: [PATCH] Enable flash attention + quantized KV cache for ollama Observed KV cache size for qwen2.5:7b at fixed 16384 context varying 224MB-896MB between model loads with flash attention off. The larger figure pushes total memory needs just past the GPU's free VRAM, so some loads only offload 25/29 layers instead of 29/29 -- causing DJ-agent pick latency to jump from ~1s to multiple minutes. This GPU (6GB) has very little slack for this model/context combination even after freeing obico and stable-diffusion's VRAM reservations. Claude-Session: https://claude.ai/code/session_01L7Rwa6guD5wK8F8tWQwcJX --- ollama/docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ollama/docker-compose.yml b/ollama/docker-compose.yml index 3cbbc0b..18bc455 100644 --- a/ollama/docker-compose.yml +++ b/ollama/docker-compose.yml @@ -10,6 +10,13 @@ services: - NVIDIA_DRIVER_CAPABILITIES=compute,utility - TZ=America/New_York - OLLAMA_HOST=0.0.0.0 + # This GPU (6GB) is right at the edge for qwen2.5:7b at 16384 context — + # KV cache size was observed varying 224MB-896MB between loads with flash + # attention off, and the larger figure tips offload to 25/29 layers + # instead of 29/29, causing multi-minute DJ-agent latency. Flash + # attention + an explicitly quantized KV cache keep it small and stable. + - OLLAMA_FLASH_ATTENTION=1 + - OLLAMA_KV_CACHE_TYPE=q8_0 volumes: - /srv/ollama:/root/.ollama runtime: nvidia