diff --git a/manifests/spacedock/ai-stack/ai-stack.yaml b/manifests/spacedock/ai-stack/ai-stack.yaml index d9a073d..ef968b1 100644 --- a/manifests/spacedock/ai-stack/ai-stack.yaml +++ b/manifests/spacedock/ai-stack/ai-stack.yaml @@ -249,6 +249,13 @@ spec: args: - | rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec* + python3 -c ' + p = "/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/triton_unified_attention.py" + s = open(p).read() + s = s.replace("return 32", "return 16") + s = s.replace("launch_num_stages: int | None = None", "launch_num_stages: int | None = 1") + open(p, "w").write(s) + ' exec vllm serve google/gemma-4-12B-it-qat-w4a16-ct \ --served-model-name Gemma-4-12B-it \ --dtype half \