From ed3884580297da3a56fb73d49cf9647e9b0711db Mon Sep 17 00:00:00 2001 From: Snarf Date: Sun, 8 Nov 2026 13:51:38 -0500 Subject: [PATCH] feat: add Kubernetes manifests for AI stack (Qdrant, vLLM, Triton) --- manifests/spacedock/ai-stack/ai-stack.yaml | 379 +++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 manifests/spacedock/ai-stack/ai-stack.yaml diff --git a/manifests/spacedock/ai-stack/ai-stack.yaml b/manifests/spacedock/ai-stack/ai-stack.yaml new file mode 100644 index 0000000..9403320 --- /dev/null +++ b/manifests/spacedock/ai-stack/ai-stack.yaml @@ -0,0 +1,379 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ai-stack + labels: + app.kubernetes.io/part-of: spacedock-ai-stack +--- +# ============================================================================== +# 1. QDRANT VECTOR DATABASE (Backed by 46.4 TiB Longhorn Flash Pool) +# ============================================================================== +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: qdrant-storage-pvc + namespace: ai-stack +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: qdrant + namespace: ai-stack + labels: + app: qdrant +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: qdrant + template: + metadata: + labels: + app: qdrant + spec: + containers: + - name: qdrant + image: qdrant/qdrant:latest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 6333 + - name: grpc + containerPort: 6334 + resources: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "8" + memory: "16Gi" + readinessProbe: + httpGet: + path: /readyz + port: 6333 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: qdrant-storage + mountPath: /qdrant/storage + volumes: + - name: qdrant-storage + persistentVolumeClaim: + claimName: qdrant-storage-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: qdrant + namespace: ai-stack + labels: + app: qdrant +spec: + type: NodePort + selector: + app: qdrant + ports: + - name: http + port: 6333 + targetPort: 6333 + nodePort: 30633 + - name: grpc + port: 6334 + targetPort: 6334 + nodePort: 30634 +--- +# ============================================================================== +# 2. vLLM HIGH-THROUGHPUT LLM INFERENCE ENGINE (Tesla T4 GPU + Longhorn Cache) +# ============================================================================== +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: vllm-models-pvc + namespace: ai-stack +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 200Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-openai + namespace: ai-stack + labels: + app: vllm-openai +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: vllm-openai + template: + metadata: + labels: + app: vllm-openai + spec: + containers: + - name: vllm + image: vllm/vllm-openai:v0.7.3 + imagePullPolicy: IfNotPresent + args: + - "--model" + - "Qwen/Qwen2.5-7B-Instruct-AWQ" + - "--served-model-name" + - "Qwen2.5-7B-Instruct" + - "--quantization" + - "awq_marlin" + - "--dtype" + - "half" + - "--max-model-len" + - "16384" + - "--gpu-memory-utilization" + - "0.90" + - "--enable-auto-tool-choice" + - "--tool-call-parser" + - "hermes" + - "--host" + - "0.0.0.0" + - "--port" + - "8000" + env: + - name: HF_HOME + value: "/root/.cache/huggingface" + - name: VLLM_WORKER_MULTIPROC_METHOD + value: "spawn" + ports: + - name: http-openai + containerPort: 8000 + resources: + requests: + cpu: "8" + memory: "24Gi" + nvidia.com/gpu: "1" + limits: + cpu: "24" + memory: "48Gi" + nvidia.com/gpu: "1" + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + volumeMounts: + - name: vllm-models + mountPath: /root/.cache/huggingface + - name: dshm + mountPath: /dev/shm + volumes: + - name: vllm-models + persistentVolumeClaim: + claimName: vllm-models-pvc + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 16Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm-openai + namespace: ai-stack + labels: + app: vllm-openai +spec: + type: NodePort + selector: + app: vllm-openai + ports: + - name: http-openai + port: 8000 + targetPort: 8000 + nodePort: 30800 +--- +# ============================================================================== +# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving) +# ============================================================================== +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: triton-models-pvc + namespace: ai-stack +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 200Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: triton-inference-server + namespace: ai-stack + labels: + app: triton-inference-server +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: triton-inference-server + template: + metadata: + labels: + app: triton-inference-server + spec: + initContainers: + - name: seed-decision-engine-model + image: busybox:1.36 + command: ["/bin/sh", "-c"] + args: + - | + mkdir -p /models/repository/decision_engine/1 + cat << 'EOF' > /models/repository/decision_engine/config.pbtxt + name: "decision_engine" + backend: "python" + max_batch_size: 64 + input [ + { + name: "FEATURES" + data_type: TYPE_FP32 + dims: [ -1 ] + } + ] + output [ + { + name: "DECISION_SCORE" + data_type: TYPE_FP32 + dims: [ 1 ] + }, + { + name: "ACTION_LABEL" + data_type: TYPE_STRING + dims: [ 1 ] + } + ] + instance_group [ + { + count: 1 + kind: KIND_GPU + } + ] + EOF + cat << 'EOF' > /models/repository/decision_engine/1/model.py + import numpy as np + import triton_python_backend_utils as pb_utils + + class TritonPythonModel: + def initialize(self, args): + pass + + def execute(self, requests): + responses = [] + for request in requests: + features = pb_utils.get_input_tensor_by_name(request, "FEATURES").as_numpy() + # Sigmoid risk/decision scoring over feature vector + mean_val = float(np.mean(features)) + score = float(1.0 / (1.0 + np.exp(-mean_val))) + label = "APPROVE_EXECUTE" if score >= 0.5 else "HOLD_REVIEW" + out_score = pb_utils.Tensor("DECISION_SCORE", np.array([[score]], dtype=np.float32)) + out_label = pb_utils.Tensor("ACTION_LABEL", np.array([[label.encode("utf-8")]], dtype=np.object_)) + responses.append(pb_utils.InferenceResponse(output_tensors=[out_score, out_label])) + return responses + + def finalize(self): + pass + EOF + echo "Seeded initial decision_engine model into /models/repository/decision_engine" + volumeMounts: + - name: triton-models + mountPath: /models + containers: + - name: triton + image: nvcr.io/nvidia/tritonserver:24.08-py3 + imagePullPolicy: IfNotPresent + command: ["tritonserver"] + args: + - "--model-repository=/models/repository" + - "--model-control-mode=poll" + - "--repository-poll-secs=10" + - "--strict-model-config=false" + - "--allow-http=true" + - "--allow-grpc=true" + - "--allow-metrics=true" + ports: + - name: http + containerPort: 8000 + - name: grpc + containerPort: 8001 + - name: metrics + containerPort: 8002 + resources: + requests: + cpu: "4" + memory: "16Gi" + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: "32Gi" + nvidia.com/gpu: "1" + readinessProbe: + httpGet: + path: /v2/health/ready + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - name: triton-models + mountPath: /models + - name: dshm + mountPath: /dev/shm + volumes: + - name: triton-models + persistentVolumeClaim: + claimName: triton-models-pvc + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: triton-inference-server + namespace: ai-stack + labels: + app: triton-inference-server +spec: + type: NodePort + selector: + app: triton-inference-server + ports: + - name: http + port: 8000 + targetPort: 8000 + nodePort: 30801 + - name: grpc + port: 8001 + targetPort: 8001 + nodePort: 30802 + - name: metrics + port: 8002 + targetPort: 8002 + nodePort: 30803