Snarf/homelab-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Kubernetes manifests for AI stack (Qdrant, vLLM, Triton)

Browse Source
This commit is contained in:
Snarf
2026-11-08 13:51:38 -05:00
parent 2bd66045c3
commit ed38845802
1 changed files with 379 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

379
manifests/spacedock/ai-stack/ai-stack.yaml Normal file
View File

@@ -0,0 +1,379 @@
apiVersion: v1
kind: Namespace
metadata:
name: ai-stack
labels:
app.kubernetes.io/part-of: spacedock-ai-stack
---
# ==============================================================================
# 1. QDRANT VECTOR DATABASE (Backed by 46.4 TiB Longhorn Flash Pool)
# ==============================================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: qdrant-storage-pvc
namespace: ai-stack
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 100Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: qdrant
namespace: ai-stack
labels:
app: qdrant
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: qdrant
template:
metadata:
labels:
app: qdrant
spec:
containers:
- name: qdrant
image: qdrant/qdrant:latest
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 6333
- name: grpc
containerPort: 6334
resources:
requests:
cpu: "2"
memory: "4Gi"
limits:
cpu: "8"
memory: "16Gi"
readinessProbe:
httpGet:
path: /readyz
port: 6333
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: qdrant-storage
mountPath: /qdrant/storage
volumes:
- name: qdrant-storage
persistentVolumeClaim:
claimName: qdrant-storage-pvc
---
apiVersion: v1
kind: Service
metadata:
name: qdrant
namespace: ai-stack
labels:
app: qdrant
spec:
type: NodePort
selector:
app: qdrant
ports:
- name: http
port: 6333
targetPort: 6333
nodePort: 30633
- name: grpc
port: 6334
targetPort: 6334
nodePort: 30634
---
# ==============================================================================
# 2. vLLM HIGH-THROUGHPUT LLM INFERENCE ENGINE (Tesla T4 GPU + Longhorn Cache)
# ==============================================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: vllm-models-pvc
namespace: ai-stack
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 200Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: vllm-openai
namespace: ai-stack
labels:
app: vllm-openai
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: vllm-openai
template:
metadata:
labels:
app: vllm-openai
spec:
containers:
- name: vllm
image: vllm/vllm-openai:v0.7.3
imagePullPolicy: IfNotPresent
args:
- "--model"
- "Qwen/Qwen2.5-7B-Instruct-AWQ"
- "--served-model-name"
- "Qwen2.5-7B-Instruct"
- "--quantization"
- "awq_marlin"
- "--dtype"
- "half"
- "--max-model-len"
- "16384"
- "--gpu-memory-utilization"
- "0.90"
- "--enable-auto-tool-choice"
- "--tool-call-parser"
- "hermes"
- "--host"
- "0.0.0.0"
- "--port"
- "8000"
env:
- name: HF_HOME
value: "/root/.cache/huggingface"
- name: VLLM_WORKER_MULTIPROC_METHOD
value: "spawn"
ports:
- name: http-openai
containerPort: 8000
resources:
requests:
cpu: "8"
memory: "24Gi"
nvidia.com/gpu: "1"
limits:
cpu: "24"
memory: "48Gi"
nvidia.com/gpu: "1"
readinessProbe:
httpGet:
path: /health
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
volumeMounts:
- name: vllm-models
mountPath: /root/.cache/huggingface
- name: dshm
mountPath: /dev/shm
volumes:
- name: vllm-models
persistentVolumeClaim:
claimName: vllm-models-pvc
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 16Gi
---
apiVersion: v1
kind: Service
metadata:
name: vllm-openai
namespace: ai-stack
labels:
app: vllm-openai
spec:
type: NodePort
selector:
app: vllm-openai
ports:
- name: http-openai
port: 8000
targetPort: 8000
nodePort: 30800
---
# ==============================================================================
# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving)
# ==============================================================================
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: triton-models-pvc
namespace: ai-stack
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 200Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: triton-inference-server
namespace: ai-stack
labels:
app: triton-inference-server
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: triton-inference-server
template:
metadata:
labels:
app: triton-inference-server
spec:
initContainers:
- name: seed-decision-engine-model
image: busybox:1.36
command: ["/bin/sh", "-c"]
args:
- |
mkdir -p /models/repository/decision_engine/1
cat << 'EOF' > /models/repository/decision_engine/config.pbtxt
name: "decision_engine"
backend: "python"
max_batch_size: 64
input [
{
name: "FEATURES"
data_type: TYPE_FP32
dims: [ -1 ]
}
]
output [
{
name: "DECISION_SCORE"
data_type: TYPE_FP32
dims: [ 1 ]
},
{
name: "ACTION_LABEL"
data_type: TYPE_STRING
dims: [ 1 ]
}
]
instance_group [
{
count: 1
kind: KIND_GPU
}
]
EOF
cat << 'EOF' > /models/repository/decision_engine/1/model.py
import numpy as np
import triton_python_backend_utils as pb_utils
class TritonPythonModel:
def initialize(self, args):
pass
def execute(self, requests):
responses = []
for request in requests:
features = pb_utils.get_input_tensor_by_name(request, "FEATURES").as_numpy()
# Sigmoid risk/decision scoring over feature vector
mean_val = float(np.mean(features))
score = float(1.0 / (1.0 + np.exp(-mean_val)))
label = "APPROVE_EXECUTE" if score >= 0.5 else "HOLD_REVIEW"
out_score = pb_utils.Tensor("DECISION_SCORE", np.array([[score]], dtype=np.float32))
out_label = pb_utils.Tensor("ACTION_LABEL", np.array([[label.encode("utf-8")]], dtype=np.object_))
responses.append(pb_utils.InferenceResponse(output_tensors=[out_score, out_label]))
return responses
def finalize(self):
pass
EOF
echo "Seeded initial decision_engine model into /models/repository/decision_engine"
volumeMounts:
- name: triton-models
mountPath: /models
containers:
- name: triton
image: nvcr.io/nvidia/tritonserver:24.08-py3
imagePullPolicy: IfNotPresent
command: ["tritonserver"]
args:
- "--model-repository=/models/repository"
- "--model-control-mode=poll"
- "--repository-poll-secs=10"
- "--strict-model-config=false"
- "--allow-http=true"
- "--allow-grpc=true"
- "--allow-metrics=true"
ports:
- name: http
containerPort: 8000
- name: grpc
containerPort: 8001
- name: metrics
containerPort: 8002
resources:
requests:
cpu: "4"
memory: "16Gi"
nvidia.com/gpu: "1"
limits:
cpu: "16"
memory: "32Gi"
nvidia.com/gpu: "1"
readinessProbe:
httpGet:
path: /v2/health/ready
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
volumeMounts:
- name: triton-models
mountPath: /models
- name: dshm
mountPath: /dev/shm
volumes:
- name: triton-models
persistentVolumeClaim:
claimName: triton-models-pvc
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 8Gi
---
apiVersion: v1
kind: Service
metadata:
name: triton-inference-server
namespace: ai-stack
labels:
app: triton-inference-server
spec:
type: NodePort
selector:
app: triton-inference-server
ports:
- name: http
port: 8000
targetPort: 8000
nodePort: 30801
- name: grpc
port: 8001
targetPort: 8001
nodePort: 30802
- name: metrics
port: 8002
targetPort: 8002
nodePort: 30803