feat: add Kubernetes manifests for AI stack (Qdrant, vLLM, Triton)
This commit is contained in:
379
manifests/spacedock/ai-stack/ai-stack.yaml
Normal file
379
manifests/spacedock/ai-stack/ai-stack.yaml
Normal file
@@ -0,0 +1,379 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: ai-stack
|
||||
labels:
|
||||
app.kubernetes.io/part-of: spacedock-ai-stack
|
||||
---
|
||||
# ==============================================================================
|
||||
# 1. QDRANT VECTOR DATABASE (Backed by 46.4 TiB Longhorn Flash Pool)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: qdrant-storage-pvc
|
||||
namespace: ai-stack
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: qdrant
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: qdrant
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: qdrant
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: qdrant
|
||||
spec:
|
||||
containers:
|
||||
- name: qdrant
|
||||
image: qdrant/qdrant:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 6333
|
||||
- name: grpc
|
||||
containerPort: 6334
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: "4Gi"
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: "16Gi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 6333
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: qdrant-storage
|
||||
mountPath: /qdrant/storage
|
||||
volumes:
|
||||
- name: qdrant-storage
|
||||
persistentVolumeClaim:
|
||||
claimName: qdrant-storage-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: qdrant
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: qdrant
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: qdrant
|
||||
ports:
|
||||
- name: http
|
||||
port: 6333
|
||||
targetPort: 6333
|
||||
nodePort: 30633
|
||||
- name: grpc
|
||||
port: 6334
|
||||
targetPort: 6334
|
||||
nodePort: 30634
|
||||
---
|
||||
# ==============================================================================
|
||||
# 2. vLLM HIGH-THROUGHPUT LLM INFERENCE ENGINE (Tesla T4 GPU + Longhorn Cache)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vllm-models-pvc
|
||||
namespace: ai-stack
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vllm-openai
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: vllm-openai
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: vllm-openai
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: vllm-openai
|
||||
spec:
|
||||
containers:
|
||||
- name: vllm
|
||||
image: vllm/vllm-openai:v0.7.3
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- "--model"
|
||||
- "Qwen/Qwen2.5-7B-Instruct-AWQ"
|
||||
- "--served-model-name"
|
||||
- "Qwen2.5-7B-Instruct"
|
||||
- "--quantization"
|
||||
- "awq_marlin"
|
||||
- "--dtype"
|
||||
- "half"
|
||||
- "--max-model-len"
|
||||
- "16384"
|
||||
- "--gpu-memory-utilization"
|
||||
- "0.90"
|
||||
- "--enable-auto-tool-choice"
|
||||
- "--tool-call-parser"
|
||||
- "hermes"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- "8000"
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: "/root/.cache/huggingface"
|
||||
- name: VLLM_WORKER_MULTIPROC_METHOD
|
||||
value: "spawn"
|
||||
ports:
|
||||
- name: http-openai
|
||||
containerPort: 8000
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: "24Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "24"
|
||||
memory: "48Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: vllm-models
|
||||
mountPath: /root/.cache/huggingface
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: vllm-models
|
||||
persistentVolumeClaim:
|
||||
claimName: vllm-models-pvc
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vllm-openai
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: vllm-openai
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: vllm-openai
|
||||
ports:
|
||||
- name: http-openai
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
nodePort: 30800
|
||||
---
|
||||
# ==============================================================================
|
||||
# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving)
|
||||
# ==============================================================================
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: triton-models-pvc
|
||||
namespace: ai-stack
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: triton-inference-server
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: triton-inference-server
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: triton-inference-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: triton-inference-server
|
||||
spec:
|
||||
initContainers:
|
||||
- name: seed-decision-engine-model
|
||||
image: busybox:1.36
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /models/repository/decision_engine/1
|
||||
cat << 'EOF' > /models/repository/decision_engine/config.pbtxt
|
||||
name: "decision_engine"
|
||||
backend: "python"
|
||||
max_batch_size: 64
|
||||
input [
|
||||
{
|
||||
name: "FEATURES"
|
||||
data_type: TYPE_FP32
|
||||
dims: [ -1 ]
|
||||
}
|
||||
]
|
||||
output [
|
||||
{
|
||||
name: "DECISION_SCORE"
|
||||
data_type: TYPE_FP32
|
||||
dims: [ 1 ]
|
||||
},
|
||||
{
|
||||
name: "ACTION_LABEL"
|
||||
data_type: TYPE_STRING
|
||||
dims: [ 1 ]
|
||||
}
|
||||
]
|
||||
instance_group [
|
||||
{
|
||||
count: 1
|
||||
kind: KIND_GPU
|
||||
}
|
||||
]
|
||||
EOF
|
||||
cat << 'EOF' > /models/repository/decision_engine/1/model.py
|
||||
import numpy as np
|
||||
import triton_python_backend_utils as pb_utils
|
||||
|
||||
class TritonPythonModel:
|
||||
def initialize(self, args):
|
||||
pass
|
||||
|
||||
def execute(self, requests):
|
||||
responses = []
|
||||
for request in requests:
|
||||
features = pb_utils.get_input_tensor_by_name(request, "FEATURES").as_numpy()
|
||||
# Sigmoid risk/decision scoring over feature vector
|
||||
mean_val = float(np.mean(features))
|
||||
score = float(1.0 / (1.0 + np.exp(-mean_val)))
|
||||
label = "APPROVE_EXECUTE" if score >= 0.5 else "HOLD_REVIEW"
|
||||
out_score = pb_utils.Tensor("DECISION_SCORE", np.array([[score]], dtype=np.float32))
|
||||
out_label = pb_utils.Tensor("ACTION_LABEL", np.array([[label.encode("utf-8")]], dtype=np.object_))
|
||||
responses.append(pb_utils.InferenceResponse(output_tensors=[out_score, out_label]))
|
||||
return responses
|
||||
|
||||
def finalize(self):
|
||||
pass
|
||||
EOF
|
||||
echo "Seeded initial decision_engine model into /models/repository/decision_engine"
|
||||
volumeMounts:
|
||||
- name: triton-models
|
||||
mountPath: /models
|
||||
containers:
|
||||
- name: triton
|
||||
image: nvcr.io/nvidia/tritonserver:24.08-py3
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["tritonserver"]
|
||||
args:
|
||||
- "--model-repository=/models/repository"
|
||||
- "--model-control-mode=poll"
|
||||
- "--repository-poll-secs=10"
|
||||
- "--strict-model-config=false"
|
||||
- "--allow-http=true"
|
||||
- "--allow-grpc=true"
|
||||
- "--allow-metrics=true"
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
- name: grpc
|
||||
containerPort: 8001
|
||||
- name: metrics
|
||||
containerPort: 8002
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: "16Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: "32Gi"
|
||||
nvidia.com/gpu: "1"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /v2/health/ready
|
||||
port: 8000
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: triton-models
|
||||
mountPath: /models
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: triton-models
|
||||
persistentVolumeClaim:
|
||||
claimName: triton-models-pvc
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 8Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: triton-inference-server
|
||||
namespace: ai-stack
|
||||
labels:
|
||||
app: triton-inference-server
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: triton-inference-server
|
||||
ports:
|
||||
- name: http
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
nodePort: 30801
|
||||
- name: grpc
|
||||
port: 8001
|
||||
targetPort: 8001
|
||||
nodePort: 30802
|
||||
- name: metrics
|
||||
port: 8002
|
||||
targetPort: 8002
|
||||
nodePort: 30803
|
||||
Reference in New Issue
Block a user