497 lines
13 KiB
YAML
497 lines
13 KiB
YAML
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: ai-stack
|
|
labels:
|
|
app.kubernetes.io/part-of: spacedock-ai-stack
|
|
---
|
|
# ==============================================================================
|
|
# 1. QDRANT VECTOR DATABASE (Backed by 46.4 TiB Longhorn Flash Pool)
|
|
# ==============================================================================
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: qdrant-storage-pvc
|
|
namespace: ai-stack
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 100Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: qdrant
|
|
namespace: ai-stack
|
|
labels:
|
|
app: qdrant
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: qdrant
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: qdrant
|
|
spec:
|
|
containers:
|
|
- name: qdrant
|
|
image: qdrant/qdrant:latest
|
|
imagePullPolicy: IfNotPresent
|
|
ports:
|
|
- name: http
|
|
containerPort: 6333
|
|
- name: grpc
|
|
containerPort: 6334
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "4Gi"
|
|
limits:
|
|
cpu: "8"
|
|
memory: "16Gi"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /readyz
|
|
port: 6333
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
volumeMounts:
|
|
- name: qdrant-storage
|
|
mountPath: /qdrant/storage
|
|
volumes:
|
|
- name: qdrant-storage
|
|
persistentVolumeClaim:
|
|
claimName: qdrant-storage-pvc
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: qdrant
|
|
namespace: ai-stack
|
|
labels:
|
|
app: qdrant
|
|
spec:
|
|
type: NodePort
|
|
selector:
|
|
app: qdrant
|
|
ports:
|
|
- name: http
|
|
port: 6333
|
|
targetPort: 6333
|
|
nodePort: 30633
|
|
- name: grpc
|
|
port: 6334
|
|
targetPort: 6334
|
|
nodePort: 30634
|
|
---
|
|
# ==============================================================================
|
|
# 2. vLLM HIGH-THROUGHPUT LLM INFERENCE ENGINE (Tesla T4 GPU + Longhorn Cache)
|
|
# ==============================================================================
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: vllm-models-pvc
|
|
namespace: ai-stack
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 200Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm-openai
|
|
namespace: ai-stack
|
|
labels:
|
|
app: vllm-openai
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: vllm-openai
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm-openai
|
|
spec:
|
|
containers:
|
|
- name: vllm
|
|
image: vllm/vllm-openai:v0.7.3
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- "--model"
|
|
- "Qwen/Qwen2.5-7B-Instruct-AWQ"
|
|
- "--served-model-name"
|
|
- "Qwen2.5-7B-Instruct"
|
|
- "--quantization"
|
|
- "awq"
|
|
- "--dtype"
|
|
- "half"
|
|
- "--max-model-len"
|
|
- "16384"
|
|
- "--gpu-memory-utilization"
|
|
- "0.90"
|
|
- "--enable-auto-tool-choice"
|
|
- "--tool-call-parser"
|
|
- "hermes"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8000"
|
|
env:
|
|
- name: HF_HOME
|
|
value: "/root/.cache/huggingface"
|
|
- name: VLLM_WORKER_MULTIPROC_METHOD
|
|
value: "spawn"
|
|
ports:
|
|
- name: http-openai
|
|
containerPort: 8000
|
|
resources:
|
|
requests:
|
|
cpu: "8"
|
|
memory: "24Gi"
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "24"
|
|
memory: "48Gi"
|
|
nvidia.com/gpu: "1"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
volumeMounts:
|
|
- name: vllm-models
|
|
mountPath: /root/.cache/huggingface
|
|
- name: dshm
|
|
mountPath: /dev/shm
|
|
volumes:
|
|
- name: vllm-models
|
|
persistentVolumeClaim:
|
|
claimName: vllm-models-pvc
|
|
- name: dshm
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 16Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: vllm-openai
|
|
namespace: ai-stack
|
|
labels:
|
|
app: vllm-openai
|
|
spec:
|
|
type: NodePort
|
|
selector:
|
|
app: vllm-openai
|
|
ports:
|
|
- name: http-openai
|
|
port: 8000
|
|
targetPort: 8000
|
|
nodePort: 30800
|
|
---
|
|
# ==============================================================================
|
|
# 2B. vLLM GEMMA 4 12B INFERENCE ENGINE (snaisvr44hrvstr02 Tesla T4 GPU)
|
|
# ==============================================================================
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: vllm-gemma4-models-pvc
|
|
namespace: ai-stack
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 200Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm-gemma4
|
|
namespace: ai-stack
|
|
labels:
|
|
app: vllm-gemma4
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: vllm-gemma4
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm-gemma4
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/hostname: snaisvr44hrvstr02
|
|
containers:
|
|
- name: vllm
|
|
image: vllm/vllm-openai:v0.32.0-cu129
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
rm -rf /usr/local/lib/python3.12/dist-packages/torchcodec*
|
|
python3 -c '
|
|
p = "/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/triton_unified_attention.py"
|
|
s = open(p).read()
|
|
s = s.replace("return 32", "return 16")
|
|
s = s.replace("launch_num_stages: int | None = None", "launch_num_stages: int | None = 1")
|
|
open(p, "w").write(s)
|
|
'
|
|
exec vllm serve google/gemma-4-12B-it-qat-w4a16-ct \
|
|
--served-model-name Gemma-4-12B-it \
|
|
--dtype half \
|
|
--max-model-len 8192 \
|
|
--gpu-memory-utilization 0.92 \
|
|
--enforce-eager \
|
|
--enable-auto-tool-choice \
|
|
--tool-call-parser gemma4 \
|
|
--reasoning-parser gemma4 \
|
|
--host 0.0.0.0 \
|
|
--port 8000
|
|
env:
|
|
- name: HF_HOME
|
|
value: "/root/.cache/huggingface"
|
|
- name: VLLM_WORKER_MULTIPROC_METHOD
|
|
value: "spawn"
|
|
ports:
|
|
- name: http-openai
|
|
containerPort: 8000
|
|
resources:
|
|
requests:
|
|
cpu: "8"
|
|
memory: "24Gi"
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "24"
|
|
memory: "48Gi"
|
|
nvidia.com/gpu: "1"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
volumeMounts:
|
|
- name: vllm-gemma4-models
|
|
mountPath: /root/.cache/huggingface
|
|
- name: dshm
|
|
mountPath: /dev/shm
|
|
volumes:
|
|
- name: vllm-gemma4-models
|
|
persistentVolumeClaim:
|
|
claimName: vllm-gemma4-models-pvc
|
|
- name: dshm
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 16Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: vllm-gemma4
|
|
namespace: ai-stack
|
|
labels:
|
|
app: vllm-gemma4
|
|
spec:
|
|
type: NodePort
|
|
selector:
|
|
app: vllm-gemma4
|
|
ports:
|
|
- name: http-openai
|
|
port: 8000
|
|
targetPort: 8000
|
|
nodePort: 30804
|
|
---
|
|
# ==============================================================================
|
|
# 3. NVIDIA TRITON INFERENCE SERVER (Decision Engine & Multi-Model ML Serving)
|
|
# ==============================================================================
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: triton-models-pvc
|
|
namespace: ai-stack
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 200Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: triton-inference-server
|
|
namespace: ai-stack
|
|
labels:
|
|
app: triton-inference-server
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: triton-inference-server
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: triton-inference-server
|
|
spec:
|
|
initContainers:
|
|
- name: seed-decision-engine-model
|
|
image: busybox:1.36
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
mkdir -p /models/repository/decision_engine/1
|
|
cat << 'EOF' > /models/repository/decision_engine/config.pbtxt
|
|
name: "decision_engine"
|
|
backend: "python"
|
|
max_batch_size: 64
|
|
input [
|
|
{
|
|
name: "FEATURES"
|
|
data_type: TYPE_FP32
|
|
dims: [ -1 ]
|
|
}
|
|
]
|
|
output [
|
|
{
|
|
name: "DECISION_SCORE"
|
|
data_type: TYPE_FP32
|
|
dims: [ 1 ]
|
|
},
|
|
{
|
|
name: "ACTION_LABEL"
|
|
data_type: TYPE_STRING
|
|
dims: [ 1 ]
|
|
}
|
|
]
|
|
instance_group [
|
|
{
|
|
count: 1
|
|
kind: KIND_GPU
|
|
}
|
|
]
|
|
EOF
|
|
cat << 'EOF' > /models/repository/decision_engine/1/model.py
|
|
import numpy as np
|
|
import triton_python_backend_utils as pb_utils
|
|
|
|
class TritonPythonModel:
|
|
def initialize(self, args):
|
|
pass
|
|
|
|
def execute(self, requests):
|
|
responses = []
|
|
for request in requests:
|
|
features = pb_utils.get_input_tensor_by_name(request, "FEATURES").as_numpy()
|
|
# Sigmoid risk/decision scoring over feature vector
|
|
mean_val = float(np.mean(features))
|
|
score = float(1.0 / (1.0 + np.exp(-mean_val)))
|
|
label = "APPROVE_EXECUTE" if score >= 0.5 else "HOLD_REVIEW"
|
|
out_score = pb_utils.Tensor("DECISION_SCORE", np.array([[score]], dtype=np.float32))
|
|
out_label = pb_utils.Tensor("ACTION_LABEL", np.array([[label.encode("utf-8")]], dtype=np.object_))
|
|
responses.append(pb_utils.InferenceResponse(output_tensors=[out_score, out_label]))
|
|
return responses
|
|
|
|
def finalize(self):
|
|
pass
|
|
EOF
|
|
echo "Seeded initial decision_engine model into /models/repository/decision_engine"
|
|
volumeMounts:
|
|
- name: triton-models
|
|
mountPath: /models
|
|
containers:
|
|
- name: triton
|
|
image: nvcr.io/nvidia/tritonserver:24.08-py3
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["tritonserver"]
|
|
args:
|
|
- "--model-repository=/models/repository"
|
|
- "--model-control-mode=poll"
|
|
- "--repository-poll-secs=10"
|
|
- "--strict-model-config=false"
|
|
- "--allow-http=true"
|
|
- "--allow-grpc=true"
|
|
- "--allow-metrics=true"
|
|
ports:
|
|
- name: http
|
|
containerPort: 8000
|
|
- name: grpc
|
|
containerPort: 8001
|
|
- name: metrics
|
|
containerPort: 8002
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: "16Gi"
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "16"
|
|
memory: "32Gi"
|
|
nvidia.com/gpu: "1"
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /v2/health/ready
|
|
port: 8000
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
volumeMounts:
|
|
- name: triton-models
|
|
mountPath: /models
|
|
- name: dshm
|
|
mountPath: /dev/shm
|
|
volumes:
|
|
- name: triton-models
|
|
persistentVolumeClaim:
|
|
claimName: triton-models-pvc
|
|
- name: dshm
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 8Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: triton-inference-server
|
|
namespace: ai-stack
|
|
labels:
|
|
app: triton-inference-server
|
|
spec:
|
|
type: NodePort
|
|
selector:
|
|
app: triton-inference-server
|
|
ports:
|
|
- name: http
|
|
port: 8000
|
|
targetPort: 8000
|
|
nodePort: 30801
|
|
- name: grpc
|
|
port: 8001
|
|
targetPort: 8001
|
|
nodePort: 30802
|
|
- name: metrics
|
|
port: 8002
|
|
targetPort: 8002
|
|
nodePort: 30803
|