# 自动化脚本：Xid Error 检测 + 节点隔离
import subprocess
import json

def check_gpu_health():
    """通过 DCGM 检查 GPU 健康状态"""
    result = subprocess.run(
        ['dcgmi', 'diag', '-r', '1', '-j'],
        capture_output=True, text=True
    )
    diagnostics = json.loads(result.stdout)

    for gpu_id, diag in diagnostics.items():
        if diag.get('error_count', 0) > 0:
            xid_errors = diag.get('xid_errors', [])
            for err in xid_errors:
                if err['xid'] in [13, 31, 43, 63, 79]:
                    isolate_gpu(gpu_id, err['xid'])

def isolate_gpu(gpu_id, xid):
    """隔离故障 GPU 所在节点"""
    node = get_node_for_gpu(gpu_id)
    # 1. 标记节点不可调度
    subprocess.run(['kubectl', 'cordon', node])
    # 2. 排空 Pod
    subprocess.run(['kubectl', 'drain', node,
                    '--ignore-daemonsets',
                    '--delete-emptydir-data',
                    '--grace-period=120'])
    # 3. 发送告警
    send_alert(f"GPU {gpu_id} on {node} Xid={xid}, node cordoned")

# Node Problem Detector：自动将不健康的 GPU 节点标记为 NotReady
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-problem-detector
  namespace: kube-system
spec:
  template:
    spec:
      containers:
      - name: node-problem-detector
        image: registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.14
        env:
        - name: CUSTOM_PLUGINS
          value: |
            # 自定义 GPU 健康检查插件
            # 当检测到 Xid Error 时，向 K8s 报告 Condition
---
# 自动恢复：GPU 故障后 Pod 自动调度到健康节点
apiVersion: apps/v1
kind: Deployment
spec:
  template:
    spec:
      # 容忍短暂的 NotReady 状态，给节点恢复时间
      tolerations:
      - key: "node.kubernetes.io/not-ready"
        operator: "Exists"
        effect: "NoExecute"
        timeoutSeconds: 60  # 60 秒后 K8s 自动迁移 Pod
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values: ["vllm"]
            topologyKey: "kubernetes.io/hostname"

# Pod 拓扑分布约束：确保跨 AZ 均匀分布
topologySpreadConstraints:
- maxSkew: 1
  topologyKey: topology.kubernetes.io/zone
  whenUnsatisfiable: DoNotSchedule
  labelSelector:
    matchLabels:
      app: vllm

# 通过 ConfigMap 热更新 vLLM 配置
# max-num-seqs: 256 → 64
# 效果：单请求延迟降低，总吞吐下降
# 适用：延迟升高但 GPU 仍有能力

# API Gateway 降级路由
# 正常路由
- match: { prefix: "/v1/chat/completions" }
  route: { cluster: vllm-70b }
# 降级路由（通过 feature flag 切换）
# - match: { prefix: "/v1/chat/completions" }
#   route: { cluster: vllm-7b }

# Token Bucket 限流（API Gateway 层）
# 核心用户：100 req/min
# 普通用户：10 req/min
# 降级期间：核心用户 50 req/min, 普通用户 2 req/min

# 排队响应
{
  "status": "queued",
  "position": 15,
  "estimated_wait_seconds": 45,
  "task_id": "task-xyz789",
  "poll_url": "/v1/tasks/task-xyz789/status"
}

# 回滚命令（K8s 原生支持）
kubectl rollout undo deployment/vllm-llm --to-revision=2
# 回滚到第 2 个版本（即上一版本）

# 回滚验证
# 1. 检查新 Pod 状态: kubectl get pods -l app=vllm
# 2. 验证健康: curl http://vllm-service/health
# 3. 冒烟测试: 发几个标准 prompt 验证输出
# 4. 监控 SLO: 确认延迟恢复正常