# 为每个租户创建独立 Namespace
apiVersion: v1
kind: Namespace
metadata:
  name: tenant-customer-service
  labels:
    tenant: customer-service
    team: support

---
# ResourceQuota：限制租户可使用的资源总量
apiVersion: v1
kind: ResourceQuota
metadata:
  name: gpu-quota
  namespace: tenant-customer-service
spec:
  hard:
    requests.nvidia.com/gpu: "4"    # 最多申请 4 张 GPU
    limits.nvidia.com/gpu: "4"
    requests.memory: "256Gi"
    limits.memory: "256Gi"
    requests.cpu: "32"
    limits.cpu: "32"
    pods: "20"                       # 最多 20 个 Pod

---
# LimitRange：限制单个容器的资源范围
apiVersion: v1
kind: LimitRange
metadata:
  name: gpu-limit-range
  namespace: tenant-customer-service
spec:
  limits:
  - type: Container
    default:                        # 默认限制
      nvidia.com/gpu: "1"
      memory: "64Gi"
      cpu: "8"
    defaultRequest:
      nvidia.com/gpu: "1"
      memory: "32Gi"
      cpu: "4"
    max:                            # 单个容器不能超过
      nvidia.com/gpu: "4"
      memory: "128Gi"
    min:
      nvidia.com/gpu: "1"

# 1. 启用 MIG 模式
nvidia-smi -mig 1

# 2. 查看支持的 MIG Profile
nvidia-smi mig -lgip

# 3. 创建 MIG 实例（例如 2 个 3g.20gb 实例）
nvidia-smi mig -cgi 3g.20gb -C 2

# 4. 创建 GPU Instance 后，再创建 Compute Instance
nvidia-smi mig -lgi   # 查看 GPU Instance
nvidia-smi mig -cci -gi <GI_ID>  # 创建 Compute Instance

# 5. 验证 MIG 实例
nvidia-smi
# +-----------------------------------------------------------------------------+
# | MIG devices:                                                                |
# |  GPU  GI  CI  Memory    GPU  SM  DEC  ENC  OFA  ECC  Compute Mode           |
# |   0    0   0   20.0GiB  40   28   2    1    1   On   Default               |
# |   0    1   0   20.0GiB  40   28   2    1    1   On   Default               |
# +-----------------------------------------------------------------------------+

# 安装 NVIDIA GPU Operator（自动管理 MIG）
helm install gpu-operator nvidia/gpu-operator \
  --namespace gpu-operator \
  --create-namespace \
  --set mig.strategy=single  # single 或 mixed

# Pod 申请 MIG 实例
apiVersion: v1
kind: Pod
metadata:
  name: tenant-a-llm
  namespace: tenant-customer-service
spec:
  containers:
  - name: vllm
    image: vllm/vllm-openai:latest
    resources:
      limits:
        # 申请一个 3g.20gb 的 MIG 实例
        nvidia.com/mig-3g.20gb: 1
      requests:
        nvidia.com/mig-3g.20gb: 1

# GPU Time-Slicing：多租户共享一张 GPU，时间片轮转
# 通过 NVIDIA GPU Operator 配置
apiVersion: v1
kind: ConfigMap
metadata:
  name: time-slicing-config
  namespace: gpu-operator
data:
  any: |
    version: v1
    flags:
      migStrategy: none
    sharing:
      timeSlicing:
        resources:
        - name: nvidia.com/gpu
          replicas: 4     # 一张 GPU 虚拟化为 4 个共享设备
---
# 4 个 Pod 可以同时申请 nvidia.com/gpu: 1
# 但实际上共享一张物理卡，时间片轮转
# 适合：开发、测试、低负载推理

# PriorityClass 定义
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: tenant-critical
value: 1000000         # 最高优先级
globalDefault: false
description: "核心业务租户，不可抢占"

---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: tenant-normal
value: 500000          # 中等优先级
globalDefault: true
description: "普通业务租户"

---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: tenant-batch
value: 100000          # 低优先级
globalDefault: false
description: "批量任务/离线推理，可被抢占"

# 高优先级 Pod（可抢占其他 Pod）
spec:
  priorityClassName: tenant-critical
  preemptionPolicy: PreemptLowerPriority  # 默认值

# 低优先级 Pod（标记为可被抢占）
spec:
  priorityClassName: tenant-batch
  # 当高优先级 Pod 需要资源时，此 Pod 会被优雅驱逐
  # terminationGracePeriodSeconds 控制等待时间

# vLLM Prometheus 指标
# 每个租户的请求统计
vllm_request_total{tenant="customer-service",model="qwen2.5-7b"}
vllm_request_success_total{tenant="customer-service"}
vllm_prompt_tokens_total{tenant="customer-service"}
vllm_generation_tokens_total{tenant="customer-service"}

# 计费 SQL 示例（从 Prometheus 查询）
# 租户 A 本月费用 = Input Tokens × $0.001/1K + Output Tokens × $0.003/1K
# + GPU 占用时长(h) × $5/h

# NetworkPolicy：租户间网络隔离
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: tenant-isolation
  namespace: tenant-customer-service
spec:
  podSelector:
    matchLabels:
      app: vllm
  policyTypes:
  - Ingress
  - Egress
  ingress:
  # 只允许 API Gateway 访问
  - from:
    - namespaceSelector:
        matchLabels:
          name: api-gateway
    ports:
    - protocol: TCP
      port: 8000
  egress:
  # 只允许访问模型存储和监控
  - to:
    - namespaceSelector:
        matchLabels:
          name: monitoring
    - namespaceSelector:
        matchLabels:
          name: model-registry

# 弹性配额（ClusterResourceQuota 或 Kueue）
# 保证配额 + 借用额度
apiVersion: kueue.x-k8s.io/v1beta1
kind: ResourceFlavor
metadata:
  name: a100-80gb
---
apiVersion: kueue.x-k8s.io/v1beta1
kind: ClusterQueue
metadata:
  name: tenant-customer-service
spec:
  namespaceSelector:
    matchLabels:
      tenant: customer-service
  resourceGroups:
  - coveredResources: ["nvidia.com/gpu"]
    flavors:
    - name: a100-80gb
      resources:
      - name: "nvidia.com/gpu"
        nominalQuota: 2       # 保证 2 张 GPU
        borrowingLimit: 2     # 最多借用 2 张
        lendingLimit: 0       # 不借出给别人