AI 应用性能、成本与可观测性教程

const sorted = latencyMs.toSorted((a, b) => a - b);
const percentile = (p: number) =>
  sorted[Math.ceil((p / 100) * sorted.length) - 1];

console.log({
  p50: percentile(50),
  p95: percentile(95),
  p99: percentile(99),
});
// 同时按模型、场景、缓存命中、是否调用工具分组，不能只看总分位数。

C_request =
  T_input  / 1_000_000 * P_input  +
  T_cached / 1_000_000 * P_cached +
  T_output / 1_000_000 * P_output +
  C_retrieval + C_tools + C_retry

// T 是 Token 数，P 是每百万 Token 单价；币种与计费时间必须随报告保存。
// 不能只算主模型：重试、Embedding、重排、OCR 和外部工具都属于业务成本。

// Little's Law：稳定系统中，并发量 L ≈ 到达率 λ × 平均停留时间 W
// 例：40 req/s × 3.0s ≈ 120 个在途请求
expectedInFlight = arrivalRatePerSec * meanE2eSeconds;

// 实测长期明显高于近似值时，检查排队、重试和下游阻塞。
// 平均数只用于容量估计；发布体验仍以 P95/P99 为准。

{
  "trace_id": "tr_7f31", "request_id": "req_0182", "tenant_id_hash": "t_91a",
  "scenario": "refund_after_delivery", "risk_level": "P0",
  "model": "provider/model", "model_version": "2026-07-15",
  "prompt_version": "refund-v12", "schema_version": "answer-v4",
  "index_version": "policy-2026-08-01", "retrieved_doc_ids": ["POL-17#3"],
  "input_tokens": 4200, "cached_tokens": 0, "output_tokens": 850,
  "queue_ms": 42, "retrieval_ms": 181, "ttft_ms": 932, "e2e_ms": 2940,
  "tool_calls": [{"name": "refund_eligibility", "status": "ok", "latency_ms": 87}],
  "estimated_cost": 0.0042, "currency": "CNY",
  "outcome": "human_review", "error_code": null, "retry_count": 0
}

given({ model: "timeout_after_first_token", retrieval: "ok" });
const result = await runCase("REFUND-017");

expect(result.publishable).toBe(false);
expect(result.state).toBe("HUMAN_REVIEW");
expect(result.partialAnswerStoredAsDraft).toBe(true);
expect(result.retryCount).toBeLessThanOrEqual(2);
expect(result.trace.errorCode).toBe("MODEL_TIMEOUT");