From 4d54c144a860d15c4d555f2e9678cd1b2e387944 Mon Sep 17 00:00:00 2001 From: Yuyan Date: Sat, 18 Apr 2026 22:01:34 +0800 Subject: [PATCH] feat: add provider analytics summary --- README.md | 1 + .../src/components/GenerationTrace.vue | 28 +++- admin-frontend/src/views/MyStories.vue | 80 +++++++++++- backend/app/api/stories.py | 14 ++ backend/app/schemas/story_schemas.py | 13 ++ backend/app/services/generation_jobs.py | 90 +++++++++---- backend/tests/test_generation_jobs.py | 120 ++++++++++++++++++ docs/planning/demo-checklist.md | 3 +- docs/planning/demo-validation-log.md | 4 + .../unified-generation-workflow-prd.md | 8 +- docs/technical/generation-job-state.md | 8 +- frontend/src/components/GenerationTrace.vue | 28 +++- frontend/src/types/generation.ts | 11 ++ frontend/src/views/MyStories.vue | 60 ++++++++- scripts/demo_smoke.sh | 5 + 15 files changed, 437 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 391f4fe..8373e80 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ npm run build | GET | `/api/generations/jobs/{job_id}` | 查询生成任务事件流 | | GET | `/api/generations/{story_id}/jobs` | 查询故事生成与重试历史 | | GET | `/api/generations/{story_id}/provider-stats` | 查询 Provider 调用聚合指标 | +| GET | `/api/generations/provider-analytics` | 查询当前用户跨故事 Provider 运营摘要 | | GET | `/api/stories` | 故事列表 | | GET | `/api/stories/{story_id}` | 故事详情 | | DELETE | `/api/stories/{story_id}` | 删除故事 | diff --git a/admin-frontend/src/components/GenerationTrace.vue b/admin-frontend/src/components/GenerationTrace.vue index 4fbb487..1234bcb 100644 --- a/admin-frontend/src/components/GenerationTrace.vue +++ b/admin-frontend/src/components/GenerationTrace.vue @@ -1,5 +1,5 @@ diff --git a/admin-frontend/src/views/MyStories.vue b/admin-frontend/src/views/MyStories.vue index 2b33caa..4efaf6c 100644 --- a/admin-frontend/src/views/MyStories.vue +++ b/admin-frontend/src/views/MyStories.vue @@ -34,8 +34,30 @@ interface StoryItem { last_error: string | null } +interface GenerationProviderStat { + capability: string + adapter: string + call_count: number + success_count: number + failure_count: number + avg_latency_ms: number | null + estimated_cost_usd: number +} + +interface GenerationProviderAnalytics { + total_calls: number + successful_calls: number + failed_calls: number + avg_latency_ms: number | null + estimated_cost_usd: number + job_count: number + story_count: number + by_provider: GenerationProviderStat[] +} + const router = useRouter() const stories = ref([]) +const providerAnalytics = ref(null) const loading = ref(true) const error = ref('') const showCreateModal = ref(false) @@ -45,10 +67,22 @@ const readableCount = computed(() => const attentionCount = computed(() => stories.value.filter((story) => needsGenerationAttention(story.generation_status)).length, ) +const providerSuccessRate = computed(() => { + if (!providerAnalytics.value?.total_calls) return null + return Math.round( + (providerAnalytics.value.successful_calls / providerAnalytics.value.total_calls) * 100, + ) +}) +const topProvider = computed(() => providerAnalytics.value?.by_provider[0] ?? null) async function fetchStories() { try { - stories.value = await api.get('/api/stories') + const [storyList, analytics] = await Promise.all([ + api.get('/api/stories'), + api.get('/api/generations/provider-analytics'), + ]) + stories.value = storyList + providerAnalytics.value = analytics } catch (e) { error.value = e instanceof Error ? e.message : '加载失败' } finally { @@ -81,6 +115,14 @@ function getStoryPath(story: StoryItem) { return story.mode === 'storybook' ? `/storybook/view/${story.id}` : `/story/${story.id}` } +function formatLatency(value?: number | null) { + return typeof value === 'number' ? `${Math.round(value)}ms` : '暂无' +} + +function formatCost(value?: number | null) { + return typeof value === 'number' ? `$${value.toFixed(4)}` : '$0.0000' +} + onMounted(() => { fetchStories() if (router.currentRoute.value.query.openCreate) { @@ -160,6 +202,42 @@ onMounted(() => { + +
+
+

Provider 运营摘要

+

+ 生成、资源补全和失败恢复留下的供应商调用轨迹。 +

+
+
+
+
成功率
+
{{ providerSuccessRate }}%
+
+
+
平均耗时
+
{{ formatLatency(providerAnalytics.avg_latency_ms) }}
+
+
+
预估成本
+
{{ formatCost(providerAnalytics.estimated_cost_usd) }}
+
+
+
调用次数
+
{{ providerAnalytics.total_calls }}
+
+
+
+

+ 当前样本中最前面的能力组合是 {{ topProvider.capability }} / {{ topProvider.adapter }}，成功 {{ topProvider.success_count }} 次，失败 {{ topProvider.failure_count }} 次。 +

+ +
float | None: return None -async def get_story_provider_stats( - db: AsyncSession, - *, - story_id: int, - user_id: str, -) -> dict[str, Any]: - """Aggregate provider call telemetry from all user-owned jobs for one story.""" - - events = ( - await db.execute( - select(GenerationJobEvent) - .join(GenerationJob, GenerationJobEvent.job_id == GenerationJob.id) - .where( - GenerationJob.story_id == story_id, - GenerationJob.user_id == user_id, - GenerationJobEvent.event_type.in_( - ["provider_call_succeeded", "provider_call_failed"] - ), - ) - .order_by(GenerationJobEvent.id) - ) - ).scalars().all() +def _aggregate_provider_events(events: list[GenerationJobEvent]) -> dict[str, Any]: + """Aggregate provider telemetry from provider call events.""" by_key: dict[tuple[str, str], dict[str, Any]] = {} total_latency = 0.0 @@ -363,7 +343,6 @@ async def get_story_provider_stats( ) return { - "story_id": story_id, "total_calls": successful_calls + failed_calls, "successful_calls": successful_calls, "failed_calls": failed_calls, @@ -371,3 +350,66 @@ async def get_story_provider_stats( "estimated_cost_usd": round(total_cost, 6), "by_provider": by_provider, } + + +async def get_story_provider_stats( + db: AsyncSession, + *, + story_id: int, + user_id: str, +) -> dict[str, Any]: + """Aggregate provider call telemetry from all user-owned jobs for one story.""" + + events = ( + await db.execute( + select(GenerationJobEvent) + .join(GenerationJob, GenerationJobEvent.job_id == GenerationJob.id) + .where( + GenerationJob.story_id == story_id, + GenerationJob.user_id == user_id, + GenerationJobEvent.event_type.in_( + ["provider_call_succeeded", "provider_call_failed"] + ), + ) + .order_by(GenerationJobEvent.id) + ) + ).scalars().all() + + return {"story_id": story_id, **_aggregate_provider_events(events)} + + +async def get_user_provider_analytics( + db: AsyncSession, + *, + user_id: str, +) -> dict[str, Any]: + """Aggregate provider telemetry across all stories owned by one user.""" + + events = ( + await db.execute( + select(GenerationJobEvent) + .join(GenerationJob, GenerationJobEvent.job_id == GenerationJob.id) + .where( + GenerationJob.user_id == user_id, + GenerationJobEvent.event_type.in_( + ["provider_call_succeeded", "provider_call_failed"] + ), + ) + .order_by(GenerationJobEvent.id) + ) + ).scalars().all() + + job_count, story_count = ( + await db.execute( + select( + func.count(GenerationJob.id), + func.count(distinct(GenerationJob.story_id)), + ).where(GenerationJob.user_id == user_id) + ) + ).one() + + return { + **_aggregate_provider_events(events), + "job_count": job_count, + "story_count": story_count, + } diff --git a/backend/tests/test_generation_jobs.py b/backend/tests/test_generation_jobs.py index f03133f..367816d 100644 --- a/backend/tests/test_generation_jobs.py +++ b/backend/tests/test_generation_jobs.py @@ -431,3 +431,123 @@ async def test_story_provider_stats_aggregate_job_events( ] finally: app.dependency_overrides.clear() + + +async def test_user_provider_analytics_aggregate_across_stories( + db_session, + auth_token, + degraded_story_with_text, + test_story, +): + async def override_get_db(): + yield db_session + + app.dependency_overrides[get_db] = override_get_db + + image_job = await create_generation_job( + db_session, + user_id=degraded_story_with_text.user_id, + output_mode="asset_retry", + input_type="image", + request_payload={"assets": ["image"]}, + story_id=degraded_story_with_text.id, + ) + await record_generation_event( + db_session, + job=image_job, + story_id=degraded_story_with_text.id, + event_type="provider_call_succeeded", + status="succeeded", + metadata={ + "capability": "image", + "adapter": "demo", + "strategy": "priority", + "latency_ms": 42, + "estimated_cost_usd": 0.01, + }, + ) + await record_generation_event( + db_session, + job=image_job, + story_id=degraded_story_with_text.id, + event_type="provider_call_failed", + status="failed", + metadata={ + "capability": "image", + "adapter": "cqtai", + "strategy": "priority", + "latency_ms": 120, + "error": "timeout", + }, + ) + + audio_job = await create_generation_job( + db_session, + user_id=test_story.user_id, + output_mode="asset_retry", + input_type="audio", + request_payload={"assets": ["audio"]}, + story_id=test_story.id, + ) + await record_generation_event( + db_session, + job=audio_job, + story_id=test_story.id, + event_type="provider_call_succeeded", + status="succeeded", + metadata={ + "capability": "tts", + "adapter": "edge_tts", + "strategy": "priority", + "latency_ms": 18, + "estimated_cost_usd": 0.003, + }, + ) + + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + client.cookies.set("access_token", auth_token) + + response = await client.get("/api/generations/provider-analytics") + + assert response.status_code == 200 + data = response.json() + assert data["job_count"] == 2 + assert data["story_count"] == 2 + assert data["total_calls"] == 3 + assert data["successful_calls"] == 2 + assert data["failed_calls"] == 1 + assert data["avg_latency_ms"] == 60.0 + assert data["estimated_cost_usd"] == 0.013 + assert data["by_provider"] == [ + { + "capability": "image", + "adapter": "cqtai", + "call_count": 1, + "success_count": 0, + "failure_count": 1, + "avg_latency_ms": 120.0, + "estimated_cost_usd": 0.0, + }, + { + "capability": "image", + "adapter": "demo", + "call_count": 1, + "success_count": 1, + "failure_count": 0, + "avg_latency_ms": 42.0, + "estimated_cost_usd": 0.01, + }, + { + "capability": "tts", + "adapter": "edge_tts", + "call_count": 1, + "success_count": 1, + "failure_count": 0, + "avg_latency_ms": 18.0, + "estimated_cost_usd": 0.003, + }, + ] + finally: + app.dependency_overrides.clear() diff --git a/docs/planning/demo-checklist.md b/docs/planning/demo-checklist.md index fc72223..be99cc7 100644 --- a/docs/planning/demo-checklist.md +++ b/docs/planning/demo-checklist.md @@ -56,6 +56,7 @@ SMOKE_AUDIO=1 ./scripts/demo_smoke.sh - [ ] 普通故事 provider stats 返回成功率、耗时和成本字段 - [ ] 普通故事封面 retry 后 `image_status=ready` - [ ] 故事详情页能看到生成轨迹和 Provider 调用结果 +- [ ] 故事库能看到跨故事 Provider 运营摘要 - [ ] `/api/generations` 能生成绘本 - [ ] 绘本生成响应返回 `generation_job_id`，且 story job history 可查询 - [ ] 绘本 provider stats 返回成功率、耗时和成本字段 @@ -123,7 +124,7 @@ DreamWeaver 是面向 3-8 岁亲子场景的个性化 AI 绘本与陪伴式讲 ### 2:20 - 3:00 取舍与下一步 -求职版优先稳定闭环和可解释性，不做支付、多租户和复杂监控。现在 job/event 已能查询 workflow、资产补全、provider 调用轨迹和聚合指标，用户端和管理端也能展示生成轨迹；下一步会迁移到后台 worker 和进度轮询。 +求职版优先稳定闭环和可解释性，不做支付、多租户和复杂监控。现在 job/event 已能查询 workflow、资产补全、provider 调用轨迹和聚合指标，用户端和管理端也能展示生成轨迹与跨故事 Provider 运营摘要；下一步会迁移到后台 worker。 --- diff --git a/docs/planning/demo-validation-log.md b/docs/planning/demo-validation-log.md index f9b8ead..10f2204 100644 --- a/docs/planning/demo-validation-log.md +++ b/docs/planning/demo-validation-log.md @@ -6,6 +6,10 @@ 补充验证： +- 新增跨故事 Provider analytics 后，`backend/.venv/bin/python -m pytest backend/tests -q` 通过，83 个测试通过。 +- 用户端与管理端 `npm run build` 均通过；生成轨迹组件已支持未终止任务自动轮询。 +- `docker compose up -d --build` 已再次用当前代码重建本地演示栈。 +- `./scripts/demo_smoke.sh` 再次通过，并新增断言 `GET /api/generations/provider-analytics` 可以返回跨故事总调用、成功率、任务数、故事数和 Provider 明细。 - 后端新增 `partial_ready`、`text_status` 与迁移 `0012_story_text_status` 后，`backend/.venv/bin/python -m pytest backend/tests -q` 通过，82 个测试通过。 - `backend/.venv/bin/python -m ruff check backend/app backend/tests backend/alembic/versions/0012_add_story_text_status_and_partial_ready.py` 通过。 - 用户端与管理端 `npm run build` 均通过。 diff --git a/docs/product/unified-generation-workflow-prd.md b/docs/product/unified-generation-workflow-prd.md index b25e6ba..67771ec 100644 --- a/docs/product/unified-generation-workflow-prd.md +++ b/docs/product/unified-generation-workflow-prd.md @@ -63,6 +63,8 @@ DreamWeaver 当前同时支持普通故事生成、完整故事生成和绘本 - Provider failover 已记录到 job event，包含 capability、adapter、strategy、latency 和 estimated cost - Provider 调用已可按故事聚合为成功率、平均耗时、预估成本和 adapter 明细 - generation job 响应已提供 `progress_percent`、`progress_label` 和 `is_terminal`，前端可直接用于进度条和轮询 +- 已新增跨故事 Provider 运营摘要 `GET /api/generations/provider-analytics`，故事库可展示总调用、成功率、平均耗时、预估成本和任务/故事覆盖数 +- 用户端与管理端生成轨迹组件会在任务未终止时自动轮询，为后续后台 worker 进度流保留前端形态 - `POST /api/generations` 响应已返回 `generation_job_id`，smoke 脚本会验证 job 查询与 story job history - 用户端与管理端的故事详情页、绘本阅读页已接入生成轨迹，展示生成/重试任务、关键事件、Provider 调用结果和聚合指标 - 故事详情页封面补全已切换到统一资产重试入口 @@ -72,11 +74,11 @@ DreamWeaver 当前同时支持普通故事生成、完整故事生成和绘本 - 普通故事、完整生成、绘本生成已有统一外部入口，内部 workflow 仍可继续减少兼容层分支 - 统一资产重试入口已覆盖普通故事封面、绘本缺失插图和故事音频，后续可继续扩展更细的资产级审计 -- 后台异步 worker 进度流、跨故事 Provider 运营分析和断点续跑仍属于后续生产化增强 +- 后台异步 worker 执行、断点续跑、跨时间窗口筛选和更完整的 Provider 运营分析仍属于后续生产化增强 ### What This Means -这份 PRD 仍然保留目标态设计，但主干能力已经可在当前代码中演示。当前最适合的继续方式，是继续把同步请求迁移到可复用的后台任务与运营分析视角，而不是继续扩大功能范围。 +这份 PRD 仍然保留目标态设计，但主干能力已经可在当前代码中演示。当前最适合的继续方式，是继续把同步请求迁移到后台 worker，并把当前首版运营摘要扩展为可筛选、可对比的分析视角，而不是继续扩大功能范围。 --- @@ -87,7 +89,7 @@ DreamWeaver 当前同时支持普通故事生成、完整故事生成和绘本 DreamWeaver 当前存在以下工作流层面问题： 1. **生成入口已建立，内部路径正在收束** - 当前前端已切到 `/api/generations`，旧的 `/api/stories/generate`、`/api/stories/generate/full`、`/api/storybook/generate` 仍作为兼容入口保留。service 内部已抽取上下文准备、主记录保存、封面补全、绘本插图补全和音频补全 helper，并用 `AssetCompletionResult` 表达资产补全结果。generation job/event 已落库并可查询，Provider 调用轨迹和聚合指标也已进入用户端与管理端展示。下一步重点是为后台异步 worker 与运营成本分析复用这些事件。 + 当前前端已切到 `/api/generations`，旧的 `/api/stories/generate`、`/api/stories/generate/full`、`/api/storybook/generate` 仍作为兼容入口保留。service 内部已抽取上下文准备、主记录保存、封面补全、绘本插图补全和音频补全 helper，并用 `AssetCompletionResult` 表达资产补全结果。generation job/event 已落库并可查询，Provider 调用轨迹、单故事聚合指标和跨故事运营摘要也已进入用户端与管理端展示。下一步重点是为后台异步 worker 复用这些事件。 2. **保存与资产补全过程正在统一** 文本故事和绘本已拥有更清晰的主记录保存 helper；普通故事封面、绘本缺失插图、故事音频生成/缓存已共用各自的 asset completion helper。服务层已经能表达资产任务结果，并会把统一入口、资产重试、绘本逐页插图和音频生成的关键节点写入 job event。 diff --git a/docs/technical/generation-job-state.md b/docs/technical/generation-job-state.md index 82b7f52..62cd783 100644 --- a/docs/technical/generation-job-state.md +++ b/docs/technical/generation-job-state.md @@ -19,8 +19,9 @@ - `GET /api/generations/jobs/{job_id}`：查询单次生成/补全任务及其事件流。 - `GET /api/generations/{story_id}/jobs`：查询某个故事或绘本的生成与重试历史。 - `GET /api/generations/{story_id}/provider-stats`：按故事聚合 Provider 调用成功率、平均耗时、预估成本和 adapter 明细。 +- `GET /api/generations/provider-analytics`：按当前用户聚合跨故事 Provider 调用、任务数、故事数、成功率、平均耗时和预估成本。 -job 响应会返回 `progress_percent`、`progress_label` 和 `is_terminal`，用户端与管理端已经消费这些查询入口，在故事详情页和绘本阅读页展示最近任务、任务历史、事件时间线、进度条和 Provider 聚合指标。 +job 响应会返回 `progress_percent`、`progress_label` 和 `is_terminal`，用户端与管理端已经消费这些查询入口，在故事详情页和绘本阅读页展示最近任务、任务历史、事件时间线、进度条和 Provider 聚合指标；当任务未终止时，前端会自动轮询，为后台 worker 进度流预留体验形态。 ## 现有状态模型 @@ -50,9 +51,8 @@ job 响应会返回 `progress_percent`、`progress_label` 和 `is_terminal`， 当前已有两层记录，未来可以继续扩展字段和事件颗粒度： -- 将 job/event 查询继续接入真正异步生成时的进度条。 -- 将当前按故事聚合的 provider 指标扩展为跨用户、跨时间窗口的运营分析。 -- 将当前同步生成请求迁移到后台 worker 后，复用现有 job 查询接口做轮询进度。 +- 将同步生成请求迁移到真正异步 worker 后，继续复用现有 job 查询和前端轮询进度条。 +- 将当前跨故事 provider 指标扩展为跨时间窗口、跨用户和失败原因维度的运营分析。 ## 面试表达 diff --git a/frontend/src/components/GenerationTrace.vue b/frontend/src/components/GenerationTrace.vue index 233ab91..a4e87c9 100644 --- a/frontend/src/components/GenerationTrace.vue +++ b/frontend/src/components/GenerationTrace.vue @@ -1,5 +1,5 @@ diff --git a/frontend/src/types/generation.ts b/frontend/src/types/generation.ts index c0b1d68..9e6a9f3 100644 --- a/frontend/src/types/generation.ts +++ b/frontend/src/types/generation.ts @@ -49,3 +49,14 @@ export interface GenerationProviderStats { estimated_cost_usd: number by_provider: GenerationProviderStat[] } + +export interface GenerationProviderAnalytics { + total_calls: number + successful_calls: number + failed_calls: number + avg_latency_ms: number | null + estimated_cost_usd: number + job_count: number + story_count: number + by_provider: GenerationProviderStat[] +} diff --git a/frontend/src/views/MyStories.vue b/frontend/src/views/MyStories.vue index 691a40f..77847ba 100644 --- a/frontend/src/views/MyStories.vue +++ b/frontend/src/views/MyStories.vue @@ -7,6 +7,7 @@ import BaseButton from '../components/ui/BaseButton.vue' import BaseCard from '../components/ui/BaseCard.vue' import EmptyState from '../components/ui/EmptyState.vue' import LoadingSpinner from '../components/ui/LoadingSpinner.vue' +import type { GenerationProviderAnalytics } from '../types/generation' import { getAssetStatusMeta, getGenerationStatusMeta, @@ -37,6 +38,7 @@ interface StoryItem { const router = useRouter() const stories = ref([]) +const providerAnalytics = ref(null) const loading = ref(true) const error = ref('') const showCreateModal = ref(false) @@ -48,10 +50,22 @@ const readableCount = computed(() => const attentionCount = computed(() => stories.value.filter((story) => needsGenerationAttention(story.generation_status)).length, ) +const providerSuccessRate = computed(() => { + if (!providerAnalytics.value?.total_calls) return null + return Math.round( + (providerAnalytics.value.successful_calls / providerAnalytics.value.total_calls) * 100, + ) +}) +const topProvider = computed(() => providerAnalytics.value?.by_provider[0] ?? null) async function fetchStories() { try { - stories.value = await api.get('/api/stories') + const [storyList, analytics] = await Promise.all([ + api.get('/api/stories'), + api.get('/api/generations/provider-analytics'), + ]) + stories.value = storyList + providerAnalytics.value = analytics } catch (e) { error.value = e instanceof Error ? e.message : '加载失败' } finally { @@ -84,6 +98,14 @@ function getStoryLink(story: StoryItem) { return story.mode === 'storybook' ? `/storybook/view/${story.id}` : `/story/${story.id}` } +function formatLatency(value?: number | null) { + return typeof value === 'number' ? `${Math.round(value)}ms` : '暂无' +} + +function formatCost(value?: number | null) { + return typeof value === 'number' ? `$${value.toFixed(4)}` : '$0.0000' +} + onMounted(() => { void fetchStories() @@ -158,6 +180,42 @@ onMounted(() => {
+ +
+
+

Provider 运营摘要

+

+ 最近生成和资源补全留下的供应商调用轨迹。 +

+
+
+
+
成功率
+
{{ providerSuccessRate }}%
+
+
+
平均耗时
+
{{ formatLatency(providerAnalytics.avg_latency_ms) }}
+
+
+
预估成本
+
{{ formatCost(providerAnalytics.estimated_cost_usd) }}
+
+
+
调用次数
+
{{ providerAnalytics.total_calls }}
+
+
+
+

+ 当前样本中最前面的能力组合是 {{ topProvider.capability }} / {{ topProvider.adapter }}，成功 {{ topProvider.success_count }} 次，失败 {{ topProvider.failure_count }} 次。 +

+ +
= 2 and (map(.id) | index("'"$storyboo echo "$story_jobs_json" | jq '[.[] | {id,output_mode,status,current_step}]' echo "$storybook_jobs_json" | jq '[.[] | {id,output_mode,status,current_step}]' +say "Checking cross-story provider analytics" +provider_analytics_json="$(get_json "$APP_URL/api/generations/provider-analytics")" +assert_jq "$provider_analytics_json" '.total_calls >= 2 and .successful_calls >= 2 and .job_count >= 4 and .story_count >= 2 and (.by_provider | length) >= 1' "provider analytics should summarize calls across generated stories" +echo "$provider_analytics_json" | jq '{total_calls,successful_calls,failed_calls,job_count,story_count,avg_latency_ms,estimated_cost_usd}' + say "Checking story list" list_json="$(get_json "$APP_URL/api/stories?limit=5")" assert_jq "$list_json" "map(.id) | index($story_id) != null" "story list should include generated story"