Expand generation harness observability

backend/tests/test_generation_jobs.py

@@ -123,14 +123,19 @@ async def test_unified_generation_is_queued_then_worker_persists_story_and_event
                 assert [event.event_type for event in events] == [
                     "request_accepted",
                     "worker_started",
+                    "workflow_planned",
                     "context_prepared",
+                    "evaluation_completed",
                     "narrative_generated",
                     "story_saved",
                     "generation_completed",
                 ]
-                assert events[2].event_metadata["has_memory_context"] is False
-                assert events[3].event_metadata["title"] == "小兔子的冒险"
-                assert events[4].story_id == job.story_id
+                assert events[2].event_metadata["plan"]["mode"] == "story"
+                assert events[3].event_metadata["has_memory_context"] is False
+                assert events[4].event_metadata["passed"] is True
+                assert events[4].event_metadata["overall_score"] >= 0.7
+                assert events[5].event_metadata["title"] == "小兔子的冒险"
+                assert events[6].story_id == job.story_id
 
                 detail_response = await client.get(f"/api/generations/jobs/{job.id}")
                 assert detail_response.status_code == 200
@@ -143,11 +148,16 @@ async def test_unified_generation_is_queued_then_worker_persists_story_and_event
                 assert [event["event_type"] for event in detail["events"]] == [
                     "request_accepted",
                     "worker_started",
+                    "workflow_planned",
                     "context_prepared",
                     "narrative_generated",
                     "story_saved",
                     "generation_completed",
                 ]
+                assert all(
+                    event["event_type"] != "evaluation_completed"
+                    for event in detail["events"]
+                )
 
                 story_response = await client.get(f"/api/generations/{job.story_id}")
                 assert story_response.status_code == 200
@@ -161,6 +171,13 @@ async def test_unified_generation_is_queued_then_worker_persists_story_and_event
                 assert [item["id"] for item in job_list] == [job.id]
                 assert job_list[0]["progress_percent"] == 100
                 assert job_list[0]["is_terminal"] is True
+
+                trace_response = await client.get(
+                    f"/api/generations/{job.story_id}/trace-summary"
+                )
+                assert trace_response.status_code == 200
+                trace = trace_response.json()
+                assert "evaluation" not in trace
     finally:
         app.dependency_overrides.clear()
 
@@ -220,13 +237,88 @@ async def test_generation_worker_records_quality_gate_failure_without_persisting
     assert [event.event_type for event in events] == [
         "request_accepted",
         "worker_started",
+        "workflow_planned",
         "context_prepared",
         "quality_gate_failed",
+        "evaluation_completed",
         "generation_failed",
     ]
-    quality_event = events[3]
+    quality_event = events[4]
     assert quality_event.event_metadata["step"] == "narrative_generation"
     assert quality_event.event_metadata["issues"][0]["code"] == "missing_story_text"
+    evaluation_event = events[5]
+    assert evaluation_event.event_metadata["step"] == "evaluation"
+    assert evaluation_event.event_metadata["passed"] is False
+    assert evaluation_event.event_metadata["blocking"] is True
+
+
+async def test_story_with_images_worker_records_plan_before_assets(
+    db_session,
+    test_user,
+    mock_text_provider,
+    mock_image_provider,
+):
+    job = await create_generation_job(
+        db_session,
+        user_id=test_user.id,
+        output_mode="story",
+        input_type="keywords",
+        request_payload={
+            "output_mode": "story",
+            "type": "keywords",
+            "data": "小兔子, 森林",
+            "generate_images": True,
+        },
+    )
+
+    await run_generation_job_service(job.id, db_session)
+
+    refreshed_job = (
+        await db_session.execute(select(GenerationJob).where(GenerationJob.id == job.id))
+    ).scalar_one()
+    assert refreshed_job.story_id is not None
+    assert refreshed_job.status == "completed"
+    assert refreshed_job.current_step == "generation_completed"
+    assert refreshed_job.result_snapshot["image_status"] == "ready"
+
+    events = (
+        await db_session.execute(
+            select(GenerationJobEvent)
+            .where(GenerationJobEvent.job_id == job.id)
+            .order_by(GenerationJobEvent.id)
+        )
+    ).scalars().all()
+    assert [event.event_type for event in events] == [
+        "request_accepted",
+        "worker_started",
+        "workflow_planned",
+        "context_prepared",
+        "evaluation_completed",
+        "narrative_generated",
+        "story_saved",
+        "cover_image_started",
+        "cover_image_succeeded",
+        "generation_completed",
+    ]
+
+    plan = events[2].event_metadata["plan"]
+    assert plan["mode"] == "story_with_assets"
+    assert [task["key"] for task in plan["tasks"]] == [
+        "prepare_context",
+        "generate_narrative",
+        "evaluate_narrative",
+        "persist_story",
+        "generate_cover_image",
+        "queue_postprocessing",
+        "complete_generation",
+    ]
+    cover_task = next(task for task in plan["tasks"] if task["key"] == "generate_cover_image")
+    assert cover_task["required"] is False
+    assert cover_task["recoverable"] is True
+    assert events[4].event_metadata["passed"] is True
+    assert events[8].event_metadata["asset"] == "cover_image"
+    mock_text_provider.assert_called_once()
+    mock_image_provider.assert_called_once()
 
 
 async def test_asset_retry_records_job_events_and_updates_retryable_assets(
@@ -279,12 +371,30 @@ async def test_asset_retry_records_job_events_and_updates_retryable_assets(
             ).scalars().all()
             assert [event.event_type for event in events] == [
                 "request_accepted",
+                "workflow_planned",
                 "asset_retry_started",
                 "cover_image_started",
                 "cover_image_succeeded",
+                "executor_completed",
                 "asset_retry_completed",
             ]
-            assert events[3].event_metadata["asset"] == "cover_image"
+            plan = events[1].event_metadata["plan"]
+            assert plan["mode"] == "asset_retry"
+            assert [task["key"] for task in plan["tasks"]] == [
+                "start_asset_retry",
+                "complete_image_asset",
+                "complete_asset_retry",
+            ]
+            image_task = next(
+                task for task in plan["tasks"] if task["key"] == "complete_image_asset"
+            )
+            assert image_task["required"] is False
+            assert image_task["recoverable"] is True
+            assert events[4].event_metadata["asset"] == "cover_image"
+            assert events[5].event_metadata["plan_mode"] == "asset_retry"
+            assert events[5].event_metadata["executed_task_keys"] == [
+                "complete_image_asset"
+            ]
     finally:
         app.dependency_overrides.clear()
 
@@ -365,10 +475,110 @@ async def test_asset_generation_job_worker_completes_cover_image(
     assert [event.event_type for event in events] == [
         "request_accepted",
         "worker_started",
+        "workflow_planned",
         "cover_image_started",
         "cover_image_succeeded",
+        "executor_completed",
         "asset_generation_completed",
     ]
+    plan = events[2].event_metadata["plan"]
+    assert plan["mode"] == "asset_generation"
+    assert [task["key"] for task in plan["tasks"]] == [
+        "start_asset_generation",
+        "complete_image_asset",
+        "complete_asset_generation",
+    ]
+    image_task = next(
+        task for task in plan["tasks"] if task["key"] == "complete_image_asset"
+    )
+    assert image_task["required"] is False
+    assert image_task["recoverable"] is True
+    executor_event = events[5]
+    assert executor_event.event_metadata["plan_mode"] == "asset_generation"
+    assert executor_event.event_metadata["executed_task_keys"] == [
+        "complete_image_asset"
+    ]
+    assert executor_event.event_metadata["ignored_task_keys"] == [
+        "start_asset_generation",
+        "complete_asset_generation",
+    ]
+    assert executor_event.event_metadata["result_assets"] == ["cover_image"]
+
+
+async def test_asset_generation_job_worker_executes_assets_in_plan_order(
+    db_session,
+    test_story,
+    mock_tts_provider,
+):
+    job = await create_generation_job(
+        db_session,
+        user_id=test_story.user_id,
+        output_mode="asset_generation",
+        input_type="audio,image",
+        request_payload={"story_id": test_story.id, "assets": ["audio", "image"]},
+        story_id=test_story.id,
+    )
+
+    with patch(
+        "app.services.story_service.generate_image",
+        new_callable=AsyncMock,
+    ) as mock_generate_image:
+        mock_generate_image.return_value = "https://example.com/plan-cover.png"
+
+        await run_generation_job_service(job.id, db_session)
+
+    refreshed_job = (
+        await db_session.execute(select(GenerationJob).where(GenerationJob.id == job.id))
+    ).scalar_one()
+    assert refreshed_job.status == "completed"
+    assert refreshed_job.current_step == "asset_generation_completed"
+    assert refreshed_job.result_snapshot["image_status"] == "ready"
+    assert refreshed_job.result_snapshot["audio_status"] == "ready"
+
+    story = (
+        await db_session.execute(
+            select(Story).where(Story.id == test_story.id)
+        )
+    ).scalar_one()
+    assert story.image_url == "https://example.com/plan-cover.png"
+    assert story.audio_status == "ready"
+    assert story.audio_path is not None
+
+    events = (
+        await db_session.execute(
+            select(GenerationJobEvent)
+            .where(GenerationJobEvent.job_id == job.id)
+            .order_by(GenerationJobEvent.id)
+        )
+    ).scalars().all()
+    assert [event.event_type for event in events] == [
+        "request_accepted",
+        "worker_started",
+        "workflow_planned",
+        "audio_started",
+        "audio_succeeded",
+        "cover_image_started",
+        "cover_image_succeeded",
+        "executor_completed",
+        "asset_generation_completed",
+    ]
+    plan = events[2].event_metadata["plan"]
+    assert plan["mode"] == "asset_generation"
+    assert [task["key"] for task in plan["tasks"]] == [
+        "start_asset_generation",
+        "complete_audio_asset",
+        "complete_image_asset",
+        "complete_asset_generation",
+    ]
+    assert events[4].event_metadata["asset"] == "audio"
+    assert events[6].event_metadata["asset"] == "cover_image"
+    assert events[7].event_metadata["executed_task_keys"] == [
+        "complete_audio_asset",
+        "complete_image_asset",
+    ]
+    assert events[7].event_metadata["result_assets"] == ["audio", "cover_image"]
+    mock_tts_provider.assert_awaited_once()
+    mock_generate_image.assert_awaited_once()
 
 
 async def test_cancel_queued_asset_generation_job_marks_it_canceled(
@@ -538,7 +748,9 @@ async def test_storybook_generation_is_queued_then_worker_records_page_image_eve
                     assert [event.event_type for event in events] == [
                         "request_accepted",
                         "worker_started",
+                        "workflow_planned",
                         "context_prepared",
+                        "evaluation_completed",
                         "narrative_generated",
                         "storybook_images_started",
                         "storybook_cover_image_succeeded",
@@ -548,13 +760,45 @@ async def test_storybook_generation_is_queued_then_worker_records_page_image_eve
                         "story_saved",
                         "generation_completed",
                     ]
+                    plan = events[2].event_metadata["plan"]
+                    assert plan["mode"] == "storybook"
+                    assert [task["key"] for task in plan["tasks"]] == [
+                        "prepare_context",
+                        "generate_storybook_pages",
+                        "evaluate_storybook_pages",
+                        "generate_storybook_images",
+                        "persist_storybook",
+                        "queue_postprocessing",
+                        "complete_generation",
+                    ]
+                    image_task = next(
+                        task
+                        for task in plan["tasks"]
+                        if task["key"] == "generate_storybook_images"
+                    )
+                    assert image_task["required"] is False
+                    assert image_task["recoverable"] is True
+                    assert events[4].event_metadata["passed"] is True
+                    assert events[4].event_metadata["artifact"] == "storybook_pages"
                     page_events = [
                         event
                         for event in events
                         if event.event_type == "storybook_page_image_succeeded"
                     ]
                     assert [event.event_metadata["page_number"] for event in page_events] == [1, 2]
-                    assert events[8].event_metadata["completed_pages"] == [1, 2]
+                    assert events[10].event_metadata["completed_pages"] == [1, 2]
+
+                    async with AsyncClient(transport=transport, base_url="http://test") as client:
+                        client.cookies.set("access_token", auth_token)
+                        detail_response = await client.get(
+                            f"/api/generations/jobs/{job.id}"
+                        )
+
+                    assert detail_response.status_code == 200
+                    detail = detail_response.json()
+                    assert "evaluation_completed" not in [
+                        event["event_type"] for event in detail["events"]
+                    ]
     finally:
         app.dependency_overrides.clear()
 
@@ -716,6 +960,414 @@ async def test_story_provider_stats_aggregate_job_events(
         app.dependency_overrides.clear()
 
 
+async def test_story_trace_summary_aggregates_steps_artifacts_and_failure_categories(
+    db_session,
+    auth_token,
+    degraded_story_with_text,
+):
+    async def override_get_db():
+        yield db_session
+
+    app.dependency_overrides[get_db] = override_get_db
+
+    job = await create_generation_job(
+        db_session,
+        user_id=degraded_story_with_text.user_id,
+        output_mode="asset_retry",
+        input_type="image",
+        request_payload={"assets": ["image"]},
+        story_id=degraded_story_with_text.id,
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="cover_image_started",
+        status="running",
+        metadata={
+            "step": "image_generation",
+            "artifact": "cover_image",
+            "failure_category": None,
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="cover_image_failed",
+        status="failed",
+        metadata={
+            "step": "image_generation",
+            "artifact": "cover_image",
+            "failure_category": "provider_error",
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="quality_gate_failed",
+        status="failed",
+        metadata={
+            "step": "narrative_generation",
+            "artifact": "story_text",
+            "failure_category": "schema_error",
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="evaluation_completed",
+        status="failed",
+        metadata={
+            "step": "evaluation",
+            "artifact": "story_text",
+            "failure_category": "schema_error",
+            "overall_score": 0.0,
+            "passed": False,
+            "blocking": True,
+            "scores": [
+                {
+                    "dimension": "structure",
+                    "score": 0.0,
+                    "reason": "故事结构未通过质量门。",
+                },
+                {
+                    "dimension": "safety",
+                    "score": 0.0,
+                    "reason": "内容未通过儿童安全或结构完整性检查。",
+                },
+            ],
+        },
+    )
+
+    transport = ASGITransport(app=app)
+    try:
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            client.cookies.set("access_token", auth_token)
+
+            response = await client.get(
+                f"/api/generations/{degraded_story_with_text.id}/trace-summary"
+            )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["story_id"] == degraded_story_with_text.id
+        assert data["total_events"] == 4
+        assert data["failed_events"] == 2
+        assert data["by_step"] == [
+            {"name": "image_generation", "count": 2},
+            {"name": "narrative_generation", "count": 1},
+        ]
+        assert data["by_artifact"] == [
+            {"name": "cover_image", "count": 2},
+            {"name": "story_text", "count": 1},
+        ]
+        assert data["failure_categories"] == [
+            {"name": "provider_error", "count": 1},
+            {"name": "schema_error", "count": 1},
+        ]
+        assert "evaluation" not in data
+        assert "overall_score" not in str(data)
+    finally:
+        app.dependency_overrides.clear()
+
+
+async def test_user_generation_job_detail_hides_internal_evaluation_step(
+    db_session,
+    auth_token,
+    test_user,
+):
+    async def override_get_db():
+        yield db_session
+
+    app.dependency_overrides[get_db] = override_get_db
+    transport = ASGITransport(app=app)
+
+    job = await create_generation_job(
+        db_session,
+        user_id=test_user.id,
+        output_mode="story",
+        input_type="keywords",
+        request_payload={
+            "output_mode": "story",
+            "type": "keywords",
+            "data": "小兔子",
+            "generate_images": False,
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        event_type="evaluation_completed",
+        status="succeeded",
+        metadata={
+            "step": "evaluation",
+            "artifact": "story_text",
+            "overall_score": 0.96,
+            "passed": True,
+            "blocking": False,
+            "scores": [
+                {"dimension": "structure", "score": 1.0, "reason": "完整。"},
+            ],
+        },
+    )
+
+    try:
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            client.cookies.set("access_token", auth_token)
+
+            response = await client.get(f"/api/generations/jobs/{job.id}")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["current_step"] == "narrative_generated"
+        assert data["progress_label"] == "正文已生成"
+        assert [event["event_type"] for event in data["events"]] == [
+            "request_accepted"
+        ]
+        assert "evaluation_completed" not in str(data)
+        assert "overall_score" not in str(data)
+    finally:
+        app.dependency_overrides.clear()
+
+
+async def test_user_generation_job_detail_sanitizes_request_payload(
+    db_session,
+    auth_token,
+    test_user,
+):
+    async def override_get_db():
+        yield db_session
+
+    app.dependency_overrides[get_db] = override_get_db
+    transport = ASGITransport(app=app)
+
+    job = await create_generation_job(
+        db_session,
+        user_id=test_user.id,
+        output_mode="story",
+        input_type="keywords",
+        request_payload={
+            "output_mode": "story",
+            "input_type": "keywords",
+            "type": "keywords",
+            "data": "不要回传原始关键词",
+            "education_theme": "勇气",
+            "generate_images": True,
+            "page_count": 6,
+            "child_profile_id": "child-public-id",
+            "universe_id": "universe-public-id",
+            "internal_dispatch_token": "secret-dispatch-token",
+            "provider_override": "internal-provider",
+            "evaluation_policy": {"threshold": 0.9},
+        },
+    )
+
+    try:
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            client.cookies.set("access_token", auth_token)
+
+            response = await client.get(f"/api/generations/jobs/{job.id}")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["request_payload"] == {
+            "child_profile_id": "child-public-id",
+            "generate_images": True,
+            "input_type": "keywords",
+            "output_mode": "story",
+            "page_count": 6,
+            "type": "keywords",
+            "universe_id": "universe-public-id",
+        }
+        payload_dump = str(data["request_payload"])
+        assert "不要回传原始关键词" not in payload_dump
+        assert "education_theme" not in payload_dump
+        assert "secret-dispatch-token" not in payload_dump
+        assert "internal-provider" not in payload_dump
+        assert "evaluation_policy" not in payload_dump
+    finally:
+        app.dependency_overrides.clear()
+
+
+async def test_user_generation_job_detail_sanitizes_public_event_metadata(
+    db_session,
+    auth_token,
+    degraded_story_with_text,
+):
+    async def override_get_db():
+        yield db_session
+
+    app.dependency_overrides[get_db] = override_get_db
+    transport = ASGITransport(app=app)
+
+    job = await create_generation_job(
+        db_session,
+        user_id=degraded_story_with_text.user_id,
+        output_mode="asset_generation",
+        input_type="image",
+        request_payload={"story_id": degraded_story_with_text.id, "assets": ["image"]},
+        story_id=degraded_story_with_text.id,
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="workflow_planned",
+        status="succeeded",
+        metadata={
+            "step": "request_acceptance",
+            "artifact": "none",
+            "plan": {
+                "mode": "asset_generation",
+                "tasks": [
+                    {
+                        "key": "complete_image_asset",
+                        "step": "image_generation",
+                        "artifact": "image",
+                        "required": False,
+                        "recoverable": True,
+                    }
+                ],
+            },
+            "internal_threshold": 0.72,
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="asset_generation_completed",
+        status="completed",
+        metadata={
+            "assets": ["image"],
+            "result_snapshot": {
+                "story_id": degraded_story_with_text.id,
+                "last_error": "internal provider detail",
+            },
+            "error": "internal provider detail",
+        },
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="executor_completed",
+        status="succeeded",
+        metadata={
+            "plan_mode": "asset_generation",
+            "planned_task_count": 3,
+            "executed_task_keys": ["complete_image_asset"],
+            "ignored_task_keys": [
+                "start_asset_generation",
+                "complete_asset_generation",
+            ],
+            "result_assets": ["cover_image"],
+        },
+    )
+
+    try:
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            client.cookies.set("access_token", auth_token)
+
+            response = await client.get(f"/api/generations/jobs/{job.id}")
+
+        assert response.status_code == 200
+        data = response.json()
+        workflow_event = next(
+            event for event in data["events"] if event["event_type"] == "workflow_planned"
+        )
+        assert workflow_event["event_metadata"] == {
+            "artifact": "none",
+            "plan_mode": "asset_generation",
+            "planned_task_count": 1,
+            "recoverable_task_count": 1,
+            "step": "request_acceptance",
+        }
+
+        completion_event = next(
+            event
+            for event in data["events"]
+            if event["event_type"] == "asset_generation_completed"
+        )
+        assert completion_event["event_metadata"] == {"assets": ["image"]}
+        assert "plan" not in workflow_event["event_metadata"]
+        assert "tasks" not in str(data["events"])
+        assert "internal_threshold" not in str(data["events"])
+        assert "result_snapshot" not in str(data["events"])
+        assert "internal provider detail" not in str(data["events"])
+        assert "executor_completed" not in str(data["events"])
+        assert "complete_image_asset" not in str(data["events"])
+    finally:
+        app.dependency_overrides.clear()
+
+
+async def test_user_generation_job_summary_hides_internal_executor_step(
+    db_session,
+    auth_token,
+    degraded_story_with_text,
+):
+    async def override_get_db():
+        yield db_session
+
+    app.dependency_overrides[get_db] = override_get_db
+    transport = ASGITransport(app=app)
+
+    job = await create_generation_job(
+        db_session,
+        user_id=degraded_story_with_text.user_id,
+        output_mode="asset_generation",
+        input_type="image",
+        request_payload={"story_id": degraded_story_with_text.id, "assets": ["image"]},
+        story_id=degraded_story_with_text.id,
+    )
+    await record_generation_event(
+        db_session,
+        job=job,
+        story_id=degraded_story_with_text.id,
+        event_type="executor_completed",
+        status="succeeded",
+        metadata={
+            "plan_mode": "asset_generation",
+            "executed_task_keys": ["complete_image_asset"],
+        },
+    )
+
+    try:
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            client.cookies.set("access_token", auth_token)
+
+            detail_response = await client.get(f"/api/generations/jobs/{job.id}")
+            list_response = await client.get(
+                f"/api/generations/{degraded_story_with_text.id}/jobs"
+            )
+            trace_summary_response = await client.get(
+                f"/api/generations/{degraded_story_with_text.id}/trace-summary"
+            )
+
+        assert detail_response.status_code == 200
+        detail = detail_response.json()
+        assert detail["current_step"] == "workflow_planned"
+        assert detail["progress_label"] == "工作流已规划"
+        assert "executor_completed" not in str(detail)
+        assert "complete_image_asset" not in str(detail)
+
+        assert list_response.status_code == 200
+        listed_job = next(item for item in list_response.json() if item["id"] == job.id)
+        assert listed_job["current_step"] == "workflow_planned"
+        assert listed_job["progress_label"] == "工作流已规划"
+
+        assert trace_summary_response.status_code == 200
+        trace_summary = trace_summary_response.json()
+        assert "executor_completed" not in str(trace_summary)
+        assert "complete_image_asset" not in str(trace_summary)
+        assert trace_summary["total_events"] == 1
+    finally:
+        app.dependency_overrides.clear()
+
+
 async def test_user_provider_analytics_aggregate_across_stories(
     db_session,
     auth_token,