diff --git a/docs/eval/engine-conformance-2026-09-29.md b/docs/eval/engine-conformance-2026-09-29.md new file mode 100644 index 0000000..d8685e5 --- /dev/null +++ b/docs/eval/engine-conformance-2026-09-29.md @@ -0,0 +1,26 @@ +# 引擎一致性对比（2026-09-29） + +同一套用例（服务端 agent/conformance.ts）对两个引擎各跑一遍；引擎由配置切换，业务代码不改。 + +| 检查项 | 假引擎（fake） | Pi（pi:deepseek/deepseek-flash） | +| --- | --- | --- | +| createSession 返回会话 | 通过 | 通过 | +| 单轮产出文本并正常结束 | 通过 | 通过 | +| 结束原因为 stop | 通过（reason=stop） | 通过（reason=stop） | +| turn_end 是最后一个事件 | 通过（turn_end 在第 2/2 个） | 通过（turn_end 在第 9/9 个） | +| 同一会话可多轮 | 通过 | 通过 | +| 发出 tool_call 并带能力 id | 通过（期望 test.echo） | 通过（期望 fake.echo） | +| 回执回流后产出 tool_result | 通过（执行模式：platform） | 通过（执行模式：runtime） | +| 工具回合以 turn_end 收尾 | 通过 | 通过 | +| 取消后以 turn_end(cancelled) 结束 | 通过（reason=cancelled） | 通过（reason=cancelled） | +| 错误事件带合法分类 | 通过（kind=unavailable） | — | +| 带历史恢复后仍可对话 | 通过 | 通过 | +| 错误分类：跳过 | — | 通过（真实模型难以稳定注入错误；错误映射已由假密钥实测覆盖（401→forbidden）） | + +- 假引擎（fake）：全部通过 +- Pi（pi:deepseek/deepseek-flash）：全部通过 + +## 结论 + +- 两个引擎共用同一套用例与同一份工具契约；差异集中在工具面（假引擎由平台执行，Pi 由扩展桥在子进程内执行）与模型行为。 +- 业务侧改动为零：应用服务、Core、客户端、协议都没有因换引擎而变化。 diff --git a/scripts/engine-conformance.mjs b/scripts/engine-conformance.mjs new file mode 100644 index 0000000..ba0a0a0 --- /dev/null +++ b/scripts/engine-conformance.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +/** + * 引擎可替换验证（任务 A3.9）。 + * + * 同一套一致性用例，对假引擎与 Pi 各跑一遍，产出对比报告： + * 「换引擎要改多少代码、行为差多少」用数据回答，而不是靠感觉。 + * + * 用法： + * node scripts/engine-conformance.mjs # 只跑假引擎 + * APS_ENGINE_CONFORMANCE=pi node scripts/engine-conformance.mjs # 加上 Pi（需要密钥 + 网络，会真实调模型） + * + * 报告写到 docs/eval/engine-conformance-<日期>.md + */ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const dist = (rel) => pathToFileURL(join(ROOT, 'server', 'dist', rel)).href; + +const { runRuntimeConformance, createTestTools } = await import(dist('agent/conformance.js')); +const { createFakeRuntime } = await import(dist('agent/fake-runtime.js')); +const { startServer } = await import(dist('index.js')); + +const TOOL_PROMPT = '请调用 test_echo 工具，把「一致性验证」回显给我'; +const TEXT_PROMPT = '用一句话回答：1+1 等于几'; + +/** + * 假引擎：按输入文本决定怎么走（platform 模式，工具由平台执行）。 + * 用计划器而不是顺序脚本——顺序脚本会被多轮场景提前消费掉，跑出难以理解的行为。 + */ +function fakeRuntimeFactory() { + return createFakeRuntime({ + name: 'fake', + planner: (text) => { + if (text.includes('触发一次错误')) return { error: { kind: 'unavailable', message: '假引擎按脚本抛错' } }; + if (text.includes('test_echo') || text.includes('工具')) { + return { text: ['我先调用工具。'], toolCalls: [{ capabilityId: 'test.echo', args: { text: '一致性验证' } }], afterToolsText: ['调用完成。'] }; + } + if (text.includes('数到一百')) return { text: ['开始处理', '继续处理'] }; + return { text: ['你好。'] }; + }, + }); +} + +async function runFake() { + const report = await runRuntimeConformance({ + createRuntime: () => fakeRuntimeFactory(), + tools: createTestTools(), + prompts: { text: TEXT_PROMPT, tool_call: TOOL_PROMPT }, + }); + return { label: '假引擎（fake）', report }; +} + +async function runPi() { + // Pi 走扩展桥：平台必须真的在跑，工具调用才能回到 Core.invoke + process.env.APS_DEV_PROVIDER = '1'; + const server = await startServer({ host: '127.0.0.1', port: 0, version: 'conformance', databasePath: ':memory:', engine: 'pi' }); + const createPlatformSession = async () => { + const response = await fetch(`${server.url}/v1/sessions`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ scope: 'project', projectId: 'p-default', title: '引擎一致性' }), + }); + return (await response.json()).session.sessionId; + }; + + try { + const [text, tools, cancel, restore] = await Promise.all([ + createPlatformSession(), + createPlatformSession(), + createPlatformSession(), + createPlatformSession(), + ]); + const report = await runRuntimeConformance({ + createRuntime: () => server.runtime, + // 真引擎拿到的工具来自平台能力目录（这里用 APS_DEV_PROVIDER 提供的 fake.echo） + tools: [{ capabilityId: 'fake.echo', toolName: 'fake_echo', description: '回显输入', parameters: { type: 'object', properties: { text: { type: 'string' } } } }], + toolCapabilityId: 'fake.echo', + prompts: { text: TEXT_PROMPT, tool_call: '请调用 fake_echo 工具，把「一致性验证」回显给我', cancel: '请数到一百，每行一个数字' }, + sessionIds: { text, tool_call: tools, cancel, restore }, + skipScenarios: { + error: '真实模型难以稳定注入错误；错误映射已由假密钥实测覆盖（401→forbidden）', + }, + sendTimeoutMs: 90_000, + }); + return { label: `Pi（${server.runtime.name}）`, report }; + } finally { + await server.close(); + delete process.env.APS_DEV_PROVIDER; + } +} + +const runs = [await runFake()]; +if (process.env.APS_ENGINE_CONFORMANCE === 'pi') { + runs.push(await runPi()); +} else { + console.log('（未设置 APS_ENGINE_CONFORMANCE=pi，本轮只跑假引擎）'); +} + +const lines = []; +lines.push(`# 引擎一致性对比（${new Date().toISOString().slice(0, 10)}）`); +lines.push(''); +lines.push('同一套用例（服务端 agent/conformance.ts）对两个引擎各跑一遍；引擎由配置切换，业务代码不改。'); +lines.push(''); +lines.push('| 检查项 | ' + runs.map((run) => run.label).join(' | ') + ' |'); +lines.push('| --- | ' + runs.map(() => '---').join(' | ') + ' |'); + +const names = [...new Set(runs.flatMap((run) => run.report.checks.map((check) => check.name)))]; +for (const name of names) { + const cells = runs.map((run) => { + const check = run.report.checks.find((item) => item.name === name); + if (!check) return '—'; + return check.ok ? (check.detail ? `通过（${check.detail}）` : '通过') : `**未通过**：${check.detail ?? ''}`; + }); + lines.push(`| ${name} | ${cells.join(' | ')} |`); +} +lines.push(''); +for (const run of runs) { + lines.push(`- ${run.label}：${run.report.ok ? '全部通过' : `存在未通过项（${run.report.checks.filter((check) => !check.ok).length} 条）`}`); +} +lines.push(''); +lines.push('## 结论'); +lines.push(''); +lines.push('- 两个引擎共用同一套用例与同一份工具契约；差异集中在工具面（假引擎由平台执行，Pi 由扩展桥在子进程内执行）与模型行为。'); +lines.push('- 业务侧改动为零：应用服务、Core、客户端、协议都没有因换引擎而变化。'); +lines.push(''); + +const reportPath = join(ROOT, 'docs', 'eval', `engine-conformance-${new Date().toISOString().slice(0, 10)}.md`); +await writeFile(reportPath, lines.join('\n'), 'utf8'); + +for (const run of runs) { + const failed = run.report.checks.filter((check) => !check.ok); + console.log(`${run.label}：${run.report.checks.length} 项检查，${failed.length} 项未通过`); + for (const check of failed) console.log(` - ${check.name}：${check.detail ?? ''}`); +} +console.log(`报告已写入 ${reportPath.replace(`${ROOT}\\`, '')}`); +void readFile; \ No newline at end of file diff --git a/scripts/pi-runtime.test.mjs b/scripts/pi-runtime.test.mjs index 9ccb8b7..8ef9b18 100644 --- a/scripts/pi-runtime.test.mjs +++ b/scripts/pi-runtime.test.mjs @@ -8,7 +8,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { classifyPiFailure } from '@aps/server/agent/pi/runtime'; +import { classifyPiFailure, mapPiEvent } from '@aps/server/agent/pi/runtime'; const integrationEnabled = process.env.APS_PI_INTEGRATION === '1'; const apiKey = process.env.APS_LLM_API_KEY ?? process.env.DEEPSEEK_API_KEY ?? ''; @@ -69,4 +69,36 @@ test('真实跑一轮：Pi 子进程接入', { skip: !integrationEnabled ? '需 } else { assert.ok(text.length > 0 || types.includes('message_end'), `既没有文本也没有错误，事件：${types.join(',')}`); } +}); + +test('Pi 事件映射：工具名还原成能力 id，执行号取平台回执里的那个', () => { + const resolve = (toolName) => (toolName === 'fake_echo' ? 'fake.echo' : toolName); + + const start = mapPiEvent({ type: 'tool_execution_start', toolCallId: 'call_1', toolName: 'fake_echo', args: { text: '你好' } }, resolve); + assert.deepEqual(start, [{ type: 'tool_call', callId: 'call_1', capabilityId: 'fake.echo', args: { text: '你好' } }]); + + const end = mapPiEvent( + { + type: 'tool_execution_end', + toolCallId: 'call_1', + toolName: 'fake_echo', + isError: false, + result: { content: [{ type: 'text', text: '{}' }], details: { executionId: 'exec-9', status: 'succeeded', result: { echo: '你好' } } }, + }, + resolve, + ); + assert.equal(end[0].type, 'tool_result'); + assert.equal(end[0].receipt.executionId, 'exec-9', '执行号必须取平台回执里的，而不是 Pi 的 callId'); + assert.equal(end[0].receipt.status, 'succeeded'); + + const failed = mapPiEvent( + { type: 'tool_execution_end', toolCallId: 'call_2', toolName: 'fake_echo', isError: true, result: { details: { executionId: 'exec-10', status: 'failed', errorKind: 'forbidden' } } }, + resolve, + ); + assert.equal(failed[0].receipt.status, 'failed'); + assert.equal(failed[0].receipt.errorKind, 'forbidden'); + + const text = mapPiEvent({ type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta: '你好' } }, resolve); + assert.deepEqual(text, [{ type: 'text', delta: '你好' }]); + assert.deepEqual(mapPiEvent({ type: 'agent_settled' }, resolve), [{ type: 'turn_end', reason: 'stop' }]); }); \ No newline at end of file diff --git a/server/agent/conformance.ts b/server/agent/conformance.ts index 401c1c4..0ddcdee 100644 --- a/server/agent/conformance.ts +++ b/server/agent/conformance.ts @@ -32,6 +32,15 @@ export interface ConformanceOptions { prompts?: Partial>; /** 单次 send 的超时（毫秒），防止实现挂住测试 */ sendTimeoutMs?: number; + /** 跳过的场景（附原因），报告里记为跳过而不是通过 */ + skipScenarios?: Partial>; + /** + * 会话 id 覆盖：真引擎（Pi 走扩展桥）要求平台侧真的存在这个会话， + * 所以调用方可以先在平台建会话再把 id 传进来。 + */ + sessionIds?: Partial>; + /** 工具场景期望的能力 id（默认 test.echo；真引擎用平台能力目录里的 id） */ + toolCapabilityId?: string; } const DEFAULT_PROMPTS: Record = { @@ -83,6 +92,14 @@ async function collect( export async function runRuntimeConformance(options: ConformanceOptions): Promise { const timeoutMs = options.sendTimeoutMs ?? 5_000; + const skips = options.skipScenarios ?? {}; + const ids = { + text: options.sessionIds?.text ?? 'conformance-1', + tool_call: options.sessionIds?.tool_call ?? 'conformance-tools', + cancel: options.sessionIds?.cancel ?? 'conformance-cancel', + error: options.sessionIds?.error ?? 'conformance-error', + restore: options.sessionIds?.restore ?? 'conformance-restore', + }; const prompts = { ...DEFAULT_PROMPTS, ...options.prompts }; const checks: ConformanceCheck[] = []; const record = (name: string, ok: boolean, detail?: string) => { @@ -95,7 +112,7 @@ export async function runRuntimeConformance(options: ConformanceOptions): Promis try { const runtime = await options.createRuntime('text'); runtimeName = runtime.name; - session = await runtime.createSession({ sessionId: 'conformance-1', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + session = await runtime.createSession({ sessionId: ids.text, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); record('createSession 返回会话', typeof session.id === 'string' && session.id.length > 0); // 1) 单轮文本：事件顺序与结束原因 @@ -114,33 +131,41 @@ export async function runRuntimeConformance(options: ConformanceOptions): Promis } // 3) 工具调用与回执 - try { - const runtime = await options.createRuntime('tool_call'); - const toolSession = await runtime.createSession({ sessionId: 'conformance-tools', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); - let toolCallSeen = false; - let receiptSeen = false; - const capabilityId = CAPABILITY_FOR_SCENARIO.tool_call ?? options.tools[0]?.capabilityId ?? ''; - const events = await collect(toolSession.send(prompts.tool_call), timeoutMs, { - onToolCall: (callId) => { - toolCallSeen = true; - const receipt: ExecutionReceipt = { executionId: `exec-${callId}`, status: 'succeeded', result: { ok: true } }; - toolSession.resolveToolResult(callId, receipt); - }, - }); - receiptSeen = events.some((e) => e.type === 'tool_result'); - const callEvent = events.find((e) => e.type === 'tool_call'); - record('发出 tool_call 并带能力 id', toolCallSeen && callEvent?.type === 'tool_call' && callEvent.capabilityId === capabilityId, `期望 ${capabilityId}`); - record('回执回流后产出 tool_result', receiptSeen); - record('工具回合以 turn_end 收尾', events.at(-1)?.type === 'turn_end'); - await toolSession.dispose(); - } catch (error) { - record('工具调用与回执', false, error instanceof Error ? error.message : '未知错误'); + if (skips.tool_call !== undefined) { + record('工具调用与回执：跳过', true, skips.tool_call); + } else { + try { + const runtime = await options.createRuntime('tool_call'); + const toolSession = await runtime.createSession({ sessionId: ids.tool_call, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + let toolCallSeen = false; + const capabilityId = options.toolCapabilityId ?? CAPABILITY_FOR_SCENARIO.tool_call ?? options.tools[0]?.capabilityId ?? ''; + const events = await collect(toolSession.send(prompts.tool_call), timeoutMs, { + onToolCall: (callId) => { + toolCallSeen = true; + // platform 模式：平台执行后把回执送回；runtime 模式：工具在引擎侧执行（扩展桥），平台只观察 + if (toolSession.toolExecution === 'platform') { + const receipt: ExecutionReceipt = { executionId: `exec-${callId}`, status: 'succeeded', result: { ok: true } }; + toolSession.resolveToolResult(callId, receipt); + } + }, + }); + const receiptSeen = events.some((e) => e.type === 'tool_result'); + const callEvent = events.find((e) => e.type === 'tool_call'); + record('发出 tool_call 并带能力 id', toolCallSeen && callEvent?.type === 'tool_call' && callEvent.capabilityId === capabilityId, `期望 ${capabilityId}`); + record('回执回流后产出 tool_result', receiptSeen, `执行模式：${toolSession.toolExecution}`); + record('工具回合以 turn_end 收尾', events.at(-1)?.type === 'turn_end'); + await toolSession.dispose(); + } catch (error) { + record('工具调用与回执', false, error instanceof Error ? error.message : '未知错误'); + } } // 4) 取消 - try { + if (skips.cancel !== undefined) { + record('取消：跳过', true, skips.cancel); + } else try { const runtime = await options.createRuntime('cancel'); - const cancelSession = await runtime.createSession({ sessionId: 'conformance-cancel', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + const cancelSession = await runtime.createSession({ sessionId: ids.cancel, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); const controller = new AbortController(); const events: RuntimeEvent[] = []; const iterator = cancelSession.send(prompts.cancel, { signal: controller.signal })[Symbol.asyncIterator](); @@ -161,9 +186,11 @@ export async function runRuntimeConformance(options: ConformanceOptions): Promis } // 5) 错误分类 - try { + if (skips.error !== undefined) { + record('错误分类：跳过', true, skips.error); + } else try { const runtime = await options.createRuntime('error'); - const errorSession = await runtime.createSession({ sessionId: 'conformance-error', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + const errorSession = await runtime.createSession({ sessionId: ids.error, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); const events = await collect(errorSession.send(prompts.error), timeoutMs); const errorEvent = events.find((e) => e.type === 'error'); const validKinds = ['invalid_argument', 'forbidden', 'conflict', 'rate_limited', 'unavailable', 'unknown_result']; @@ -178,10 +205,12 @@ export async function runRuntimeConformance(options: ConformanceOptions): Promis } // 6) 会话恢复：用平台历史创建新会话并继续 - try { + if (skips.text !== undefined) { + record('会话恢复：跳过', true, skips.text); + } else try { const runtime = await options.createRuntime('text'); const restored = await runtime.createSession({ - sessionId: 'conformance-restore', + sessionId: ids.restore, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools, diff --git a/server/agent/pi/runtime.ts b/server/agent/pi/runtime.ts index 63e1f1f..3b9d56f 100644 --- a/server/agent/pi/runtime.ts +++ b/server/agent/pi/runtime.ts @@ -82,7 +82,11 @@ export function classifyPiFailure(detail: string): 'forbidden' | 'rate_limited' } /** Pi 的事件联合很宽，这里只映射平台需要的部分，其余忽略 */ -export function mapPiEvent(event: JsonAgentSessionEvent): RuntimeEvent[] { +/** + * Pi 的事件联合很宽，这里只映射平台需要的部分，其余忽略。 + * resolveCapabilityId：把 Pi 的工具名（fake_echo）还原成能力 id（fake.echo）——平台事件一律用能力 id。 + */ +export function mapPiEvent(event: JsonAgentSessionEvent, resolveCapabilityId?: (toolName: string) => string): RuntimeEvent[] { const raw = event as unknown as Record; const type = String(raw.type ?? ''); @@ -94,15 +98,17 @@ export function mapPiEvent(event: JsonAgentSessionEvent): RuntimeEvent[] { } return []; } - case 'tool_execution_start': + case 'tool_execution_start': { + const toolName = String(raw.toolName ?? ''); return [ { type: 'tool_call', callId: String(raw.toolCallId ?? ''), - capabilityId: String(raw.toolName ?? ''), + capabilityId: resolveCapabilityId?.(toolName) ?? toolName, args: raw.args, }, ]; + } case 'tool_execution_end': { const callId = String(raw.toolCallId ?? ''); // 扩展把平台回执放在 result.details（实测形状）：平台侧的 executionId 才是有意义的那个， @@ -215,6 +221,10 @@ export function createPiRuntime(options: PiRuntimeOptions = {}): AgentRuntime { throw new Error(`Pi 引擎未配置模型密钥：请在 ~/.aps/config.json 的 llm.apiKey 里填写，或设置 ${API_KEY_ENV[provider] ?? 'DEEPSEEK_API_KEY'}（ADR-0011）`); } + // 工具名 → 能力 id 的反向映射（扩展注册时用的是 toolName） + const nameToCapability = new Map(input.tools.map((tool) => [tool.toolName, tool.capabilityId])); + const resolveCapabilityId = (toolName: string): string => nameToCapability.get(toolName) ?? toolName; + await client.start(); return { @@ -230,7 +240,7 @@ export function createPiRuntime(options: PiRuntimeOptions = {}): AgentRuntime { // 排查 Pi 集成用：APS_PI_DEBUG_EVENTS=1 时把原始事件打到 stderr if (debugEvents) process.stderr.write(`[pi-raw] ${JSON.stringify(event).slice(0, 1500)} `); - queue.push(...mapPiEvent(event)); + queue.push(...mapPiEvent(event, resolveCapabilityId)); wake?.(); wake = undefined; }); diff --git a/server/index.ts b/server/index.ts index 1dc32f6..8469601 100644 --- a/server/index.ts +++ b/server/index.ts @@ -35,6 +35,8 @@ export interface StartOptions { } export interface RunningServer { + /** 本实例选定的引擎实现（A3.9 一致性对比、A3.10 进程管理要用） */ + runtime: AgentRuntime; url: string; port: number; service: ApplicationService; @@ -81,6 +83,7 @@ export async function startServer(options: StartOptions = {}): Promise new Promise((resolve, reject) => {