/** * 引擎一致性测试套件(任务 A3.2)。 * * 同一套用例跑任何运行时实现:多轮对话、工具调用与回执、事件顺序、取消、错误分类、会话恢复 * (架构设计 5.2 的验收方式)。FakeRuntime 现在跑,PiRuntime 在 A3.3 接上后跑同一套; * A3.9「引擎可替换验证」也是拿它做判据。 */ import type { ExecutionReceipt } from '@aps/contracts'; import type { AgentRuntime, RuntimeEvent, RuntimeSession, ToolDescriptor } from './runtime.js'; export type RuntimeScenario = 'text' | 'tool_call' | 'cancel' | 'error'; export interface ConformanceCheck { name: string; ok: boolean; detail?: string; } export interface ConformanceReport { runtimeName: string; ok: boolean; checks: ConformanceCheck[]; } export interface ConformanceOptions { /** 为某个场景创建一个运行时;同一场景可能创建多次 */ createRuntime: (scenario: RuntimeScenario) => AgentRuntime | Promise; /** 能力目录投影 */ tools: ToolDescriptor[]; /** 各场景的输入文本;真引擎用真实问法,假引擎用任意文本 */ prompts?: Partial>; /** 单次 send 的超时(毫秒),防止实现挂住测试 */ sendTimeoutMs?: number; /** 跳过的场景(附原因),报告里记为跳过而不是通过 */ skipScenarios?: Partial>; /** * 会话 id 覆盖:真引擎(Pi 走扩展桥)要求平台侧真的存在这个会话, * 所以调用方可以先在平台建会话再把 id 传进来。 */ sessionIds?: Partial>; /** 工具场景期望的能力 id(默认 test.echo;真引擎用平台能力目录里的 id) */ toolCapabilityId?: string; } const DEFAULT_PROMPTS: Record = { text: '你好,介绍一下你自己', tool_call: '查一下下周的排产结果', cancel: '跑一个很长的任务', error: '触发一次错误', }; const CAPABILITY_FOR_SCENARIO: Partial> = { tool_call: 'test.echo', }; export function createTestTools(): ToolDescriptor[] { return [ { capabilityId: 'test.echo', toolName: 'test_echo', description: '回显', parameters: { type: 'object' } }, { capabilityId: 'test.write', toolName: 'test_write', description: '假写入', parameters: { type: 'object' } }, ]; } /** 单次迭代也要有超时:真引擎挂住时不能让测试跟着挂住 */ function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { return Promise.race([ promise, new Promise((_resolve, reject) => { setTimeout(() => reject(new Error(`${label} 超过 ${timeoutMs}ms 未返回`)), timeoutMs).unref(); }), ]); } async function collect( events: AsyncIterable, timeoutMs: number, hooks?: { onToolCall?: (callId: string) => void }, ): Promise { const collected: RuntimeEvent[] = []; const iterator = events[Symbol.asyncIterator](); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const next = await withTimeout(iterator.next(), Math.max(50, deadline - Date.now()), '等待运行时事件'); if (next.done) return collected; const event = next.value as RuntimeEvent; collected.push(event); if (event.type === 'tool_call') hooks?.onToolCall?.(event.callId); if (event.type === 'turn_end') return collected; } throw new Error(`send 未在 ${timeoutMs}ms 内结束;已收到:${collected.map((e) => e.type).join(',') || '(无)'}`); } export async function runRuntimeConformance(options: ConformanceOptions): Promise { const timeoutMs = options.sendTimeoutMs ?? 5_000; const skips = options.skipScenarios ?? {}; const ids = { text: options.sessionIds?.text ?? 'conformance-1', tool_call: options.sessionIds?.tool_call ?? 'conformance-tools', cancel: options.sessionIds?.cancel ?? 'conformance-cancel', error: options.sessionIds?.error ?? 'conformance-error', restore: options.sessionIds?.restore ?? 'conformance-restore', }; const prompts = { ...DEFAULT_PROMPTS, ...options.prompts }; const checks: ConformanceCheck[] = []; const record = (name: string, ok: boolean, detail?: string) => { checks.push(detail === undefined ? { name, ok } : { name, ok, detail }); }; let runtimeName = '(未知)'; let session: RuntimeSession | undefined; try { const runtime = await options.createRuntime('text'); runtimeName = runtime.name; session = await runtime.createSession({ sessionId: ids.text, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); record('createSession 返回会话', typeof session.id === 'string' && session.id.length > 0); // 1) 单轮文本:事件顺序与结束原因 const textEvents = await collect(session.send(prompts.text), timeoutMs); record('单轮产出文本并正常结束', textEvents.some((e) => e.type === 'text') && textEvents.at(-1)?.type === 'turn_end'); const last = textEvents.at(-1); record('结束原因为 stop', last?.type === 'turn_end' && last.reason === 'stop', last?.type === 'turn_end' ? `reason=${last.reason}` : '缺少 turn_end'); const endIndex = textEvents.findIndex((e) => e.type === 'turn_end'); record('turn_end 是最后一个事件', endIndex === textEvents.length - 1, `turn_end 在第 ${endIndex + 1}/${textEvents.length} 个`); // 2) 多轮:同一会话连续两轮 const secondTurn = await collect(session.send(prompts.text), timeoutMs); record('同一会话可多轮', secondTurn.at(-1)?.type === 'turn_end' && session.id === session.id); } catch (error) { record('基础回合(文本与多轮)', false, error instanceof Error ? error.message : '未知错误'); } // 3) 工具调用与回执 if (skips.tool_call !== undefined) { record('工具调用与回执:跳过', true, skips.tool_call); } else { try { const runtime = await options.createRuntime('tool_call'); const toolSession = await runtime.createSession({ sessionId: ids.tool_call, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); let toolCallSeen = false; const capabilityId = options.toolCapabilityId ?? CAPABILITY_FOR_SCENARIO.tool_call ?? options.tools[0]?.capabilityId ?? ''; const events = await collect(toolSession.send(prompts.tool_call), timeoutMs, { onToolCall: (callId) => { toolCallSeen = true; // platform 模式:平台执行后把回执送回;runtime 模式:工具在引擎侧执行(扩展桥),平台只观察 if (toolSession.toolExecution === 'platform') { const receipt: ExecutionReceipt = { executionId: `exec-${callId}`, status: 'succeeded', result: { ok: true } }; toolSession.resolveToolResult(callId, receipt); } }, }); const receiptSeen = events.some((e) => e.type === 'tool_result'); const callEvent = events.find((e) => e.type === 'tool_call'); record('发出 tool_call 并带能力 id', toolCallSeen && callEvent?.type === 'tool_call' && callEvent.capabilityId === capabilityId, `期望 ${capabilityId}`); record('回执回流后产出 tool_result', receiptSeen, `执行模式:${toolSession.toolExecution}`); record('工具回合以 turn_end 收尾', events.at(-1)?.type === 'turn_end'); await toolSession.dispose(); } catch (error) { record('工具调用与回执', false, error instanceof Error ? error.message : '未知错误'); } } // 4) 取消 if (skips.cancel !== undefined) { record('取消:跳过', true, skips.cancel); } else try { const runtime = await options.createRuntime('cancel'); const cancelSession = await runtime.createSession({ sessionId: ids.cancel, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); const controller = new AbortController(); const events: RuntimeEvent[] = []; const iterator = cancelSession.send(prompts.cancel, { signal: controller.signal })[Symbol.asyncIterator](); cancelSession.abort(); controller.abort(); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const next = await withTimeout(iterator.next(), timeoutMs, '等待取消后的事件'); if (next.done) break; events.push(next.value as RuntimeEvent); if ((next.value as RuntimeEvent).type === 'turn_end') break; } const end = events.at(-1); record('取消后以 turn_end(cancelled) 结束', end?.type === 'turn_end' && end.reason === 'cancelled', end?.type === 'turn_end' ? `reason=${end.reason}` : '缺少 turn_end'); await cancelSession.dispose(); } catch (error) { record('取消', false, error instanceof Error ? error.message : '未知错误'); } // 5) 错误分类 if (skips.error !== undefined) { record('错误分类:跳过', true, skips.error); } else try { const runtime = await options.createRuntime('error'); const errorSession = await runtime.createSession({ sessionId: ids.error, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); const events = await collect(errorSession.send(prompts.error), timeoutMs); const errorEvent = events.find((e) => e.type === 'error'); const validKinds = ['invalid_argument', 'forbidden', 'conflict', 'rate_limited', 'unavailable', 'unknown_result']; record( '错误事件带合法分类', errorEvent?.type === 'error' && validKinds.includes(errorEvent.kind), errorEvent?.type === 'error' ? `kind=${errorEvent.kind}` : '未收到 error 事件', ); await errorSession.dispose(); } catch (error) { record('错误分类', false, error instanceof Error ? error.message : '未知错误'); } // 6) 会话恢复:用平台历史创建新会话并继续 if (skips.text !== undefined) { record('会话恢复:跳过', true, skips.text); } else try { const runtime = await options.createRuntime('text'); const restored = await runtime.createSession({ sessionId: ids.restore, subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools, history: [ { messageId: 'm-1', role: 'user', ts: Date.now(), text: '之前的问题' }, { messageId: 'm-2', role: 'assistant', ts: Date.now(), text: '之前的回答' }, ], }); const events = await collect(restored.send(prompts.text), timeoutMs); record('带历史恢复后仍可对话', events.at(-1)?.type === 'turn_end'); await restored.dispose(); } catch (error) { record('会话恢复', false, error instanceof Error ? error.message : '未知错误'); } if (session) await session.dispose(); return { runtimeName, ok: checks.every((check) => check.ok), checks }; }