/** * 引擎一致性测试(任务 A3.2)。 * * 同一套用例先跑假引擎,A3.3 接真 Pi 后跑同一套(A3.9 引擎可替换验证也用它)。 * 第二个用例故意给一个坏运行时,证明这套检查不是摆设。 */ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { runRuntimeConformance, createTestTools } from '@aps/server/agent/conformance'; import { createFakeRuntime } from '@aps/server/agent/fake-runtime'; /** 各场景的脚本:假引擎按这个产出事件 */ function turnsFor(scenario) { switch (scenario) { case 'tool_call': return [ { text: ['我先查一下。'], toolCalls: [{ capabilityId: 'test.echo', args: { query: '下周排产' } }], afterToolsText: ['查到了,共 3 条。'], }, ]; case 'cancel': return [{ text: ['开始处理', '继续处理'] }]; case 'error': return [{ error: { kind: 'unavailable', message: '上游不可用' } }]; default: return [{ text: ['我是假引擎,'], afterToolsText: [] }]; } } test('假引擎通过一致性测试', async () => { const report = await runRuntimeConformance({ createRuntime: (scenario) => createFakeRuntime({ name: 'fake', turns: turnsFor(scenario) }), tools: createTestTools(), }); assert.equal(report.runtimeName, 'fake'); assert.equal(report.ok, true, JSON.stringify(report.checks.filter((check) => !check.ok), null, 2)); assert.ok(report.checks.length >= 10, `检查项应有十条以上,实际 ${report.checks.length}`); }); test('坏运行时被一致性测试拦下(负例)', async () => { const broken = { name: 'broken', async createSession() { return { id: 'broken-1', async *send(_text, options) { if (options?.signal?.aborted) { // 取消场景:仍然按 stop 收尾,属于违规 yield { type: 'turn_end', reason: 'stop' }; return; } yield { type: 'text', delta: '只说一句' }; yield { type: 'turn_end', reason: 'stop' }; }, resolveToolResult() { /* 不回填:工具回合永远拿不到 tool_result */ }, abort() { /* 忽略取消 */ }, async dispose() {}, }; }, }; const report = await runRuntimeConformance({ createRuntime: () => broken, tools: createTestTools() }); assert.equal(report.ok, false, '坏运行时不应当通过'); const failed = report.checks.filter((check) => !check.ok).map((check) => check.name); assert.ok(failed.includes('取消后以 turn_end(cancelled) 结束'), `应检出取消违规,实际:${failed.join('、')}`); assert.ok(failed.includes('回执回流后产出 tool_result'), `应检出工具回执未回流,实际:${failed.join('、')}`); assert.ok(failed.includes('错误事件带合法分类'), `应检出缺少错误事件,实际:${failed.join('、')}`); });