139 lines
6.2 KiB
JavaScript
139 lines
6.2 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* 引擎可替换验证（任务 A3.9）。
|
||
*
|
||
* 同一套一致性用例，对假引擎与 Pi 各跑一遍，产出对比报告：
|
||
* 「换引擎要改多少代码、行为差多少」用数据回答，而不是靠感觉。
|
||
*
|
||
* 用法：
|
||
* node scripts/engine-conformance.mjs # 只跑假引擎
|
||
* APS_ENGINE_CONFORMANCE=pi node scripts/engine-conformance.mjs # 加上 Pi（需要密钥 + 网络，会真实调模型）
|
||
*
|
||
* 报告写到 docs/eval/engine-conformance-<日期>.md
|
||
*/
|
||
import { readFile, writeFile } from 'node:fs/promises';
|
||
import { join } from 'node:path';
|
||
import process from 'node:process';
|
||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||
|
||
const ROOT = fileURLToPath(new URL('..', import.meta.url));
|
||
const dist = (rel) => pathToFileURL(join(ROOT, 'server', 'dist', rel)).href;
|
||
|
||
const { runRuntimeConformance, createTestTools } = await import(dist('agent/conformance.js'));
|
||
const { createFakeRuntime } = await import(dist('agent/fake-runtime.js'));
|
||
const { startServer } = await import(dist('index.js'));
|
||
|
||
const TOOL_PROMPT = '请调用 test_echo 工具，把「一致性验证」回显给我';
|
||
const TEXT_PROMPT = '用一句话回答：1+1 等于几';
|
||
|
||
/**
|
||
* 假引擎：按输入文本决定怎么走（platform 模式，工具由平台执行）。
|
||
* 用计划器而不是顺序脚本——顺序脚本会被多轮场景提前消费掉，跑出难以理解的行为。
|
||
*/
|
||
function fakeRuntimeFactory() {
|
||
return createFakeRuntime({
|
||
name: 'fake',
|
||
planner: (text) => {
|
||
if (text.includes('触发一次错误')) return { error: { kind: 'unavailable', message: '假引擎按脚本抛错' } };
|
||
if (text.includes('test_echo') || text.includes('工具')) {
|
||
return { text: ['我先调用工具。'], toolCalls: [{ capabilityId: 'test.echo', args: { text: '一致性验证' } }], afterToolsText: ['调用完成。'] };
|
||
}
|
||
if (text.includes('数到一百')) return { text: ['开始处理', '继续处理'] };
|
||
return { text: ['你好。'] };
|
||
},
|
||
});
|
||
}
|
||
|
||
async function runFake() {
|
||
const report = await runRuntimeConformance({
|
||
createRuntime: () => fakeRuntimeFactory(),
|
||
tools: createTestTools(),
|
||
prompts: { text: TEXT_PROMPT, tool_call: TOOL_PROMPT },
|
||
});
|
||
return { label: '假引擎（fake）', report };
|
||
}
|
||
|
||
async function runPi() {
|
||
// Pi 走扩展桥：平台必须真的在跑，工具调用才能回到 Core.invoke
|
||
process.env.APS_DEV_PROVIDER = '1';
|
||
const server = await startServer({ host: '127.0.0.1', port: 0, version: 'conformance', databasePath: ':memory:', engine: 'pi' });
|
||
const createPlatformSession = async () => {
|
||
const response = await fetch(`${server.url}/v1/sessions`, {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({ scope: 'project', projectId: 'p-default', title: '引擎一致性' }),
|
||
});
|
||
return (await response.json()).session.sessionId;
|
||
};
|
||
|
||
try {
|
||
const [text, tools, cancel, restore] = await Promise.all([
|
||
createPlatformSession(),
|
||
createPlatformSession(),
|
||
createPlatformSession(),
|
||
createPlatformSession(),
|
||
]);
|
||
const report = await runRuntimeConformance({
|
||
createRuntime: () => server.runtime,
|
||
// 真引擎拿到的工具来自平台能力目录（这里用 APS_DEV_PROVIDER 提供的 fake.echo）
|
||
tools: [{ capabilityId: 'fake.echo', toolName: 'fake_echo', description: '回显输入', parameters: { type: 'object', properties: { text: { type: 'string' } } } }],
|
||
toolCapabilityId: 'fake.echo',
|
||
prompts: { text: TEXT_PROMPT, tool_call: '请调用 fake_echo 工具，把「一致性验证」回显给我', cancel: '请数到一百，每行一个数字' },
|
||
sessionIds: { text, tool_call: tools, cancel, restore },
|
||
skipScenarios: {
|
||
error: '真实模型难以稳定注入错误；错误映射已由假密钥实测覆盖（401→forbidden）',
|
||
},
|
||
sendTimeoutMs: 90_000,
|
||
});
|
||
return { label: `Pi（${server.runtime.name}）`, report };
|
||
} finally {
|
||
await server.close();
|
||
delete process.env.APS_DEV_PROVIDER;
|
||
}
|
||
}
|
||
|
||
const runs = [await runFake()];
|
||
if (process.env.APS_ENGINE_CONFORMANCE === 'pi') {
|
||
runs.push(await runPi());
|
||
} else {
|
||
console.log('（未设置 APS_ENGINE_CONFORMANCE=pi，本轮只跑假引擎）');
|
||
}
|
||
|
||
const lines = [];
|
||
lines.push(`# 引擎一致性对比（${new Date().toISOString().slice(0, 10)}）`);
|
||
lines.push('');
|
||
lines.push('同一套用例（服务端 agent/conformance.ts）对两个引擎各跑一遍；引擎由配置切换，业务代码不改。');
|
||
lines.push('');
|
||
lines.push('| 检查项 | ' + runs.map((run) => run.label).join(' | ') + ' |');
|
||
lines.push('| --- | ' + runs.map(() => '---').join(' | ') + ' |');
|
||
|
||
const names = [...new Set(runs.flatMap((run) => run.report.checks.map((check) => check.name)))];
|
||
for (const name of names) {
|
||
const cells = runs.map((run) => {
|
||
const check = run.report.checks.find((item) => item.name === name);
|
||
if (!check) return '—';
|
||
return check.ok ? (check.detail ? `通过（${check.detail}）` : '通过') : `**未通过**：${check.detail ?? ''}`;
|
||
});
|
||
lines.push(`| ${name} | ${cells.join(' | ')} |`);
|
||
}
|
||
lines.push('');
|
||
for (const run of runs) {
|
||
lines.push(`- ${run.label}：${run.report.ok ? '全部通过' : `存在未通过项（${run.report.checks.filter((check) => !check.ok).length} 条）`}`);
|
||
}
|
||
lines.push('');
|
||
lines.push('## 结论');
|
||
lines.push('');
|
||
lines.push('- 两个引擎共用同一套用例与同一份工具契约；差异集中在工具面（假引擎由平台执行，Pi 由扩展桥在子进程内执行）与模型行为。');
|
||
lines.push('- 业务侧改动为零：应用服务、Core、客户端、协议都没有因换引擎而变化。');
|
||
lines.push('');
|
||
|
||
const reportPath = join(ROOT, 'docs', 'eval', `engine-conformance-${new Date().toISOString().slice(0, 10)}.md`);
|
||
await writeFile(reportPath, lines.join('\n'), 'utf8');
|
||
|
||
for (const run of runs) {
|
||
const failed = run.report.checks.filter((check) => !check.ok);
|
||
console.log(`${run.label}：${run.report.checks.length} 项检查，${failed.length} 项未通过`);
|
||
for (const check of failed) console.log(` - ${check.name}：${check.detail ?? ''}`);
|
||
}
|
||
console.log(`报告已写入 ${reportPath.replace(`${ROOT}\\`, '')}`);
|
||
void readFile; |