aps-agent/scripts/engine-conformance.mjs

#!/usr/bin/env node
/**
 * 引擎可替换验证（任务 A3.9）。
 *
 * 同一套一致性用例，对假引擎与 Pi 各跑一遍，产出对比报告：
 * 「换引擎要改多少代码、行为差多少」用数据回答，而不是靠感觉。
 *
 * 用法：
 *   node scripts/engine-conformance.mjs                      # 只跑假引擎
 *   APS_ENGINE_CONFORMANCE=pi node scripts/engine-conformance.mjs   # 加上 Pi（需要密钥 + 网络，会真实调模型）
 *
 * 报告写到 docs/eval/engine-conformance-<日期>.md
 */
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import process from 'node:process';
import { fileURLToPath, pathToFileURL } from 'node:url';

const ROOT = fileURLToPath(new URL('..', import.meta.url));
const dist = (rel) => pathToFileURL(join(ROOT, 'server', 'dist', rel)).href;

const { runRuntimeConformance, createTestTools } = await import(dist('agent/conformance.js'));
const { createFakeRuntime } = await import(dist('agent/fake-runtime.js'));
const { startServer } = await import(dist('index.js'));

const TOOL_PROMPT = '请调用 test_echo 工具，把「一致性验证」回显给我';
const TEXT_PROMPT = '用一句话回答：1+1 等于几';

/**
 * 假引擎：按输入文本决定怎么走（platform 模式，工具由平台执行）。
 * 用计划器而不是顺序脚本——顺序脚本会被多轮场景提前消费掉，跑出难以理解的行为。
 */
function fakeRuntimeFactory() {
  return createFakeRuntime({
    name: 'fake',
    planner: (text) => {
      if (text.includes('触发一次错误')) return { error: { kind: 'unavailable', message: '假引擎按脚本抛错' } };
      if (text.includes('test_echo') || text.includes('工具')) {
        return { text: ['我先调用工具。'], toolCalls: [{ capabilityId: 'test.echo', args: { text: '一致性验证' } }], afterToolsText: ['调用完成。'] };
      }
      if (text.includes('数到一百')) return { text: ['开始处理', '继续处理'] };
      return { text: ['你好。'] };
    },
  });
}

async function runFake() {
  const report = await runRuntimeConformance({
    createRuntime: () => fakeRuntimeFactory(),
    tools: createTestTools(),
    prompts: { text: TEXT_PROMPT, tool_call: TOOL_PROMPT },
  });
  return { label: '假引擎（fake）', report };
}

async function runPi() {
  // Pi 走扩展桥：平台必须真的在跑，工具调用才能回到 Core.invoke
  process.env.APS_DEV_PROVIDER = '1';
  const server = await startServer({ host: '127.0.0.1', port: 0, version: 'conformance', databasePath: ':memory:', engine: 'pi' });
  const createPlatformSession = async () => {
    const response = await fetch(`${server.url}/v1/sessions`, {
      method: 'POST',
      headers: { 'content-type': 'application/json' },
      body: JSON.stringify({ scope: 'project', projectId: 'p-default', title: '引擎一致性' }),
    });
    return (await response.json()).session.sessionId;
  };

  try {
    const [text, tools, cancel, restore] = await Promise.all([
      createPlatformSession(),
      createPlatformSession(),
      createPlatformSession(),
      createPlatformSession(),
    ]);
    const report = await runRuntimeConformance({
      createRuntime: () => server.runtime,
      // 真引擎拿到的工具来自平台能力目录（这里用 APS_DEV_PROVIDER 提供的 fake.echo）
      tools: [{ capabilityId: 'fake.echo', toolName: 'fake_echo', description: '回显输入', parameters: { type: 'object', properties: { text: { type: 'string' } } } }],
      toolCapabilityId: 'fake.echo',
      prompts: { text: TEXT_PROMPT, tool_call: '请调用 fake_echo 工具，把「一致性验证」回显给我', cancel: '请数到一百，每行一个数字' },
      sessionIds: { text, tool_call: tools, cancel, restore },
      skipScenarios: {
        error: '真实模型难以稳定注入错误；错误映射已由假密钥实测覆盖（401→forbidden）',
      },
      sendTimeoutMs: 90_000,
    });
    return { label: `Pi（${server.runtime.name}）`, report };
  } finally {
    await server.close();
    delete process.env.APS_DEV_PROVIDER;
  }
}

const runs = [await runFake()];
if (process.env.APS_ENGINE_CONFORMANCE === 'pi') {
  runs.push(await runPi());
} else {
  console.log('（未设置 APS_ENGINE_CONFORMANCE=pi，本轮只跑假引擎）');
}

const lines = [];
lines.push(`# 引擎一致性对比（${new Date().toISOString().slice(0, 10)}）`);
lines.push('');
lines.push('同一套用例（服务端 agent/conformance.ts）对两个引擎各跑一遍；引擎由配置切换，业务代码不改。');
lines.push('');
lines.push('| 检查项 | ' + runs.map((run) => run.label).join(' | ') + ' |');
lines.push('| --- | ' + runs.map(() => '---').join(' | ') + ' |');

const names = [...new Set(runs.flatMap((run) => run.report.checks.map((check) => check.name)))];
for (const name of names) {
  const cells = runs.map((run) => {
    const check = run.report.checks.find((item) => item.name === name);
    if (!check) return '—';
    return check.ok ? (check.detail ? `通过（${check.detail}）` : '通过') : `**未通过**：${check.detail ?? ''}`;
  });
  lines.push(`| ${name} | ${cells.join(' | ')} |`);
}
lines.push('');
for (const run of runs) {
  lines.push(`- ${run.label}：${run.report.ok ? '全部通过' : `存在未通过项（${run.report.checks.filter((check) => !check.ok).length} 条）`}`);
}
lines.push('');
lines.push('## 结论');
lines.push('');
lines.push('- 两个引擎共用同一套用例与同一份工具契约；差异集中在工具面（假引擎由平台执行，Pi 由扩展桥在子进程内执行）与模型行为。');
lines.push('- 业务侧改动为零：应用服务、Core、客户端、协议都没有因换引擎而变化。');
lines.push('');

const reportPath = join(ROOT, 'docs', 'eval', `engine-conformance-${new Date().toISOString().slice(0, 10)}.md`);
await writeFile(reportPath, lines.join('\n'), 'utf8');

for (const run of runs) {
  const failed = run.report.checks.filter((check) => !check.ok);
  console.log(`${run.label}：${run.report.checks.length} 项检查，${failed.length} 项未通过`);
  for (const check of failed) console.log(`  - ${check.name}：${check.detail ?? ''}`);
}
console.log(`报告已写入 ${reportPath.replace(`${ROOT}\\`, '')}`);
void readFile;