231 lines
9.3 KiB
JavaScript
231 lines
9.3 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
/**
|
|||
|
|
* 问法评测跑手（任务 A2.7）。
|
|||
|
|
*
|
|||
|
|
* 读 docs/eval/intent-cases.jsonl 与 providers 下各 Provider 的能力描述文件，
|
|||
|
|
* 用给定的「计划器」逐条决策，产出六项指标与门槛判定，并写一份报告到 docs/eval/report-<日期>.md。
|
|||
|
|
*
|
|||
|
|
* 现在跑的是规则计划器基线——它的意义是给出地板值：真引擎（Pi）必须明显高过它。
|
|||
|
|
* 换引擎只换 planner，用例与指标不变。
|
|||
|
|
*
|
|||
|
|
* 用法：node scripts/eval-intents.mjs [--engine=rule]
|
|||
|
|
*/
|
|||
|
|
import { createRequire } from 'node:module';
|
|||
|
|
import { readFile, readdir, writeFile } from 'node:fs/promises';
|
|||
|
|
import { existsSync } from 'node:fs';
|
|||
|
|
import { join, relative } from 'node:path';
|
|||
|
|
import process from 'node:process';
|
|||
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|||
|
|
|
|||
|
|
const ROOT = fileURLToPath(new URL('..', import.meta.url));
|
|||
|
|
const require = createRequire(import.meta.url);
|
|||
|
|
|
|||
|
|
const THRESHOLDS = [
|
|||
|
|
{ key: 'toolSelection', label: '工具选择准确率', target: '≥ 90%', pass: (v) => v >= 0.9 },
|
|||
|
|
{ key: 'paramHit', label: '参数命中率', target: '≥ 90%', pass: (v) => v >= 0.9 },
|
|||
|
|
{ key: 'clarify', label: '澄清率', target: '≥ 90%', pass: (v) => v >= 0.9 },
|
|||
|
|
{ key: 'overClarify', label: '误澄清率', target: '≤ 10%', pass: (v) => v <= 0.1 },
|
|||
|
|
{ key: 'hallucination', label: '幻觉率', target: '0', pass: (v) => v === 0 },
|
|||
|
|
{ key: 'overreach', label: '越权率', target: '0', pass: (v) => v === 0 },
|
|||
|
|
];
|
|||
|
|
|
|||
|
|
async function loadCatalog() {
|
|||
|
|
const providersDir = join(ROOT, 'providers');
|
|||
|
|
const tools = [];
|
|||
|
|
if (!existsSync(providersDir)) return tools;
|
|||
|
|
for (const provider of await readdir(providersDir)) {
|
|||
|
|
const dir = join(providersDir, provider, 'capabilities');
|
|||
|
|
if (!existsSync(dir)) continue;
|
|||
|
|
for (const file of await readdir(dir)) {
|
|||
|
|
if (!file.endsWith('.capability.json')) continue;
|
|||
|
|
const capability = JSON.parse(await readFile(join(dir, file), 'utf8'));
|
|||
|
|
tools.push({
|
|||
|
|
capabilityId: capability.id,
|
|||
|
|
toolName: capability.id.replace(/\./g, '_'),
|
|||
|
|
description: capability.purpose,
|
|||
|
|
parameters: capability.inputSchema ?? { type: 'object' },
|
|||
|
|
synonyms: capability.synonyms ?? [],
|
|||
|
|
effect: capability.effect,
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return tools;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function loadCases() {
|
|||
|
|
const text = await readFile(join(ROOT, 'docs', 'eval', 'intent-cases.jsonl'), 'utf8');
|
|||
|
|
return text
|
|||
|
|
.split(/\r?\n/)
|
|||
|
|
.filter((line) => line.trim())
|
|||
|
|
.map((line) => JSON.parse(line));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/** 把计划器的决策翻译成用例的期望词汇 */
|
|||
|
|
function normalize(decision) {
|
|||
|
|
if (decision.behavior === 'tool') return { behavior: '调用', capability: decision.capabilityId ?? null, args: decision.args ?? {} };
|
|||
|
|
if (decision.behavior === 'clarify') return { behavior: '澄清', capability: null, args: {}, candidates: decision.candidates ?? [] };
|
|||
|
|
return { behavior: '拒绝', capability: null, args: {} };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function rate(hit, total) {
|
|||
|
|
return total === 0 ? 1 : hit / total;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function formatRate(value) {
|
|||
|
|
return `${(value * 100).toFixed(1)}%`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function main() {
|
|||
|
|
const engineArg = process.argv.find((arg) => arg.startsWith('--engine='));
|
|||
|
|
const engine = engineArg ? engineArg.split('=')[1] : 'rule';
|
|||
|
|
|
|||
|
|
const distPath = join(ROOT, 'server', 'dist', 'agent', 'rule-planner.js');
|
|||
|
|
if (!existsSync(distPath)) {
|
|||
|
|
console.error('缺少编译产物：请先跑 pnpm build（或 pnpm typecheck）');
|
|||
|
|
process.exitCode = 1;
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
const { createRulePlanner } = await import(pathToFileURL(distPath).href);
|
|||
|
|
if (engine !== 'rule') {
|
|||
|
|
console.error(`暂不支持的引擎：${engine}（当前只有 rule；Pi 在 A3.3 之后接入）`);
|
|||
|
|
process.exitCode = 1;
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const tools = await loadCatalog();
|
|||
|
|
const cases = await loadCases();
|
|||
|
|
const planner = createRulePlanner();
|
|||
|
|
const results = [];
|
|||
|
|
|
|||
|
|
let callTotal = 0;
|
|||
|
|
let callHit = 0;
|
|||
|
|
let paramTotal = 0;
|
|||
|
|
let paramHit = 0;
|
|||
|
|
let clarifyTotal = 0;
|
|||
|
|
let clarifyHit = 0;
|
|||
|
|
let overClarifyTotal = 0;
|
|||
|
|
let overClarifyHit = 0;
|
|||
|
|
let hallucination = 0;
|
|||
|
|
let overreach = 0;
|
|||
|
|
const catalogIds = new Set(tools.map((tool) => tool.capabilityId));
|
|||
|
|
|
|||
|
|
for (const testCase of cases) {
|
|||
|
|
const decision = normalize(planner(testCase.utterance, tools));
|
|||
|
|
const expected = testCase.expect;
|
|||
|
|
let ok = true;
|
|||
|
|
let detail = '';
|
|||
|
|
|
|||
|
|
if (expected.behavior === '调用') {
|
|||
|
|
callTotal += 1;
|
|||
|
|
const capabilityOk = decision.behavior === '调用' && decision.capability === expected.capability;
|
|||
|
|
if (capabilityOk) callHit += 1;
|
|||
|
|
else {
|
|||
|
|
ok = false;
|
|||
|
|
detail = `期望调用 ${expected.capability}，实际 ${decision.behavior}${decision.capability ? ` ${decision.capability}` : ''}`;
|
|||
|
|
}
|
|||
|
|
overClarifyTotal += 1;
|
|||
|
|
if (decision.behavior === '澄清') overClarifyHit += 1;
|
|||
|
|
|
|||
|
|
const expectedParams = expected.params ?? {};
|
|||
|
|
const keys = Object.keys(expectedParams);
|
|||
|
|
if (keys.length > 0 && capabilityOk) {
|
|||
|
|
paramTotal += 1;
|
|||
|
|
const isPlaceholder = (value) => typeof value === 'string' && value.startsWith('<');
|
|||
|
|
const missing = keys.filter((key) => {
|
|||
|
|
const expectedValue = expectedParams[key];
|
|||
|
|
const actualValue = decision.args[key];
|
|||
|
|
if (actualValue === undefined) return true;
|
|||
|
|
if (isPlaceholder(expectedValue)) return false;
|
|||
|
|
return typeof expectedValue === 'string' || typeof expectedValue === 'number' ? actualValue !== expectedValue : false;
|
|||
|
|
});
|
|||
|
|
if (missing.length === 0) paramHit += 1;
|
|||
|
|
else {
|
|||
|
|
ok = false;
|
|||
|
|
detail = `${detail ? `${detail}；` : ''}缺参数：${missing.join('、')}`;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} else if (expected.behavior === '澄清') {
|
|||
|
|
clarifyTotal += 1;
|
|||
|
|
if (decision.behavior === '澄清') clarifyHit += 1;
|
|||
|
|
else {
|
|||
|
|
ok = false;
|
|||
|
|
detail = `期望澄清，实际 ${decision.behavior}${decision.capability ? ` ${decision.capability}` : ''}`;
|
|||
|
|
}
|
|||
|
|
} else if (expected.behavior === '拒绝') {
|
|||
|
|
if (decision.behavior === '调用') {
|
|||
|
|
overreach += 1;
|
|||
|
|
ok = false;
|
|||
|
|
detail = `期望拒绝，实际调用了 ${decision.capability}`;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (decision.behavior === '调用' && decision.capability && !catalogIds.has(decision.capability)) {
|
|||
|
|
hallucination += 1;
|
|||
|
|
ok = false;
|
|||
|
|
detail = `${detail ? `${detail}；` : ''}调用了不存在的能力 ${decision.capability}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
results.push({ testCase, decision, expected, ok, detail });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const metrics = {
|
|||
|
|
toolSelection: rate(callHit, callTotal),
|
|||
|
|
paramHit: rate(paramHit, paramTotal),
|
|||
|
|
clarify: rate(clarifyHit, clarifyTotal),
|
|||
|
|
overClarify: rate(overClarifyHit, overClarifyTotal),
|
|||
|
|
hallucination,
|
|||
|
|
overreach,
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const failed = results.filter((result) => !result.ok);
|
|||
|
|
const lines = [];
|
|||
|
|
lines.push(`# 问法评测报告（${new Date().toISOString().slice(0, 10)}）`);
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push(`- 引擎：规则计划器基线（engine=${engine}）`);
|
|||
|
|
lines.push(`- 用例：${cases.length} 条，来自 docs/eval/intent-cases.jsonl`);
|
|||
|
|
lines.push(`- 能力目录：${tools.length} 项，来自 providers/*/capabilities/`);
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('## 指标');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('| 指标 | 值 | 门槛 | 判定 |');
|
|||
|
|
lines.push('| --- | --- | --- | --- |');
|
|||
|
|
for (const threshold of THRESHOLDS) {
|
|||
|
|
const value = metrics[threshold.key];
|
|||
|
|
const shown = threshold.key === 'hallucination' || threshold.key === 'overreach' ? String(value) : formatRate(value);
|
|||
|
|
lines.push(`| ${threshold.label} | ${shown} | ${threshold.target} | ${threshold.pass(value) ? '达标' : '未达标'} |`);
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('## 未通过用例');
|
|||
|
|
lines.push('');
|
|||
|
|
if (failed.length === 0) {
|
|||
|
|
lines.push('无。');
|
|||
|
|
} else {
|
|||
|
|
lines.push('| 用例 | 类型 | 问法 | 期望 | 实际 | 说明 |');
|
|||
|
|
lines.push('| --- | --- | --- | --- | --- | --- |');
|
|||
|
|
for (const item of failed) {
|
|||
|
|
const expected = item.expected.behavior === '调用' ? `调用 ${item.expected.capability}` : item.expected.behavior;
|
|||
|
|
const actual = item.decision.behavior === '调用' ? `调用 ${item.decision.capability}` : item.decision.behavior;
|
|||
|
|
lines.push(`| ${item.testCase.id} | ${item.testCase.kind} | ${item.testCase.utterance} | ${expected} | ${actual} | ${item.detail} |`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('## 怎么用这份报告');
|
|||
|
|
lines.push('');
|
|||
|
|
lines.push('- 门槛未达标不阻塞开发，但要逐条记录结论：是改能力描述、改用例，还是等真引擎。');
|
|||
|
|
lines.push('- 接上 Pi 之后用同一套用例与方法再跑一遍，与这份基线对比（docs/eval/README.md 第 4 节）。');
|
|||
|
|
lines.push('');
|
|||
|
|
|
|||
|
|
const reportPath = join(ROOT, 'docs', 'eval', `report-${new Date().toISOString().slice(0, 10)}.md`);
|
|||
|
|
await writeFile(reportPath, lines.join('\n'), 'utf8');
|
|||
|
|
|
|||
|
|
console.log(`用例 ${cases.length} 条，能力 ${tools.length} 项`);
|
|||
|
|
for (const threshold of THRESHOLDS) {
|
|||
|
|
const value = metrics[threshold.key];
|
|||
|
|
const shown = threshold.key === 'hallucination' || threshold.key === 'overreach' ? String(value) : formatRate(value);
|
|||
|
|
console.log(` ${threshold.label.padEnd(12, ' ')} ${shown.padStart(8, ' ')} 门槛 ${threshold.target} ${threshold.pass(value) ? '达标' : '未达标'}`);
|
|||
|
|
}
|
|||
|
|
console.log(`未通过 ${failed.length} 条；报告已写入 ${relative(ROOT, reportPath)}`);
|
|||
|
|
void require;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
await main();
|