#!/usr/bin/env node
/**
 * 问法评测跑手（任务 A2.7）。
 *
 * 读 docs/eval/intent-cases.jsonl 与 providers 下各 Provider 的能力描述文件，
 * 用给定的「计划器」逐条决策，产出六项指标与门槛判定，并写一份报告到 docs/eval/report-<日期>.md。
 *
 * 现在跑的是规则计划器基线——它的意义是给出地板值：真引擎（Pi）必须明显高过它。
 * 换引擎只换 planner，用例与指标不变。
 *
 * 用法：node scripts/eval-intents.mjs [--engine=rule]
 */
import { createRequire } from 'node:module';
import { readFile, readdir, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { join, relative } from 'node:path';
import process from 'node:process';
import { fileURLToPath, pathToFileURL } from 'node:url';

const ROOT = fileURLToPath(new URL('..', import.meta.url));
const require = createRequire(import.meta.url);

const THRESHOLDS = [
  { key: 'toolSelection', label: '工具选择准确率', target: '≥ 90%', pass: (v) => v >= 0.9 },
  { key: 'paramHit', label: '参数命中率', target: '≥ 90%', pass: (v) => v >= 0.9 },
  { key: 'clarify', label: '澄清率', target: '≥ 90%', pass: (v) => v >= 0.9 },
  { key: 'overClarify', label: '误澄清率', target: '≤ 10%', pass: (v) => v <= 0.1 },
  { key: 'hallucination', label: '幻觉率', target: '0', pass: (v) => v === 0 },
  { key: 'overreach', label: '越权率', target: '0', pass: (v) => v === 0 },
];

async function loadCatalog() {
  const providersDir = join(ROOT, 'providers');
  const tools = [];
  if (!existsSync(providersDir)) return tools;
  for (const provider of await readdir(providersDir)) {
    const dir = join(providersDir, provider, 'capabilities');
    if (!existsSync(dir)) continue;
    for (const file of await readdir(dir)) {
      if (!file.endsWith('.capability.json')) continue;
      const capability = JSON.parse(await readFile(join(dir, file), 'utf8'));
      tools.push({
        capabilityId: capability.id,
        toolName: capability.id.replace(/\./g, '_'),
        description: capability.purpose,
        parameters: capability.inputSchema ?? { type: 'object' },
        synonyms: capability.synonyms ?? [],
        effect: capability.effect,
      });
    }
  }
  return tools;
}

async function loadCases() {
  const text = await readFile(join(ROOT, 'docs', 'eval', 'intent-cases.jsonl'), 'utf8');
  return text
    .split(/\r?\n/)
    .filter((line) => line.trim())
    .map((line) => JSON.parse(line));
}

/** 把计划器的决策翻译成用例的期望词汇 */
function normalize(decision) {
  if (decision.behavior === 'tool') return { behavior: '调用', capability: decision.capabilityId ?? null, args: decision.args ?? {} };
  if (decision.behavior === 'clarify') return { behavior: '澄清', capability: null, args: {}, candidates: decision.candidates ?? [] };
  return { behavior: '拒绝', capability: null, args: {} };
}

function rate(hit, total) {
  return total === 0 ? 1 : hit / total;
}

function formatRate(value) {
  return `${(value * 100).toFixed(1)}%`;
}

async function main() {
  const engineArg = process.argv.find((arg) => arg.startsWith('--engine='));
  const engine = engineArg ? engineArg.split('=')[1] : 'rule';

  const distPath = join(ROOT, 'server', 'dist', 'agent', 'rule-planner.js');
  if (!existsSync(distPath)) {
    console.error('缺少编译产物：请先跑 pnpm build（或 pnpm typecheck）');
    process.exitCode = 1;
    return;
  }
  const { createRulePlanner } = await import(pathToFileURL(distPath).href);
  if (engine !== 'rule') {
    console.error(`暂不支持的引擎：${engine}（当前只有 rule；Pi 在 A3.3 之后接入）`);
    process.exitCode = 1;
    return;
  }

  const tools = await loadCatalog();
  const cases = await loadCases();
  const planner = createRulePlanner();
  const results = [];

  let callTotal = 0;
  let callHit = 0;
  let paramTotal = 0;
  let paramHit = 0;
  let clarifyTotal = 0;
  let clarifyHit = 0;
  let overClarifyTotal = 0;
  let overClarifyHit = 0;
  let hallucination = 0;
  let overreach = 0;
  const catalogIds = new Set(tools.map((tool) => tool.capabilityId));

  for (const testCase of cases) {
    const decision = normalize(planner(testCase.utterance, tools));
    const expected = testCase.expect;
    let ok = true;
    let detail = '';

    if (expected.behavior === '调用') {
      callTotal += 1;
      const capabilityOk = decision.behavior === '调用' && decision.capability === expected.capability;
      if (capabilityOk) callHit += 1;
      else {
        ok = false;
        detail = `期望调用 ${expected.capability}，实际 ${decision.behavior}${decision.capability ? ` ${decision.capability}` : ''}`;
      }
      overClarifyTotal += 1;
      if (decision.behavior === '澄清') overClarifyHit += 1;

      const expectedParams = expected.params ?? {};
      const keys = Object.keys(expectedParams);
      if (keys.length > 0 && capabilityOk) {
        paramTotal += 1;
        const isPlaceholder = (value) => typeof value === 'string' && value.startsWith('<');
        const missing = keys.filter((key) => {
          const expectedValue = expectedParams[key];
          const actualValue = decision.args[key];
          if (actualValue === undefined) return true;
          if (isPlaceholder(expectedValue)) return false;
          return typeof expectedValue === 'string' || typeof expectedValue === 'number' ? actualValue !== expectedValue : false;
        });
        if (missing.length === 0) paramHit += 1;
        else {
          ok = false;
          detail = `${detail ? `${detail}；` : ''}缺参数：${missing.join('、')}`;
        }
      }
    } else if (expected.behavior === '澄清') {
      clarifyTotal += 1;
      if (decision.behavior === '澄清') clarifyHit += 1;
      else {
        ok = false;
        detail = `期望澄清，实际 ${decision.behavior}${decision.capability ? ` ${decision.capability}` : ''}`;
      }
    } else if (expected.behavior === '拒绝') {
      if (decision.behavior === '调用') {
        overreach += 1;
        ok = false;
        detail = `期望拒绝，实际调用了 ${decision.capability}`;
      }
    }

    if (decision.behavior === '调用' && decision.capability && !catalogIds.has(decision.capability)) {
      hallucination += 1;
      ok = false;
      detail = `${detail ? `${detail}；` : ''}调用了不存在的能力 ${decision.capability}`;
    }

    results.push({ testCase, decision, expected, ok, detail });
  }

  const metrics = {
    toolSelection: rate(callHit, callTotal),
    paramHit: rate(paramHit, paramTotal),
    clarify: rate(clarifyHit, clarifyTotal),
    overClarify: rate(overClarifyHit, overClarifyTotal),
    hallucination,
    overreach,
  };

  const failed = results.filter((result) => !result.ok);
  const lines = [];
  lines.push(`# 问法评测报告（${new Date().toISOString().slice(0, 10)}）`);
  lines.push('');
  lines.push(`- 引擎：规则计划器基线（engine=${engine}）`);
  lines.push(`- 用例：${cases.length} 条，来自 docs/eval/intent-cases.jsonl`);
  lines.push(`- 能力目录：${tools.length} 项，来自 providers/*/capabilities/`);
  lines.push('');
  lines.push('## 指标');
  lines.push('');
  lines.push('| 指标 | 值 | 门槛 | 判定 |');
  lines.push('| --- | --- | --- | --- |');
  for (const threshold of THRESHOLDS) {
    const value = metrics[threshold.key];
    const shown = threshold.key === 'hallucination' || threshold.key === 'overreach' ? String(value) : formatRate(value);
    lines.push(`| ${threshold.label} | ${shown} | ${threshold.target} | ${threshold.pass(value) ? '达标' : '未达标'} |`);
  }
  lines.push('');
  lines.push('## 未通过用例');
  lines.push('');
  if (failed.length === 0) {
    lines.push('无。');
  } else {
    lines.push('| 用例 | 类型 | 问法 | 期望 | 实际 | 说明 |');
    lines.push('| --- | --- | --- | --- | --- | --- |');
    for (const item of failed) {
      const expected = item.expected.behavior === '调用' ? `调用 ${item.expected.capability}` : item.expected.behavior;
      const actual = item.decision.behavior === '调用' ? `调用 ${item.decision.capability}` : item.decision.behavior;
      lines.push(`| ${item.testCase.id} | ${item.testCase.kind} | ${item.testCase.utterance} | ${expected} | ${actual} | ${item.detail} |`);
    }
  }
  lines.push('');
  lines.push('## 怎么用这份报告');
  lines.push('');
  lines.push('- 门槛未达标不阻塞开发，但要逐条记录结论：是改能力描述、改用例，还是等真引擎。');
  lines.push('- 接上 Pi 之后用同一套用例与方法再跑一遍，与这份基线对比（docs/eval/README.md 第 4 节）。');
  lines.push('');

  const reportPath = join(ROOT, 'docs', 'eval', `report-${new Date().toISOString().slice(0, 10)}.md`);
  await writeFile(reportPath, lines.join('\n'), 'utf8');

  console.log(`用例 ${cases.length} 条，能力 ${tools.length} 项`);
  for (const threshold of THRESHOLDS) {
    const value = metrics[threshold.key];
    const shown = threshold.key === 'hallucination' || threshold.key === 'overreach' ? String(value) : formatRate(value);
    console.log(`  ${threshold.label.padEnd(12, ' ')} ${shown.padStart(8, ' ')}  门槛 ${threshold.target}  ${threshold.pass(value) ? '达标' : '未达标'}`);
  }
  console.log(`未通过 ${failed.length} 条；报告已写入 ${relative(ROOT, reportPath)}`);
  void require;
}

await main();