feat(agent): 引擎接口、假引擎与一致性测试套件(A3.1 / A3.2)

runtime.ts 按架构设计 5.2 定义 AgentRuntime / RuntimeSession / RuntimeEvent(text、tool_call、tool_result、approval_required、turn_end、error),补两处实现必需项:createSession 带 sessionId、resolveToolResult 让工具回执回流。fake-runtime 为确定性假引擎,支持脚本与计划器,事件顺序与真引擎一致,用于一致性测试与 A2.7 评测。conformance.ts 是一套跑任何运行时的验收用例:单轮事件顺序、多轮、工具调用与回执、取消、错误分类、带历史恢复。测试 44 → 46 全绿;坏运行时负例证明套件不是摆设。过程中修掉两个自身缺陷:假引擎在 yield 之后才登记回执回调导致回执丢失;一致性套件超时定时器 unref 导致事件循环空转退出。
This commit is contained in:
zh 2026-09-28 15:14:11 +08:00
parent 6fe43f1d4b
commit 89b9535138
5 changed files with 518 additions and 0 deletions

View File

@ -0,0 +1,75 @@
/**
* 引擎一致性测试(任务 A3.2)。
*
* 同一套用例先跑假引擎,A3.3 接真 Pi 后跑同一套(A3.9 引擎可替换验证也用它)。
* 第二个用例故意给一个坏运行时,证明这套检查不是摆设。
*/
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { runRuntimeConformance, createTestTools } from '@aps/server/agent/conformance';
import { createFakeRuntime } from '@aps/server/agent/fake-runtime';
/** 各场景的脚本:假引擎按这个产出事件 */
function turnsFor(scenario) {
switch (scenario) {
case 'tool_call':
return [
{
text: ['我先查一下。'],
toolCalls: [{ capabilityId: 'test.echo', args: { query: '下周排产' } }],
afterToolsText: ['查到了,共 3 条。'],
},
];
case 'cancel':
return [{ text: ['开始处理', '继续处理'] }];
case 'error':
return [{ error: { kind: 'unavailable', message: '上游不可用' } }];
default:
return [{ text: ['我是假引擎,'], afterToolsText: [] }];
}
}
test('假引擎通过一致性测试', async () => {
const report = await runRuntimeConformance({
createRuntime: (scenario) => createFakeRuntime({ name: 'fake', turns: turnsFor(scenario) }),
tools: createTestTools(),
});
assert.equal(report.runtimeName, 'fake');
assert.equal(report.ok, true, JSON.stringify(report.checks.filter((check) => !check.ok), null, 2));
assert.ok(report.checks.length >= 10, `检查项应有十条以上,实际 ${report.checks.length}`);
});
test('坏运行时被一致性测试拦下(负例)', async () => {
const broken = {
name: 'broken',
async createSession() {
return {
id: 'broken-1',
async *send(_text, options) {
if (options?.signal?.aborted) {
// 取消场景:仍然按 stop 收尾,属于违规
yield { type: 'turn_end', reason: 'stop' };
return;
}
yield { type: 'text', delta: '只说一句' };
yield { type: 'turn_end', reason: 'stop' };
},
resolveToolResult() {
/* 不回填:工具回合永远拿不到 tool_result */
},
abort() {
/* 忽略取消 */
},
async dispose() {},
};
},
};
const report = await runRuntimeConformance({ createRuntime: () => broken, tools: createTestTools() });
assert.equal(report.ok, false, '坏运行时不应当通过');
const failed = report.checks.filter((check) => !check.ok).map((check) => check.name);
assert.ok(failed.includes('取消后以 turn_end(cancelled) 结束'), `应检出取消违规,实际:${failed.join('、')}`);
assert.ok(failed.includes('回执回流后产出 tool_result'), `应检出工具回执未回流,实际:${failed.join('、')}`);
assert.ok(failed.includes('错误事件带合法分类'), `应检出缺少错误事件,实际:${failed.join('、')}`);
});

203
server/agent/conformance.ts Normal file
View File

@ -0,0 +1,203 @@
/**
* 引擎一致性测试套件(任务 A3.2)。
*
* 同一套用例跑任何运行时实现:多轮对话、工具调用与回执、事件顺序、取消、错误分类、会话恢复
* (架构设计 5.2 的验收方式)。FakeRuntime 现在跑,PiRuntime 在 A3.3 接上后跑同一套;
* A3.9「引擎可替换验证」也是拿它做判据。
*/
import type { ExecutionReceipt } from '@aps/contracts';
import type { AgentRuntime, RuntimeEvent, RuntimeSession, ToolDescriptor } from './runtime.js';
export type RuntimeScenario = 'text' | 'tool_call' | 'cancel' | 'error';
export interface ConformanceCheck {
name: string;
ok: boolean;
detail?: string;
}
export interface ConformanceReport {
runtimeName: string;
ok: boolean;
checks: ConformanceCheck[];
}
export interface ConformanceOptions {
/** 为某个场景创建一个运行时;同一场景可能创建多次 */
createRuntime: (scenario: RuntimeScenario) => AgentRuntime | Promise<AgentRuntime>;
/** 能力目录投影 */
tools: ToolDescriptor[];
/** 各场景的输入文本;真引擎用真实问法,假引擎用任意文本 */
prompts?: Partial<Record<RuntimeScenario, string>>;
/** 单次 send 的超时(毫秒),防止实现挂住测试 */
sendTimeoutMs?: number;
}
const DEFAULT_PROMPTS: Record<RuntimeScenario, string> = {
text: '你好,介绍一下你自己',
tool_call: '查一下下周的排产结果',
cancel: '跑一个很长的任务',
error: '触发一次错误',
};
const CAPABILITY_FOR_SCENARIO: Partial<Record<RuntimeScenario, string>> = {
tool_call: 'test.echo',
};
export function createTestTools(): ToolDescriptor[] {
return [
{ capabilityId: 'test.echo', toolName: 'test_echo', description: '回显', parameters: { type: 'object' } },
{ capabilityId: 'test.write', toolName: 'test_write', description: '假写入', parameters: { type: 'object' } },
];
}
/** 单次迭代也要有超时:真引擎挂住时不能让测试跟着挂住 */
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
return Promise.race([
promise,
new Promise<T>((_resolve, reject) => {
setTimeout(() => reject(new Error(`${label} 超过 ${timeoutMs}ms 未返回`)), timeoutMs).unref();
}),
]);
}
async function collect(
events: AsyncIterable<RuntimeEvent>,
timeoutMs: number,
hooks?: { onToolCall?: (callId: string) => void },
): Promise<RuntimeEvent[]> {
const collected: RuntimeEvent[] = [];
const iterator = events[Symbol.asyncIterator]();
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const next = await withTimeout(iterator.next(), Math.max(50, deadline - Date.now()), '等待运行时事件');
if (next.done) return collected;
const event = next.value as RuntimeEvent;
collected.push(event);
if (event.type === 'tool_call') hooks?.onToolCall?.(event.callId);
if (event.type === 'turn_end') return collected;
}
throw new Error(`send 未在 ${timeoutMs}ms 内结束;已收到:${collected.map((e) => e.type).join(',') || '(无)'}`);
}
export async function runRuntimeConformance(options: ConformanceOptions): Promise<ConformanceReport> {
const timeoutMs = options.sendTimeoutMs ?? 5_000;
const prompts = { ...DEFAULT_PROMPTS, ...options.prompts };
const checks: ConformanceCheck[] = [];
const record = (name: string, ok: boolean, detail?: string) => {
checks.push(detail === undefined ? { name, ok } : { name, ok, detail });
};
let runtimeName = '(未知)';
let session: RuntimeSession | undefined;
try {
const runtime = await options.createRuntime('text');
runtimeName = runtime.name;
session = await runtime.createSession({ sessionId: 'conformance-1', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools });
record('createSession 返回会话', typeof session.id === 'string' && session.id.length > 0);
// 1) 单轮文本:事件顺序与结束原因
const textEvents = await collect(session.send(prompts.text), timeoutMs);
record('单轮产出文本并正常结束', textEvents.some((e) => e.type === 'text') && textEvents.at(-1)?.type === 'turn_end');
const last = textEvents.at(-1);
record('结束原因为 stop', last?.type === 'turn_end' && last.reason === 'stop', last?.type === 'turn_end' ? `reason=${last.reason}` : '缺少 turn_end');
const endIndex = textEvents.findIndex((e) => e.type === 'turn_end');
record('turn_end 是最后一个事件', endIndex === textEvents.length - 1, `turn_end 在第 ${endIndex + 1}/${textEvents.length} 个`);
// 2) 多轮:同一会话连续两轮
const secondTurn = await collect(session.send(prompts.text), timeoutMs);
record('同一会话可多轮', secondTurn.at(-1)?.type === 'turn_end' && session.id === session.id);
} catch (error) {
record('基础回合(文本与多轮)', false, error instanceof Error ? error.message : '未知错误');
}
// 3) 工具调用与回执
try {
const runtime = await options.createRuntime('tool_call');
const toolSession = await runtime.createSession({ sessionId: 'conformance-tools', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools });
let toolCallSeen = false;
let receiptSeen = false;
const capabilityId = CAPABILITY_FOR_SCENARIO.tool_call ?? options.tools[0]?.capabilityId ?? '';
const events = await collect(toolSession.send(prompts.tool_call), timeoutMs, {
onToolCall: (callId) => {
toolCallSeen = true;
const receipt: ExecutionReceipt = { executionId: `exec-${callId}`, status: 'succeeded', result: { ok: true } };
toolSession.resolveToolResult(callId, receipt);
},
});
receiptSeen = events.some((e) => e.type === 'tool_result');
const callEvent = events.find((e) => e.type === 'tool_call');
record('发出 tool_call 并带能力 id', toolCallSeen && callEvent?.type === 'tool_call' && callEvent.capabilityId === capabilityId, `期望 ${capabilityId}`);
record('回执回流后产出 tool_result', receiptSeen);
record('工具回合以 turn_end 收尾', events.at(-1)?.type === 'turn_end');
await toolSession.dispose();
} catch (error) {
record('工具调用与回执', false, error instanceof Error ? error.message : '未知错误');
}
// 4) 取消
try {
const runtime = await options.createRuntime('cancel');
const cancelSession = await runtime.createSession({ sessionId: 'conformance-cancel', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools });
const controller = new AbortController();
const events: RuntimeEvent[] = [];
const iterator = cancelSession.send(prompts.cancel, { signal: controller.signal })[Symbol.asyncIterator]();
cancelSession.abort();
controller.abort();
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const next = await withTimeout(iterator.next(), timeoutMs, '等待取消后的事件');
if (next.done) break;
events.push(next.value as RuntimeEvent);
if ((next.value as RuntimeEvent).type === 'turn_end') break;
}
const end = events.at(-1);
record('取消后以 turn_end(cancelled) 结束', end?.type === 'turn_end' && end.reason === 'cancelled', end?.type === 'turn_end' ? `reason=${end.reason}` : '缺少 turn_end');
await cancelSession.dispose();
} catch (error) {
record('取消', false, error instanceof Error ? error.message : '未知错误');
}
// 5) 错误分类
try {
const runtime = await options.createRuntime('error');
const errorSession = await runtime.createSession({ sessionId: 'conformance-error', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools });
const events = await collect(errorSession.send(prompts.error), timeoutMs);
const errorEvent = events.find((e) => e.type === 'error');
const validKinds = ['invalid_argument', 'forbidden', 'conflict', 'rate_limited', 'unavailable', 'unknown_result'];
record(
'错误事件带合法分类',
errorEvent?.type === 'error' && validKinds.includes(errorEvent.kind),
errorEvent?.type === 'error' ? `kind=${errorEvent.kind}` : '未收到 error 事件',
);
await errorSession.dispose();
} catch (error) {
record('错误分类', false, error instanceof Error ? error.message : '未知错误');
}
// 6) 会话恢复:用平台历史创建新会话并继续
try {
const runtime = await options.createRuntime('text');
const restored = await runtime.createSession({
sessionId: 'conformance-restore',
subject: { userId: 'u', tenantId: 't' },
projectId: null,
tools: options.tools,
history: [
{ messageId: 'm-1', role: 'user', ts: Date.now(), text: '之前的问题' },
{ messageId: 'm-2', role: 'assistant', ts: Date.now(), text: '之前的回答' },
],
});
const events = await collect(restored.send(prompts.text), timeoutMs);
record('带历史恢复后仍可对话', events.at(-1)?.type === 'turn_end');
await restored.dispose();
} catch (error) {
record('会话恢复', false, error instanceof Error ? error.message : '未知错误');
}
if (session) await session.dispose();
return { runtimeName, ok: checks.every((check) => check.ok), checks };
}

View File

@ -0,0 +1,131 @@
/**
* 假引擎(任务 A3.2):确定性的运行时,用于一致性测试与问法评测(A2.7)。
*
* 它不做任何「理解」:按脚本或计划器产出事件,形状与真引擎一致。
* 这样一致性测试可以断言事件顺序,评测装置也能在无模型的情况下跑通。
*/
import { randomUUID } from 'node:crypto';
import type { ExecutionReceipt } from '@aps/contracts';
import type { ErrorKind } from '@aps/provider-sdk';
import type { AgentRuntime, CreateRuntimeSessionInput, RuntimeEvent, RuntimeSession, SendOptions, ToolDescriptor } from './runtime.js';
/** 一个回合的脚本:先说话、再调工具、拿到回执后继续说话,或直接报错 */
export interface FakeTurn {
text?: string[];
toolCalls?: Array<{ capabilityId: string; args: unknown }>;
afterToolsText?: string[];
error?: { kind: ErrorKind; message: string };
/** 事件之间的间隔,默认 0(测试要快) */
delayMs?: number;
}
export interface FakeRuntimeOptions {
name?: string;
/** 按回合顺序播放;用完后回到默认行为 */
turns?: FakeTurn[];
/** 或者由计划器决定每个回合怎么走 */
planner?: (text: string, tools: ToolDescriptor[]) => FakeTurn;
}
export interface FakeRuntime extends AgentRuntime {
stats: { sessions: number; turns: number; aborts: number };
}
const sleep = (ms: number) => (ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve());
export function createFakeRuntime(options: FakeRuntimeOptions = {}): FakeRuntime {
const stats = { sessions: 0, turns: 0, aborts: 0 };
const scriptedTurns = [...(options.turns ?? [])];
return {
name: options.name ?? 'fake',
stats,
async createSession(input: CreateRuntimeSessionInput): Promise<RuntimeSession> {
stats.sessions += 1;
const id = `fake-${input.sessionId}-${randomUUID().slice(0, 8)}`;
const pending = new Map<string, (receipt: ExecutionReceipt) => void>();
let aborted = false;
let disposed = false;
const nextTurn = (text: string): FakeTurn => {
const scripted = scriptedTurns.shift();
if (scripted) return scripted;
if (options.planner) return options.planner(text, input.tools);
return { text: [`(假引擎)收到:${text}`] };
};
async function* runTurn(text: string, sendOptions?: SendOptions): AsyncGenerator<RuntimeEvent> {
if (disposed) throw new Error('运行时已释放');
aborted = false;
stats.turns += 1;
const turn = nextTurn(text);
const delay = turn.delayMs ?? 0;
const checkAbort = () => aborted || sendOptions?.signal?.aborted === true;
if (checkAbort()) {
yield { type: 'turn_end', reason: 'cancelled' };
return;
}
for (const delta of turn.text ?? []) {
yield { type: 'text', delta };
}
if (turn.error) {
yield { type: 'error', kind: turn.error.kind, message: turn.error.message };
yield { type: 'turn_end', reason: 'error' };
return;
}
for (const call of turn.toolCalls ?? []) {
const callId = `call-${randomUUID().slice(0, 8)}`;
// 先登记回调再 yield:消费者可能在收到事件的同一刻就把回执送回来
const receiptPromise = new Promise<ExecutionReceipt>((resolve) => {
pending.set(callId, resolve);
});
yield { type: 'tool_call', callId, capabilityId: call.capabilityId, args: call.args };
// 等平台把回执送回来(Pi 走扩展桥,这里走内存回调)
const receipt = await receiptPromise;
pending.delete(callId);
yield { type: 'tool_result', callId, receipt };
if (checkAbort()) {
yield { type: 'turn_end', reason: 'cancelled' };
return;
}
}
await sleep(delay);
for (const delta of turn.afterToolsText ?? []) {
yield { type: 'text', delta };
}
yield { type: 'turn_end', reason: checkAbort() ? 'cancelled' : 'stop', usage: { turns: 1 } };
}
return {
id,
send(text: string, sendOptions?: SendOptions): AsyncIterable<RuntimeEvent> {
return runTurn(text, sendOptions);
},
resolveToolResult(callId: string, receipt: ExecutionReceipt): void {
const resolve = pending.get(callId);
if (resolve) resolve(receipt);
},
abort(): void {
stats.aborts += 1;
aborted = true;
for (const resolve of pending.values()) resolve({ executionId: 'aborted', status: 'failed', errorKind: 'conflict' });
pending.clear();
},
async dispose(): Promise<void> {
disposed = true;
pending.clear();
},
};
},
};
}

89
server/agent/runtime.ts Normal file
View File

@ -0,0 +1,89 @@
/**
* 引擎接口(任务 A3.1,架构设计 5.2)。
*
* 接口只描述「怎么驱动一个循环」,不描述「允许做什么」——权限与门禁全在 Core。
* 三个固定点:领域类型、工具出口(Core.invoke)、状态(平台保存)。
*
* 两个实现:FakeRuntime(A3.2,用于一致性测试与评测)、PiRuntime(A3.3,长驻子进程)。
* 与设计文档的两处补充:
* 1. createSession 带 sessionId,便于适配层关联平台会话
* 2. RuntimeSession 增加 resolveToolResult——工具回执要能回流(Pi 由扩展桥触发,假引擎走内存回调)
*/
import type { ChatMessage, ExecutionReceipt } from '@aps/contracts';
import type { ErrorKind } from '@aps/provider-sdk';
/** 工具描述:能力目录向引擎的投影(只有引擎需要的那部分) */
export interface ToolDescriptor {
/** 能力 id,如 aps.schedule.preview */
capabilityId: string;
/** 引擎侧的工具名(Pi/Codex 的工具名规则不同,由适配层翻译) */
toolName: string;
description: string;
parameters: Record<string, unknown>;
}
export interface RuntimeUsage {
inputTokens?: number;
outputTokens?: number;
turns?: number;
}
/** 运行时提出的确认请求:与协议 v1 的 approval.requested 数据一致 */
export interface RuntimeApprovalRequest {
approvalId: string;
capabilityId: string;
paramsDigest: string;
impactSummary: string;
revision: number;
expiresAt: number;
}
interface RuntimeEventMap {
text: { delta: string };
tool_call: { callId: string; capabilityId: string; args: unknown };
tool_result: { callId: string; receipt: ExecutionReceipt };
approval_required: { request: RuntimeApprovalRequest };
turn_end: { reason: 'stop' | 'cancelled' | 'error'; usage?: RuntimeUsage };
error: { kind: ErrorKind; message: string };
}
/** 统一运行时事件:客户端与 Core 都不感知引擎原生协议 */
export type RuntimeEvent = {
[K in keyof RuntimeEventMap]: { type: K } & RuntimeEventMap[K];
}[keyof RuntimeEventMap];
export type RuntimeEventType = RuntimeEvent['type'];
export interface CreateRuntimeSessionInput {
/** 平台会话 id,用于适配层关联 */
sessionId: string;
subject: { userId: string; tenantId: string };
projectId: string | null;
/** 系统提示词;A3.1 起由应用服务提供 */
system?: string;
/** 能力目录投影 */
tools: ToolDescriptor[];
/** 平台保存的历史:换进程或重启后据此恢复(架构设计 5.1「会话落库」) */
history?: ChatMessage[];
}
export interface SendOptions {
signal?: AbortSignal;
}
export interface RuntimeSession {
readonly id: string;
/** 发一轮输入,产出事件流;流结束表示这一回合结束 */
send(text: string, options?: SendOptions): AsyncIterable<RuntimeEvent>;
/** 工具执行完回填回执,运行时据此继续生成后续事件 */
resolveToolResult(callId: string, receipt: ExecutionReceipt): void;
/** 取消当前回合 */
abort(): void;
/** 释放运行时资源(子进程、连接) */
dispose(): Promise<void>;
}
export interface AgentRuntime {
readonly name: string;
createSession(input: CreateRuntimeSessionInput): Promise<RuntimeSession>;
}

View File

@ -9,6 +9,26 @@
".": { ".": {
"types": "./dist/index.d.ts", "types": "./dist/index.d.ts",
"default": "./dist/index.js" "default": "./dist/index.js"
},
"./agent/runtime": {
"types": "./dist/agent/runtime.d.ts",
"default": "./dist/agent/runtime.js"
},
"./agent/fake-runtime": {
"types": "./dist/agent/fake-runtime.d.ts",
"default": "./dist/agent/fake-runtime.js"
},
"./agent/conformance": {
"types": "./dist/agent/conformance.d.ts",
"default": "./dist/agent/conformance.js"
},
"./core/execution/invoke": {
"types": "./dist/core/execution/invoke.d.ts",
"default": "./dist/core/execution/invoke.js"
},
"./core/registry/registry": {
"types": "./dist/core/registry/registry.d.ts",
"default": "./dist/core/registry/registry.js"
} }
}, },
"scripts": { "scripts": {