diff --git a/scripts/runtime-conformance.test.mjs b/scripts/runtime-conformance.test.mjs new file mode 100644 index 0000000..e1bf727 --- /dev/null +++ b/scripts/runtime-conformance.test.mjs @@ -0,0 +1,75 @@ +/** + * 引擎一致性测试(任务 A3.2)。 + * + * 同一套用例先跑假引擎,A3.3 接真 Pi 后跑同一套(A3.9 引擎可替换验证也用它)。 + * 第二个用例故意给一个坏运行时,证明这套检查不是摆设。 + */ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { runRuntimeConformance, createTestTools } from '@aps/server/agent/conformance'; +import { createFakeRuntime } from '@aps/server/agent/fake-runtime'; + +/** 各场景的脚本:假引擎按这个产出事件 */ +function turnsFor(scenario) { + switch (scenario) { + case 'tool_call': + return [ + { + text: ['我先查一下。'], + toolCalls: [{ capabilityId: 'test.echo', args: { query: '下周排产' } }], + afterToolsText: ['查到了,共 3 条。'], + }, + ]; + case 'cancel': + return [{ text: ['开始处理', '继续处理'] }]; + case 'error': + return [{ error: { kind: 'unavailable', message: '上游不可用' } }]; + default: + return [{ text: ['我是假引擎,'], afterToolsText: [] }]; + } +} + +test('假引擎通过一致性测试', async () => { + const report = await runRuntimeConformance({ + createRuntime: (scenario) => createFakeRuntime({ name: 'fake', turns: turnsFor(scenario) }), + tools: createTestTools(), + }); + assert.equal(report.runtimeName, 'fake'); + assert.equal(report.ok, true, JSON.stringify(report.checks.filter((check) => !check.ok), null, 2)); + assert.ok(report.checks.length >= 10, `检查项应有十条以上,实际 ${report.checks.length}`); +}); + +test('坏运行时被一致性测试拦下(负例)', async () => { + const broken = { + name: 'broken', + async createSession() { + return { + id: 'broken-1', + async *send(_text, options) { + if (options?.signal?.aborted) { + // 取消场景:仍然按 stop 收尾,属于违规 + yield { type: 'turn_end', reason: 'stop' }; + return; + } + yield { type: 'text', delta: '只说一句' }; + yield { type: 'turn_end', reason: 'stop' }; + }, + resolveToolResult() { + /* 不回填:工具回合永远拿不到 tool_result */ + }, + abort() { + /* 忽略取消 */ + }, + async dispose() {}, + }; + }, + }; + + const report = await runRuntimeConformance({ createRuntime: () => broken, tools: createTestTools() }); + assert.equal(report.ok, false, '坏运行时不应当通过'); + const failed = report.checks.filter((check) => !check.ok).map((check) => check.name); + assert.ok(failed.includes('取消后以 turn_end(cancelled) 结束'), `应检出取消违规,实际:${failed.join('、')}`); + assert.ok(failed.includes('回执回流后产出 tool_result'), `应检出工具回执未回流,实际:${failed.join('、')}`); + assert.ok(failed.includes('错误事件带合法分类'), `应检出缺少错误事件,实际:${failed.join('、')}`); +}); \ No newline at end of file diff --git a/server/agent/conformance.ts b/server/agent/conformance.ts new file mode 100644 index 0000000..401c1c4 --- /dev/null +++ b/server/agent/conformance.ts @@ -0,0 +1,203 @@ +/** + * 引擎一致性测试套件(任务 A3.2)。 + * + * 同一套用例跑任何运行时实现:多轮对话、工具调用与回执、事件顺序、取消、错误分类、会话恢复 + * (架构设计 5.2 的验收方式)。FakeRuntime 现在跑,PiRuntime 在 A3.3 接上后跑同一套; + * A3.9「引擎可替换验证」也是拿它做判据。 + */ +import type { ExecutionReceipt } from '@aps/contracts'; + +import type { AgentRuntime, RuntimeEvent, RuntimeSession, ToolDescriptor } from './runtime.js'; + +export type RuntimeScenario = 'text' | 'tool_call' | 'cancel' | 'error'; + +export interface ConformanceCheck { + name: string; + ok: boolean; + detail?: string; +} + +export interface ConformanceReport { + runtimeName: string; + ok: boolean; + checks: ConformanceCheck[]; +} + +export interface ConformanceOptions { + /** 为某个场景创建一个运行时;同一场景可能创建多次 */ + createRuntime: (scenario: RuntimeScenario) => AgentRuntime | Promise; + /** 能力目录投影 */ + tools: ToolDescriptor[]; + /** 各场景的输入文本;真引擎用真实问法,假引擎用任意文本 */ + prompts?: Partial>; + /** 单次 send 的超时(毫秒),防止实现挂住测试 */ + sendTimeoutMs?: number; +} + +const DEFAULT_PROMPTS: Record = { + text: '你好,介绍一下你自己', + tool_call: '查一下下周的排产结果', + cancel: '跑一个很长的任务', + error: '触发一次错误', +}; + +const CAPABILITY_FOR_SCENARIO: Partial> = { + tool_call: 'test.echo', +}; + +export function createTestTools(): ToolDescriptor[] { + return [ + { capabilityId: 'test.echo', toolName: 'test_echo', description: '回显', parameters: { type: 'object' } }, + { capabilityId: 'test.write', toolName: 'test_write', description: '假写入', parameters: { type: 'object' } }, + ]; +} + +/** 单次迭代也要有超时:真引擎挂住时不能让测试跟着挂住 */ +function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error(`${label} 超过 ${timeoutMs}ms 未返回`)), timeoutMs).unref(); + }), + ]); +} + +async function collect( + events: AsyncIterable, + timeoutMs: number, + hooks?: { onToolCall?: (callId: string) => void }, +): Promise { + const collected: RuntimeEvent[] = []; + const iterator = events[Symbol.asyncIterator](); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const next = await withTimeout(iterator.next(), Math.max(50, deadline - Date.now()), '等待运行时事件'); + if (next.done) return collected; + const event = next.value as RuntimeEvent; + collected.push(event); + if (event.type === 'tool_call') hooks?.onToolCall?.(event.callId); + if (event.type === 'turn_end') return collected; + } + throw new Error(`send 未在 ${timeoutMs}ms 内结束;已收到:${collected.map((e) => e.type).join(',') || '(无)'}`); +} + +export async function runRuntimeConformance(options: ConformanceOptions): Promise { + const timeoutMs = options.sendTimeoutMs ?? 5_000; + const prompts = { ...DEFAULT_PROMPTS, ...options.prompts }; + const checks: ConformanceCheck[] = []; + const record = (name: string, ok: boolean, detail?: string) => { + checks.push(detail === undefined ? { name, ok } : { name, ok, detail }); + }; + + let runtimeName = '(未知)'; + let session: RuntimeSession | undefined; + + try { + const runtime = await options.createRuntime('text'); + runtimeName = runtime.name; + session = await runtime.createSession({ sessionId: 'conformance-1', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + record('createSession 返回会话', typeof session.id === 'string' && session.id.length > 0); + + // 1) 单轮文本:事件顺序与结束原因 + const textEvents = await collect(session.send(prompts.text), timeoutMs); + record('单轮产出文本并正常结束', textEvents.some((e) => e.type === 'text') && textEvents.at(-1)?.type === 'turn_end'); + const last = textEvents.at(-1); + record('结束原因为 stop', last?.type === 'turn_end' && last.reason === 'stop', last?.type === 'turn_end' ? `reason=${last.reason}` : '缺少 turn_end'); + const endIndex = textEvents.findIndex((e) => e.type === 'turn_end'); + record('turn_end 是最后一个事件', endIndex === textEvents.length - 1, `turn_end 在第 ${endIndex + 1}/${textEvents.length} 个`); + + // 2) 多轮:同一会话连续两轮 + const secondTurn = await collect(session.send(prompts.text), timeoutMs); + record('同一会话可多轮', secondTurn.at(-1)?.type === 'turn_end' && session.id === session.id); + } catch (error) { + record('基础回合(文本与多轮)', false, error instanceof Error ? error.message : '未知错误'); + } + + // 3) 工具调用与回执 + try { + const runtime = await options.createRuntime('tool_call'); + const toolSession = await runtime.createSession({ sessionId: 'conformance-tools', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + let toolCallSeen = false; + let receiptSeen = false; + const capabilityId = CAPABILITY_FOR_SCENARIO.tool_call ?? options.tools[0]?.capabilityId ?? ''; + const events = await collect(toolSession.send(prompts.tool_call), timeoutMs, { + onToolCall: (callId) => { + toolCallSeen = true; + const receipt: ExecutionReceipt = { executionId: `exec-${callId}`, status: 'succeeded', result: { ok: true } }; + toolSession.resolveToolResult(callId, receipt); + }, + }); + receiptSeen = events.some((e) => e.type === 'tool_result'); + const callEvent = events.find((e) => e.type === 'tool_call'); + record('发出 tool_call 并带能力 id', toolCallSeen && callEvent?.type === 'tool_call' && callEvent.capabilityId === capabilityId, `期望 ${capabilityId}`); + record('回执回流后产出 tool_result', receiptSeen); + record('工具回合以 turn_end 收尾', events.at(-1)?.type === 'turn_end'); + await toolSession.dispose(); + } catch (error) { + record('工具调用与回执', false, error instanceof Error ? error.message : '未知错误'); + } + + // 4) 取消 + try { + const runtime = await options.createRuntime('cancel'); + const cancelSession = await runtime.createSession({ sessionId: 'conformance-cancel', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + const controller = new AbortController(); + const events: RuntimeEvent[] = []; + const iterator = cancelSession.send(prompts.cancel, { signal: controller.signal })[Symbol.asyncIterator](); + cancelSession.abort(); + controller.abort(); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const next = await withTimeout(iterator.next(), timeoutMs, '等待取消后的事件'); + if (next.done) break; + events.push(next.value as RuntimeEvent); + if ((next.value as RuntimeEvent).type === 'turn_end') break; + } + const end = events.at(-1); + record('取消后以 turn_end(cancelled) 结束', end?.type === 'turn_end' && end.reason === 'cancelled', end?.type === 'turn_end' ? `reason=${end.reason}` : '缺少 turn_end'); + await cancelSession.dispose(); + } catch (error) { + record('取消', false, error instanceof Error ? error.message : '未知错误'); + } + + // 5) 错误分类 + try { + const runtime = await options.createRuntime('error'); + const errorSession = await runtime.createSession({ sessionId: 'conformance-error', subject: { userId: 'u', tenantId: 't' }, projectId: null, tools: options.tools }); + const events = await collect(errorSession.send(prompts.error), timeoutMs); + const errorEvent = events.find((e) => e.type === 'error'); + const validKinds = ['invalid_argument', 'forbidden', 'conflict', 'rate_limited', 'unavailable', 'unknown_result']; + record( + '错误事件带合法分类', + errorEvent?.type === 'error' && validKinds.includes(errorEvent.kind), + errorEvent?.type === 'error' ? `kind=${errorEvent.kind}` : '未收到 error 事件', + ); + await errorSession.dispose(); + } catch (error) { + record('错误分类', false, error instanceof Error ? error.message : '未知错误'); + } + + // 6) 会话恢复:用平台历史创建新会话并继续 + try { + const runtime = await options.createRuntime('text'); + const restored = await runtime.createSession({ + sessionId: 'conformance-restore', + subject: { userId: 'u', tenantId: 't' }, + projectId: null, + tools: options.tools, + history: [ + { messageId: 'm-1', role: 'user', ts: Date.now(), text: '之前的问题' }, + { messageId: 'm-2', role: 'assistant', ts: Date.now(), text: '之前的回答' }, + ], + }); + const events = await collect(restored.send(prompts.text), timeoutMs); + record('带历史恢复后仍可对话', events.at(-1)?.type === 'turn_end'); + await restored.dispose(); + } catch (error) { + record('会话恢复', false, error instanceof Error ? error.message : '未知错误'); + } + + if (session) await session.dispose(); + + return { runtimeName, ok: checks.every((check) => check.ok), checks }; +} \ No newline at end of file diff --git a/server/agent/fake-runtime.ts b/server/agent/fake-runtime.ts new file mode 100644 index 0000000..05b1898 --- /dev/null +++ b/server/agent/fake-runtime.ts @@ -0,0 +1,131 @@ +/** + * 假引擎(任务 A3.2):确定性的运行时,用于一致性测试与问法评测(A2.7)。 + * + * 它不做任何「理解」:按脚本或计划器产出事件,形状与真引擎一致。 + * 这样一致性测试可以断言事件顺序,评测装置也能在无模型的情况下跑通。 + */ +import { randomUUID } from 'node:crypto'; + +import type { ExecutionReceipt } from '@aps/contracts'; +import type { ErrorKind } from '@aps/provider-sdk'; + +import type { AgentRuntime, CreateRuntimeSessionInput, RuntimeEvent, RuntimeSession, SendOptions, ToolDescriptor } from './runtime.js'; + +/** 一个回合的脚本:先说话、再调工具、拿到回执后继续说话,或直接报错 */ +export interface FakeTurn { + text?: string[]; + toolCalls?: Array<{ capabilityId: string; args: unknown }>; + afterToolsText?: string[]; + error?: { kind: ErrorKind; message: string }; + /** 事件之间的间隔,默认 0(测试要快) */ + delayMs?: number; +} + +export interface FakeRuntimeOptions { + name?: string; + /** 按回合顺序播放;用完后回到默认行为 */ + turns?: FakeTurn[]; + /** 或者由计划器决定每个回合怎么走 */ + planner?: (text: string, tools: ToolDescriptor[]) => FakeTurn; +} + +export interface FakeRuntime extends AgentRuntime { + stats: { sessions: number; turns: number; aborts: number }; +} + +const sleep = (ms: number) => (ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve()); + +export function createFakeRuntime(options: FakeRuntimeOptions = {}): FakeRuntime { + const stats = { sessions: 0, turns: 0, aborts: 0 }; + const scriptedTurns = [...(options.turns ?? [])]; + + return { + name: options.name ?? 'fake', + stats, + async createSession(input: CreateRuntimeSessionInput): Promise { + stats.sessions += 1; + const id = `fake-${input.sessionId}-${randomUUID().slice(0, 8)}`; + const pending = new Map void>(); + let aborted = false; + let disposed = false; + + const nextTurn = (text: string): FakeTurn => { + const scripted = scriptedTurns.shift(); + if (scripted) return scripted; + if (options.planner) return options.planner(text, input.tools); + return { text: [`(假引擎)收到:${text}`] }; + }; + + async function* runTurn(text: string, sendOptions?: SendOptions): AsyncGenerator { + if (disposed) throw new Error('运行时已释放'); + aborted = false; + stats.turns += 1; + const turn = nextTurn(text); + const delay = turn.delayMs ?? 0; + + const checkAbort = () => aborted || sendOptions?.signal?.aborted === true; + if (checkAbort()) { + yield { type: 'turn_end', reason: 'cancelled' }; + return; + } + + for (const delta of turn.text ?? []) { + yield { type: 'text', delta }; + } + + if (turn.error) { + yield { type: 'error', kind: turn.error.kind, message: turn.error.message }; + yield { type: 'turn_end', reason: 'error' }; + return; + } + + for (const call of turn.toolCalls ?? []) { + const callId = `call-${randomUUID().slice(0, 8)}`; + // 先登记回调再 yield:消费者可能在收到事件的同一刻就把回执送回来 + const receiptPromise = new Promise((resolve) => { + pending.set(callId, resolve); + }); + yield { type: 'tool_call', callId, capabilityId: call.capabilityId, args: call.args }; + + // 等平台把回执送回来(Pi 走扩展桥,这里走内存回调) + const receipt = await receiptPromise; + pending.delete(callId); + yield { type: 'tool_result', callId, receipt }; + + if (checkAbort()) { + yield { type: 'turn_end', reason: 'cancelled' }; + return; + } + } + + await sleep(delay); + for (const delta of turn.afterToolsText ?? []) { + yield { type: 'text', delta }; + } + + yield { type: 'turn_end', reason: checkAbort() ? 'cancelled' : 'stop', usage: { turns: 1 } }; + } + + return { + id, + send(text: string, sendOptions?: SendOptions): AsyncIterable { + return runTurn(text, sendOptions); + }, + resolveToolResult(callId: string, receipt: ExecutionReceipt): void { + const resolve = pending.get(callId); + if (resolve) resolve(receipt); + }, + abort(): void { + stats.aborts += 1; + aborted = true; + for (const resolve of pending.values()) resolve({ executionId: 'aborted', status: 'failed', errorKind: 'conflict' }); + pending.clear(); + }, + async dispose(): Promise { + disposed = true; + pending.clear(); + }, + }; + }, + }; +} \ No newline at end of file diff --git a/server/agent/runtime.ts b/server/agent/runtime.ts new file mode 100644 index 0000000..5d6c3c9 --- /dev/null +++ b/server/agent/runtime.ts @@ -0,0 +1,89 @@ +/** + * 引擎接口(任务 A3.1,架构设计 5.2)。 + * + * 接口只描述「怎么驱动一个循环」,不描述「允许做什么」——权限与门禁全在 Core。 + * 三个固定点:领域类型、工具出口(Core.invoke)、状态(平台保存)。 + * + * 两个实现:FakeRuntime(A3.2,用于一致性测试与评测)、PiRuntime(A3.3,长驻子进程)。 + * 与设计文档的两处补充: + * 1. createSession 带 sessionId,便于适配层关联平台会话 + * 2. RuntimeSession 增加 resolveToolResult——工具回执要能回流(Pi 由扩展桥触发,假引擎走内存回调) + */ +import type { ChatMessage, ExecutionReceipt } from '@aps/contracts'; +import type { ErrorKind } from '@aps/provider-sdk'; + +/** 工具描述:能力目录向引擎的投影(只有引擎需要的那部分) */ +export interface ToolDescriptor { + /** 能力 id,如 aps.schedule.preview */ + capabilityId: string; + /** 引擎侧的工具名(Pi/Codex 的工具名规则不同,由适配层翻译) */ + toolName: string; + description: string; + parameters: Record; +} + +export interface RuntimeUsage { + inputTokens?: number; + outputTokens?: number; + turns?: number; +} + +/** 运行时提出的确认请求:与协议 v1 的 approval.requested 数据一致 */ +export interface RuntimeApprovalRequest { + approvalId: string; + capabilityId: string; + paramsDigest: string; + impactSummary: string; + revision: number; + expiresAt: number; +} + +interface RuntimeEventMap { + text: { delta: string }; + tool_call: { callId: string; capabilityId: string; args: unknown }; + tool_result: { callId: string; receipt: ExecutionReceipt }; + approval_required: { request: RuntimeApprovalRequest }; + turn_end: { reason: 'stop' | 'cancelled' | 'error'; usage?: RuntimeUsage }; + error: { kind: ErrorKind; message: string }; +} + +/** 统一运行时事件:客户端与 Core 都不感知引擎原生协议 */ +export type RuntimeEvent = { + [K in keyof RuntimeEventMap]: { type: K } & RuntimeEventMap[K]; +}[keyof RuntimeEventMap]; + +export type RuntimeEventType = RuntimeEvent['type']; + +export interface CreateRuntimeSessionInput { + /** 平台会话 id,用于适配层关联 */ + sessionId: string; + subject: { userId: string; tenantId: string }; + projectId: string | null; + /** 系统提示词;A3.1 起由应用服务提供 */ + system?: string; + /** 能力目录投影 */ + tools: ToolDescriptor[]; + /** 平台保存的历史:换进程或重启后据此恢复(架构设计 5.1「会话落库」) */ + history?: ChatMessage[]; +} + +export interface SendOptions { + signal?: AbortSignal; +} + +export interface RuntimeSession { + readonly id: string; + /** 发一轮输入,产出事件流;流结束表示这一回合结束 */ + send(text: string, options?: SendOptions): AsyncIterable; + /** 工具执行完回填回执,运行时据此继续生成后续事件 */ + resolveToolResult(callId: string, receipt: ExecutionReceipt): void; + /** 取消当前回合 */ + abort(): void; + /** 释放运行时资源(子进程、连接) */ + dispose(): Promise; +} + +export interface AgentRuntime { + readonly name: string; + createSession(input: CreateRuntimeSessionInput): Promise; +} \ No newline at end of file diff --git a/server/package.json b/server/package.json index 86be7fc..e00c6c5 100644 --- a/server/package.json +++ b/server/package.json @@ -9,6 +9,26 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./agent/runtime": { + "types": "./dist/agent/runtime.d.ts", + "default": "./dist/agent/runtime.js" + }, + "./agent/fake-runtime": { + "types": "./dist/agent/fake-runtime.d.ts", + "default": "./dist/agent/fake-runtime.js" + }, + "./agent/conformance": { + "types": "./dist/agent/conformance.d.ts", + "default": "./dist/agent/conformance.js" + }, + "./core/execution/invoke": { + "types": "./dist/core/execution/invoke.d.ts", + "default": "./dist/core/execution/invoke.js" + }, + "./core/registry/registry": { + "types": "./dist/core/registry/registry.d.ts", + "default": "./dist/core/registry/registry.js" } }, "scripts": {