/** * e2e.mjs - real Pi CLI end-to-end validation for the pi-aps extension. * * Chain under test: * real APS HTTP server -> real Pi CLI -> pi-aps extension -> mock LLM (scripted) * * The mock LLM never generates APS data. If the APS server exposes * /api/agent/intents with order.pool, the script asks the model to call * aps_order_pool. Otherwise it falls back to aps_health so the real Pi-to-APS * HTTP chain is still exercised. * * Pi headless always exits 0; verdict is based on JSONL events. */ import { spawn } from "node:child_process"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import net from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; const E2E_DIR = path.dirname(fileURLToPath(import.meta.url)); const PACKAGE_ROOT = path.resolve(E2E_DIR, ".."); const REPO_ROOT = path.resolve(E2E_DIR, "..", "..", ".."); const PI_CLI = path.join( REPO_ROOT, "poc", "pi-fallback", "runtime", "node_modules", "@mariozechner", "pi-coding-agent", "dist", "cli.js", ); const EXTENSION_FILE = path.join(PACKAGE_ROOT, "extensions", "aps.ts"); const MOCK_LLM_FILE = path.join(E2E_DIR, "mock-llm.mjs"); const PI_HOME = path.join(E2E_DIR, "pi-home"); const EVENTS_FILE = path.join(E2E_DIR, "last-events.jsonl"); const SUMMARY_FILE = path.join(E2E_DIR, "last-summary.json"); const MOCK_LOG_FILE = path.join(E2E_DIR, "mock-requests-last.jsonl"); const DEFAULT_APS_URL = "http://127.0.0.1:8000"; const MODEL_PROVIDER = "aps-e2e"; const MODEL_ID = "mock-local"; function env(key, fallback = "") { return process.env[key] || fallback; } function parseArgs() { const out = { apsUrl: env("APS_BASE_URL", DEFAULT_APS_URL).trim().replace(/\/+$/, ""), token: env("APS_AGENT_TOKEN", "").trim() || undefined, configFile: env("APS_CONFIG_FILE", "").trim() || undefined, configOnly: ["1", "true", "yes"].includes( env("APS_E2E_CONFIG_FILE_ONLY", "").trim().toLowerCase(), ), forceTool: env("APS_E2E_TOOL", "").trim() || undefined, timeoutMs: Number(env("APS_E2E_TIMEOUT_MS", "120000")), prompt: env( "APS_E2E_PROMPT", "请用 APS 只读工具查询订单池,拿到真实返回后给出一句中文总结。", ), }; const argv = process.argv.slice(2); for (let i = 0; i < argv.length; i += 1) { if (argv[i] === "--aps-url") out.apsUrl = String(argv[i + 1] || out.apsUrl).replace(/\/+$/, ""); if (argv[i] === "--tool") out.forceTool = argv[i + 1] || out.forceTool; if (argv[i] === "--prompt") out.prompt = argv[i + 1] || out.prompt; if (argv[i] === "--timeout-ms") out.timeoutMs = Number(argv[i + 1] || out.timeoutMs); } return out; } function resolveConfigOnlyPath(configFile) { const resolved = path.resolve(String(configFile || "").trim()); const home = path.resolve(PI_HOME); if (!resolved.startsWith(home + path.sep)) { throw new Error( `[e2e] APS_E2E_CONFIG_FILE_ONLY=1 requires APS_CONFIG_FILE under ${home}`, ); } return resolved; } function normalizeToolName(intentName) { const normalized = String(intentName || "") .trim() .toLowerCase() .replace(/\./g, "_") .replace(/[^a-z0-9_]+/g, "_") .replace(/_+/g, "_") .replace(/^_+|_+$/g, ""); if (!normalized) return undefined; const toolName = `aps_${normalized}`; return /^[a-z][a-z0-9_]*$/.test(toolName) ? toolName : undefined; } async function httpJson(url, options = {}) { const headers = { Accept: "application/json", "Content-Type": "application/json" }; if (options.token) headers.Authorization = `Bearer ${options.token}`; const response = await fetch(url, { headers, signal: AbortSignal.timeout(options.timeoutMs || 8000) }); const text = await response.text(); let body = {}; if (text) { try { body = JSON.parse(text); } catch { body = { raw: text.slice(0, 300) }; } } return { status: response.status, body }; } async function waitForHealth(apsUrl, timeoutMs = 25000) { const started = Date.now(); let lastError = ""; while (Date.now() - started < timeoutMs) { try { const { status, body } = await httpJson(`${apsUrl}/api/health`, { timeoutMs: 4000 }); if (status === 200 && body?.ok !== false) return { status, body }; lastError = `status=${status}`; } catch (err) { lastError = err instanceof Error ? err.message : String(err); } await new Promise((resolve) => setTimeout(resolve, 750)); } throw new Error(`APS server not healthy within ${timeoutMs}ms at ${apsUrl}: ${lastError}`); } async function detectCatalog(apsUrl, token) { const { status, body } = await httpJson(`${apsUrl}/api/agent/intents`, { token, timeoutMs: 6000 }); if (status !== 200) return { available: false, status }; const entries = Array.isArray(body?.intents) ? body.intents : []; if (!entries.length) return { available: true, status, intents: [] }; const explicit = env("APS_E2E_INTENT", "").trim(); let entry = entries.find((item) => item && String(item.name || "").trim() === explicit); if (!entry) entry = entries.find((item) => item && String(item.name || "") === "order.pool"); if (!entry && !explicit) entry = entries[0]; return { available: true, status, intents: entries, selected: entry || null }; } function freePort() { return new Promise((resolve, reject) => { const server = net.createServer(); server.once("error", reject); server.listen(0, "127.0.0.1", () => { const port = server.address().port; server.close(() => resolve(port)); }); }); } async function waitForPortFile(file, expectedPort, timeoutMs = 10000) { const started = Date.now(); while (Date.now() - started < timeoutMs) { try { const text = await readFile(file, "utf8"); const match = String(text).match(/MOCK_LLM_PORT=(\d+)/); if (match && Number(match[1]) === expectedPort) return text; } catch { await new Promise((resolve) => setTimeout(resolve, 200)); } } throw new Error(`mock LLM did not publish port file: ${file}`); } function collectProcess(name, child) { return new Promise((resolve, reject) => { let stdout = ""; let stderr = ""; child.stdout.on("data", (c) => { stdout += c.toString("utf8"); if (stdout.length > 20_000_000) stdout = stdout.slice(-10_000_000); }); child.stderr.on("data", (c) => { stderr += c.toString("utf8"); if (stderr.length > 2_000_000) stderr = stderr.slice(-1_000_000); }); child.on("error", reject); child.on("exit", (code, signal) => resolve({ name, code, signal, stdout, stderr })); }); } function promptForTool(toolName) { if (toolName === "aps_health") { return "请用 aps_health 工具检查 APS 健康状态,拿到真实返回后给出一句中文总结。"; } if (toolName === "aps_summary") { return "请用 aps_summary 工具读取 APS 世界摘要,拿到真实返回后给出一句中文总结。"; } return `请用 ${toolName} 工具查询 APS 只读业务数据,拿到真实返回后给出一句中文总结。`; } function redactOptions(options) { const rest = { apsUrl: options.apsUrl, configOnly: Boolean(options.configOnly), forceTool: options.forceTool, timeoutMs: options.timeoutMs, prompt: options.prompt, configFile: options.configFile ? "configured" : undefined, }; if (!options.token) return { ...rest, token: "not configured" }; return { ...rest, token: "configured (ends " + options.token.slice(-4) + ")" }; } async function main() { const options = parseArgs(); let generatedConfigFile; if (options.configOnly) { if (!options.configFile) { throw new Error( "[e2e] APS_E2E_CONFIG_FILE_ONLY=1 requires APS_CONFIG_FILE", ); } options.configFile = resolveConfigOnlyPath(options.configFile); generatedConfigFile = options.configFile; } const checks = {}; const summary = { options: redactOptions(options), checks, ranAt: new Date().toISOString() }; const mockProcesses = []; try { checks.serverHealth = false; const health = await waitForHealth(options.apsUrl); checks.serverHealth = true; console.log(`[e2e] APS health OK: ${JSON.stringify(health.body)}`); const catalog = await detectCatalog(options.apsUrl, options.token); let targetTool; let catalogIntent; if (catalog.available && catalog.selected) { catalogIntent = String(catalog.selected.name || "").trim(); targetTool = normalizeToolName(catalogIntent) || options.forceTool || "aps_health"; console.log(`[e2e] APS agent catalog available: intent=${catalogIntent} tool=${targetTool}`); } else { targetTool = options.forceTool || "aps_health"; console.log( `[e2e] APS agent catalog unavailable (status=${catalog.status}); ` + `falling back to ${targetTool}`, ); } summary.catalog = { available: Boolean(catalog.available), status: catalog.status, selectedIntent: catalogIntent || null, targetTool, }; const mockPort = await freePort(); const portFile = path.join(E2E_DIR, "mock-port.txt"); await rm(portFile, { force: true }); await rm(MOCK_LOG_FILE, { force: true }); const mockEnv = { ...process.env, APS_E2E_TOOL: targetTool, APS_MOCK_REQUEST_LOG: MOCK_LOG_FILE, NO_PROXY: "127.0.0.1,localhost", no_proxy: "127.0.0.1,localhost", }; for (const key of [ "APS_BASE_URL", "APS_AGENT_TOKEN", "APS_REQUEST_TIMEOUT_MS", "APS_CATALOG_TIMEOUT_MS", "APS_CONFIG_FILE", ]) { delete mockEnv[key]; } const mockChild = spawn( process.execPath, [MOCK_LLM_FILE, "--port", String(mockPort), "--port-file", portFile], { env: mockEnv, stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, ); mockProcesses.push(mockChild); const mockCollect = collectProcess("mock-llm", mockChild); const portText = await waitForPortFile(portFile, mockPort); const actualPort = Number(String(portText).match(/MOCK_LLM_PORT=(\d+)/)?.[1] || mockPort); summary.mockPort = actualPort; console.log(`[e2e] mock LLM on ${actualPort}`); await mkdir(PI_HOME, { recursive: true }); const models = { providers: { [MODEL_PROVIDER]: { baseUrl: `http://127.0.0.1:${actualPort}/v1`, api: "openai-completions", apiKey: "aps-e2e-mock-no-secret", models: [ { id: MODEL_ID, name: "APS E2E Scripted Mock", reasoning: false, input: ["text"], contextWindow: 32768, maxTokens: 2048, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, }, ], }, }, }; await writeFile(path.join(PI_HOME, "models.json"), `${JSON.stringify(models, null, 2)}\n`, "utf8"); await writeFile(path.join(PI_HOME, "auth.json"), "{}\n", "utf8"); if (options.configOnly) { await mkdir(path.dirname(options.configFile), { recursive: true }); const configPayload = { aps: { baseUrl: options.apsUrl, ...(options.token ? { token: options.token } : {}), }, }; await writeFile( options.configFile, `${JSON.stringify(configPayload, null, 2)}\n`, "utf8", ); console.log("[e2e] APS config file mode: token/base URL passed through aps.json"); } const prompt = promptForTool(targetTool); const piArgs = [ PI_CLI, "--mode", "json", "--no-session", "--model", `${MODEL_PROVIDER}/${MODEL_ID}`, "--extension", EXTENSION_FILE, "--no-extensions", "--no-builtin-tools", "--no-skills", "--no-context-files", "--verbose", "--print", prompt, ]; const piEnv = { ...process.env, PI_CODING_AGENT_DIR: PI_HOME, APS_E2E_TOOL: targetTool, NO_PROXY: "127.0.0.1,localhost", no_proxy: "127.0.0.1,localhost", }; if (options.configOnly) { delete piEnv.APS_BASE_URL; delete piEnv.APS_AGENT_TOKEN; delete piEnv.APS_REQUEST_TIMEOUT_MS; delete piEnv.APS_CATALOG_TIMEOUT_MS; piEnv.APS_CONFIG_FILE = options.configFile; } else { piEnv.APS_BASE_URL = options.apsUrl; if (options.token) piEnv.APS_AGENT_TOKEN = options.token; } console.log(`[e2e] starting real Pi CLI (timeout ${options.timeoutMs}ms)`); const piChild = spawn(process.execPath, piArgs, { cwd: PACKAGE_ROOT, env: piEnv, stdio: ["ignore", "pipe", "pipe"], windowsHide: true, }); const piCollect = collectProcess("pi-cli", piChild); const timer = setTimeout(() => { console.error("[e2e] Pi CLI timeout, killing process"); piChild.kill("SIGTERM"); }, options.timeoutMs); const piResult = await piCollect; clearTimeout(timer); const mockResult = await Promise.race([mockCollect, new Promise((r) => setTimeout(() => r(null), 3000))]); await writeFile(EVENTS_FILE, piResult.stdout, "utf8"); const events = piResult.stdout .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean) .map((line) => { try { return JSON.parse(line); } catch { return { type: "non_json", raw: line.slice(0, 1000) }; } }); const starts = events.filter((e) => e.type === "tool_execution_start" && e.toolName === targetTool); const ends = events.filter((e) => e.type === "tool_execution_end" && e.toolName === targetTool); const finalMessages = events .filter((e) => e.type === "agent_end") .flatMap((e) => Array.isArray(e.messages) ? e.messages : []) .filter((m) => m?.role === "assistant"); const finalAssistant = [...finalMessages].reverse().find((m) => m?.stopReason === "stop"); const failedRetry = events.some((e) => e.type === "auto_retry_end" && e.success === false); const extensionError = events.find((e) => e.type === "extension_error"); const toolEndOk = ends.some((e) => e.isError === false); const toolStartOk = starts.length > 0; const finalStopOk = Boolean(finalAssistant) && finalAssistant.stopReason === "stop"; checks.toolExecutionStart = toolStartOk; checks.toolExecutionSuccess = toolEndOk; checks.finalAssistantStop = finalStopOk; checks.noFatalProviderRetry = !failedRetry; checks.noExtensionLoadError = !extensionError; summary.events = { rawLines: piResult.stdout.split(/\r?\n/).filter(Boolean).length, parsedEvents: events.filter((e) => e.type !== "non_json").length, toolStartCount: starts.length, toolEndCount: ends.length, toolEndOk, finalStopReason: finalAssistant?.stopReason || "missing", finalText: (finalAssistant?.content || []) .filter((c) => c?.type === "text") .map((c) => c.text) .join("") .slice(0, 1200), extensionError: extensionError || null, }; summary.pi = { exitCode: piResult.code, signal: piResult.signal, stderrTail: piResult.stderr.slice(-1200), mockExitCode: mockResult?.code ?? null, }; const allOk = [toolStartOk, toolEndOk, finalStopOk, !failedRetry, !extensionError].every(Boolean); summary.pass = allOk; await writeFile(SUMMARY_FILE, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); console.log(`[e2e] verdict: ${allOk ? "PASS" : "FAIL"}`); console.log(`[e2e] checks: ${JSON.stringify(checks)}`); return allOk ? 0 : 1; } catch (err) { summary.pass = false; summary.fatalError = err instanceof Error ? err.message : String(err); try { await writeFile(SUMMARY_FILE, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); } catch { // best effort } console.error(`[e2e] fatal: ${summary.fatalError}`); return 2; } finally { if (generatedConfigFile) { try { await rm(generatedConfigFile, { force: true }); } catch { // Best-effort cleanup of a token-bearing temp config. } } for (const child of mockProcesses) { if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM"); } } } process.exitCode = await main();