453 lines
16 KiB
JavaScript
453 lines
16 KiB
JavaScript
/**
|
|
* e2e.mjs - real Pi CLI end-to-end validation for the pi-aps extension.
|
|
*
|
|
* Chain under test:
|
|
* real APS HTTP server -> real Pi CLI -> pi-aps extension -> mock LLM (scripted)
|
|
*
|
|
* The mock LLM never generates APS data. If the APS server exposes
|
|
* /api/agent/intents with order.pool, the script asks the model to call
|
|
* aps_order_pool. Otherwise it falls back to aps_health so the real Pi-to-APS
|
|
* HTTP chain is still exercised.
|
|
*
|
|
* Pi headless always exits 0; verdict is based on JSONL events.
|
|
*/
|
|
import { spawn } from "node:child_process";
|
|
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
import net from "node:net";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const E2E_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
const PACKAGE_ROOT = path.resolve(E2E_DIR, "..");
|
|
const REPO_ROOT = path.resolve(E2E_DIR, "..", "..", "..");
|
|
const PI_CLI = path.join(
|
|
REPO_ROOT,
|
|
"poc",
|
|
"pi-fallback",
|
|
"runtime",
|
|
"node_modules",
|
|
"@mariozechner",
|
|
"pi-coding-agent",
|
|
"dist",
|
|
"cli.js",
|
|
);
|
|
const EXTENSION_FILE = path.join(PACKAGE_ROOT, "extensions", "aps.ts");
|
|
const MOCK_LLM_FILE = path.join(E2E_DIR, "mock-llm.mjs");
|
|
const PI_HOME = path.join(E2E_DIR, "pi-home");
|
|
const EVENTS_FILE = path.join(E2E_DIR, "last-events.jsonl");
|
|
const SUMMARY_FILE = path.join(E2E_DIR, "last-summary.json");
|
|
const MOCK_LOG_FILE = path.join(E2E_DIR, "mock-requests-last.jsonl");
|
|
|
|
const DEFAULT_APS_URL = "http://127.0.0.1:8000";
|
|
const MODEL_PROVIDER = "aps-e2e";
|
|
const MODEL_ID = "mock-local";
|
|
|
|
function env(key, fallback = "") {
|
|
return process.env[key] || fallback;
|
|
}
|
|
|
|
function parseArgs() {
|
|
const out = {
|
|
apsUrl: env("APS_BASE_URL", DEFAULT_APS_URL).trim().replace(/\/+$/, ""),
|
|
token: env("APS_AGENT_TOKEN", "").trim() || undefined,
|
|
configFile: env("APS_CONFIG_FILE", "").trim() || undefined,
|
|
configOnly: ["1", "true", "yes"].includes(
|
|
env("APS_E2E_CONFIG_FILE_ONLY", "").trim().toLowerCase(),
|
|
),
|
|
forceTool: env("APS_E2E_TOOL", "").trim() || undefined,
|
|
timeoutMs: Number(env("APS_E2E_TIMEOUT_MS", "120000")),
|
|
prompt: env(
|
|
"APS_E2E_PROMPT",
|
|
"请用 APS 只读工具查询订单池，拿到真实返回后给出一句中文总结。",
|
|
),
|
|
};
|
|
const argv = process.argv.slice(2);
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
if (argv[i] === "--aps-url") out.apsUrl = String(argv[i + 1] || out.apsUrl).replace(/\/+$/, "");
|
|
if (argv[i] === "--tool") out.forceTool = argv[i + 1] || out.forceTool;
|
|
if (argv[i] === "--prompt") out.prompt = argv[i + 1] || out.prompt;
|
|
if (argv[i] === "--timeout-ms") out.timeoutMs = Number(argv[i + 1] || out.timeoutMs);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function resolveConfigOnlyPath(configFile) {
|
|
const resolved = path.resolve(String(configFile || "").trim());
|
|
const home = path.resolve(PI_HOME);
|
|
if (!resolved.startsWith(home + path.sep)) {
|
|
throw new Error(
|
|
`[e2e] APS_E2E_CONFIG_FILE_ONLY=1 requires APS_CONFIG_FILE under ${home}`,
|
|
);
|
|
}
|
|
return resolved;
|
|
}
|
|
|
|
function normalizeToolName(intentName) {
|
|
const normalized = String(intentName || "")
|
|
.trim()
|
|
.toLowerCase()
|
|
.replace(/\./g, "_")
|
|
.replace(/[^a-z0-9_]+/g, "_")
|
|
.replace(/_+/g, "_")
|
|
.replace(/^_+|_+$/g, "");
|
|
if (!normalized) return undefined;
|
|
const toolName = `aps_${normalized}`;
|
|
return /^[a-z][a-z0-9_]*$/.test(toolName) ? toolName : undefined;
|
|
}
|
|
|
|
async function httpJson(url, options = {}) {
|
|
const headers = { Accept: "application/json", "Content-Type": "application/json" };
|
|
if (options.token) headers.Authorization = `Bearer ${options.token}`;
|
|
const response = await fetch(url, { headers, signal: AbortSignal.timeout(options.timeoutMs || 8000) });
|
|
const text = await response.text();
|
|
let body = {};
|
|
if (text) {
|
|
try {
|
|
body = JSON.parse(text);
|
|
} catch {
|
|
body = { raw: text.slice(0, 300) };
|
|
}
|
|
}
|
|
return { status: response.status, body };
|
|
}
|
|
|
|
async function waitForHealth(apsUrl, timeoutMs = 25000) {
|
|
const started = Date.now();
|
|
let lastError = "";
|
|
while (Date.now() - started < timeoutMs) {
|
|
try {
|
|
const { status, body } = await httpJson(`${apsUrl}/api/health`, { timeoutMs: 4000 });
|
|
if (status === 200 && body?.ok !== false) return { status, body };
|
|
lastError = `status=${status}`;
|
|
} catch (err) {
|
|
lastError = err instanceof Error ? err.message : String(err);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 750));
|
|
}
|
|
throw new Error(`APS server not healthy within ${timeoutMs}ms at ${apsUrl}: ${lastError}`);
|
|
}
|
|
|
|
async function detectCatalog(apsUrl, token) {
|
|
const { status, body } = await httpJson(`${apsUrl}/api/agent/intents`, { token, timeoutMs: 6000 });
|
|
if (status !== 200) return { available: false, status };
|
|
const entries = Array.isArray(body?.intents) ? body.intents : [];
|
|
if (!entries.length) return { available: true, status, intents: [] };
|
|
const explicit = env("APS_E2E_INTENT", "").trim();
|
|
let entry = entries.find((item) => item && String(item.name || "").trim() === explicit);
|
|
if (!entry) entry = entries.find((item) => item && String(item.name || "") === "order.pool");
|
|
if (!entry && !explicit) entry = entries[0];
|
|
return { available: true, status, intents: entries, selected: entry || null };
|
|
}
|
|
|
|
function freePort() {
|
|
return new Promise((resolve, reject) => {
|
|
const server = net.createServer();
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", () => {
|
|
const port = server.address().port;
|
|
server.close(() => resolve(port));
|
|
});
|
|
});
|
|
}
|
|
|
|
async function waitForPortFile(file, expectedPort, timeoutMs = 10000) {
|
|
const started = Date.now();
|
|
while (Date.now() - started < timeoutMs) {
|
|
try {
|
|
const text = await readFile(file, "utf8");
|
|
const match = String(text).match(/MOCK_LLM_PORT=(\d+)/);
|
|
if (match && Number(match[1]) === expectedPort) return text;
|
|
} catch {
|
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
}
|
|
}
|
|
throw new Error(`mock LLM did not publish port file: ${file}`);
|
|
}
|
|
|
|
function collectProcess(name, child) {
|
|
return new Promise((resolve, reject) => {
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdout.on("data", (c) => {
|
|
stdout += c.toString("utf8");
|
|
if (stdout.length > 20_000_000) stdout = stdout.slice(-10_000_000);
|
|
});
|
|
child.stderr.on("data", (c) => {
|
|
stderr += c.toString("utf8");
|
|
if (stderr.length > 2_000_000) stderr = stderr.slice(-1_000_000);
|
|
});
|
|
child.on("error", reject);
|
|
child.on("exit", (code, signal) => resolve({ name, code, signal, stdout, stderr }));
|
|
});
|
|
}
|
|
|
|
function promptForTool(toolName) {
|
|
if (toolName === "aps_health") {
|
|
return "请用 aps_health 工具检查 APS 健康状态，拿到真实返回后给出一句中文总结。";
|
|
}
|
|
if (toolName === "aps_summary") {
|
|
return "请用 aps_summary 工具读取 APS 世界摘要，拿到真实返回后给出一句中文总结。";
|
|
}
|
|
return `请用 ${toolName} 工具查询 APS 只读业务数据，拿到真实返回后给出一句中文总结。`;
|
|
}
|
|
|
|
|
|
function redactOptions(options) {
|
|
const rest = {
|
|
apsUrl: options.apsUrl,
|
|
configOnly: Boolean(options.configOnly),
|
|
forceTool: options.forceTool,
|
|
timeoutMs: options.timeoutMs,
|
|
prompt: options.prompt,
|
|
configFile: options.configFile ? "configured" : undefined,
|
|
};
|
|
if (!options.token) return { ...rest, token: "not configured" };
|
|
return { ...rest, token: "configured (ends " + options.token.slice(-4) + ")" };
|
|
}
|
|
|
|
async function main() {
|
|
const options = parseArgs();
|
|
let generatedConfigFile;
|
|
if (options.configOnly) {
|
|
if (!options.configFile) {
|
|
throw new Error(
|
|
"[e2e] APS_E2E_CONFIG_FILE_ONLY=1 requires APS_CONFIG_FILE",
|
|
);
|
|
}
|
|
options.configFile = resolveConfigOnlyPath(options.configFile);
|
|
generatedConfigFile = options.configFile;
|
|
}
|
|
const checks = {};
|
|
const summary = { options: redactOptions(options), checks, ranAt: new Date().toISOString() };
|
|
const mockProcesses = [];
|
|
|
|
try {
|
|
checks.serverHealth = false;
|
|
const health = await waitForHealth(options.apsUrl);
|
|
checks.serverHealth = true;
|
|
console.log(`[e2e] APS health OK: ${JSON.stringify(health.body)}`);
|
|
|
|
const catalog = await detectCatalog(options.apsUrl, options.token);
|
|
let targetTool;
|
|
let catalogIntent;
|
|
if (catalog.available && catalog.selected) {
|
|
catalogIntent = String(catalog.selected.name || "").trim();
|
|
targetTool = normalizeToolName(catalogIntent) || options.forceTool || "aps_health";
|
|
console.log(`[e2e] APS agent catalog available: intent=${catalogIntent} tool=${targetTool}`);
|
|
} else {
|
|
targetTool = options.forceTool || "aps_health";
|
|
console.log(
|
|
`[e2e] APS agent catalog unavailable (status=${catalog.status}); ` +
|
|
`falling back to ${targetTool}`,
|
|
);
|
|
}
|
|
summary.catalog = {
|
|
available: Boolean(catalog.available),
|
|
status: catalog.status,
|
|
selectedIntent: catalogIntent || null,
|
|
targetTool,
|
|
};
|
|
|
|
const mockPort = await freePort();
|
|
const portFile = path.join(E2E_DIR, "mock-port.txt");
|
|
await rm(portFile, { force: true });
|
|
await rm(MOCK_LOG_FILE, { force: true });
|
|
const mockEnv = {
|
|
...process.env,
|
|
APS_E2E_TOOL: targetTool,
|
|
APS_MOCK_REQUEST_LOG: MOCK_LOG_FILE,
|
|
NO_PROXY: "127.0.0.1,localhost",
|
|
no_proxy: "127.0.0.1,localhost",
|
|
};
|
|
for (const key of [
|
|
"APS_BASE_URL",
|
|
"APS_AGENT_TOKEN",
|
|
"APS_REQUEST_TIMEOUT_MS",
|
|
"APS_CATALOG_TIMEOUT_MS",
|
|
"APS_CONFIG_FILE",
|
|
]) {
|
|
delete mockEnv[key];
|
|
}
|
|
const mockChild = spawn(
|
|
process.execPath,
|
|
[MOCK_LLM_FILE, "--port", String(mockPort), "--port-file", portFile],
|
|
{ env: mockEnv, stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
|
|
);
|
|
mockProcesses.push(mockChild);
|
|
const mockCollect = collectProcess("mock-llm", mockChild);
|
|
const portText = await waitForPortFile(portFile, mockPort);
|
|
const actualPort = Number(String(portText).match(/MOCK_LLM_PORT=(\d+)/)?.[1] || mockPort);
|
|
summary.mockPort = actualPort;
|
|
console.log(`[e2e] mock LLM on ${actualPort}`);
|
|
|
|
await mkdir(PI_HOME, { recursive: true });
|
|
const models = {
|
|
providers: {
|
|
[MODEL_PROVIDER]: {
|
|
baseUrl: `http://127.0.0.1:${actualPort}/v1`,
|
|
api: "openai-completions",
|
|
apiKey: "aps-e2e-mock-no-secret",
|
|
models: [
|
|
{
|
|
id: MODEL_ID,
|
|
name: "APS E2E Scripted Mock",
|
|
reasoning: false,
|
|
input: ["text"],
|
|
contextWindow: 32768,
|
|
maxTokens: 2048,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
};
|
|
await writeFile(path.join(PI_HOME, "models.json"), `${JSON.stringify(models, null, 2)}\n`, "utf8");
|
|
await writeFile(path.join(PI_HOME, "auth.json"), "{}\n", "utf8");
|
|
if (options.configOnly) {
|
|
await mkdir(path.dirname(options.configFile), { recursive: true });
|
|
const configPayload = {
|
|
aps: {
|
|
baseUrl: options.apsUrl,
|
|
...(options.token ? { token: options.token } : {}),
|
|
},
|
|
};
|
|
await writeFile(
|
|
options.configFile,
|
|
`${JSON.stringify(configPayload, null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
console.log("[e2e] APS config file mode: token/base URL passed through aps.json");
|
|
}
|
|
|
|
const prompt = promptForTool(targetTool);
|
|
const piArgs = [
|
|
PI_CLI,
|
|
"--mode", "json",
|
|
"--no-session",
|
|
"--model", `${MODEL_PROVIDER}/${MODEL_ID}`,
|
|
"--extension", EXTENSION_FILE,
|
|
"--no-extensions",
|
|
"--no-builtin-tools",
|
|
"--no-skills",
|
|
"--no-context-files",
|
|
"--verbose",
|
|
"--print",
|
|
prompt,
|
|
];
|
|
const piEnv = {
|
|
...process.env,
|
|
PI_CODING_AGENT_DIR: PI_HOME,
|
|
APS_E2E_TOOL: targetTool,
|
|
NO_PROXY: "127.0.0.1,localhost",
|
|
no_proxy: "127.0.0.1,localhost",
|
|
};
|
|
if (options.configOnly) {
|
|
delete piEnv.APS_BASE_URL;
|
|
delete piEnv.APS_AGENT_TOKEN;
|
|
delete piEnv.APS_REQUEST_TIMEOUT_MS;
|
|
delete piEnv.APS_CATALOG_TIMEOUT_MS;
|
|
piEnv.APS_CONFIG_FILE = options.configFile;
|
|
} else {
|
|
piEnv.APS_BASE_URL = options.apsUrl;
|
|
if (options.token) piEnv.APS_AGENT_TOKEN = options.token;
|
|
}
|
|
console.log(`[e2e] starting real Pi CLI (timeout ${options.timeoutMs}ms)`);
|
|
const piChild = spawn(process.execPath, piArgs, {
|
|
cwd: PACKAGE_ROOT,
|
|
env: piEnv,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
windowsHide: true,
|
|
});
|
|
const piCollect = collectProcess("pi-cli", piChild);
|
|
const timer = setTimeout(() => {
|
|
console.error("[e2e] Pi CLI timeout, killing process");
|
|
piChild.kill("SIGTERM");
|
|
}, options.timeoutMs);
|
|
const piResult = await piCollect;
|
|
clearTimeout(timer);
|
|
const mockResult = await Promise.race([mockCollect, new Promise((r) => setTimeout(() => r(null), 3000))]);
|
|
await writeFile(EVENTS_FILE, piResult.stdout, "utf8");
|
|
|
|
const events = piResult.stdout
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
try {
|
|
return JSON.parse(line);
|
|
} catch {
|
|
return { type: "non_json", raw: line.slice(0, 1000) };
|
|
}
|
|
});
|
|
const starts = events.filter((e) => e.type === "tool_execution_start" && e.toolName === targetTool);
|
|
const ends = events.filter((e) => e.type === "tool_execution_end" && e.toolName === targetTool);
|
|
const finalMessages = events
|
|
.filter((e) => e.type === "agent_end")
|
|
.flatMap((e) => Array.isArray(e.messages) ? e.messages : [])
|
|
.filter((m) => m?.role === "assistant");
|
|
const finalAssistant = [...finalMessages].reverse().find((m) => m?.stopReason === "stop");
|
|
const failedRetry = events.some((e) => e.type === "auto_retry_end" && e.success === false);
|
|
const extensionError = events.find((e) => e.type === "extension_error");
|
|
|
|
const toolEndOk = ends.some((e) => e.isError === false);
|
|
const toolStartOk = starts.length > 0;
|
|
const finalStopOk = Boolean(finalAssistant) && finalAssistant.stopReason === "stop";
|
|
checks.toolExecutionStart = toolStartOk;
|
|
checks.toolExecutionSuccess = toolEndOk;
|
|
checks.finalAssistantStop = finalStopOk;
|
|
checks.noFatalProviderRetry = !failedRetry;
|
|
checks.noExtensionLoadError = !extensionError;
|
|
|
|
summary.events = {
|
|
rawLines: piResult.stdout.split(/\r?\n/).filter(Boolean).length,
|
|
parsedEvents: events.filter((e) => e.type !== "non_json").length,
|
|
toolStartCount: starts.length,
|
|
toolEndCount: ends.length,
|
|
toolEndOk,
|
|
finalStopReason: finalAssistant?.stopReason || "missing",
|
|
finalText: (finalAssistant?.content || [])
|
|
.filter((c) => c?.type === "text")
|
|
.map((c) => c.text)
|
|
.join("")
|
|
.slice(0, 1200),
|
|
extensionError: extensionError || null,
|
|
};
|
|
summary.pi = {
|
|
exitCode: piResult.code,
|
|
signal: piResult.signal,
|
|
stderrTail: piResult.stderr.slice(-1200),
|
|
mockExitCode: mockResult?.code ?? null,
|
|
};
|
|
|
|
const allOk = [toolStartOk, toolEndOk, finalStopOk, !failedRetry, !extensionError].every(Boolean);
|
|
summary.pass = allOk;
|
|
await writeFile(SUMMARY_FILE, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
console.log(`[e2e] verdict: ${allOk ? "PASS" : "FAIL"}`);
|
|
console.log(`[e2e] checks: ${JSON.stringify(checks)}`);
|
|
return allOk ? 0 : 1;
|
|
} catch (err) {
|
|
summary.pass = false;
|
|
summary.fatalError = err instanceof Error ? err.message : String(err);
|
|
try {
|
|
await writeFile(SUMMARY_FILE, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
} catch {
|
|
// best effort
|
|
}
|
|
console.error(`[e2e] fatal: ${summary.fatalError}`);
|
|
return 2;
|
|
} finally {
|
|
if (generatedConfigFile) {
|
|
try {
|
|
await rm(generatedConfigFile, { force: true });
|
|
} catch {
|
|
// Best-effort cleanup of a token-bearing temp config.
|
|
}
|
|
}
|
|
for (const child of mockProcesses) {
|
|
if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM");
|
|
}
|
|
}
|
|
}
|
|
|
|
process.exitCode = await main();
|