644 lines
30 KiB
Python
644 lines
30 KiB
Python
|
# ============================================================
|
|||
|
|
# 审计独立介质锚定 v1（moduleId: core-audit-ledger, 可重生 ✅）
|
|||
|
|
# plan.md §3.6 / 矩阵「全审计」：独立介质 + 外部锚定根 + 篡改检测
|
|||
|
|
# 与 write_audit 的世界内 SHA-256 链互补：账本提供跨介质锚定证明
|
|||
|
|
# ============================================================
|
|||
|
|
from __future__ import annotations # 前向类型引用
|
|||
|
|
|
|||
|
|
import hashlib # SHA-256
|
|||
|
|
import base64 # 加密信封 base64 编解码
|
|||
|
|
import json # JSONL 序列化
|
|||
|
|
import os # 目录/文件操作
|
|||
|
|
import threading # 并发写锁
|
|||
|
|
from pathlib import Path # 路径处理
|
|||
|
|
from typing import Any # 类型标注
|
|||
|
|
|
|||
|
|
# 创世锚定根（空账本 / 空事件列表）
|
|||
|
|
GENESIS_ROOT = hashlib.sha256(b"aps-audit-genesis").hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# WORM 模拟清单：归档目录密封承诺（矩阵 104/112/116 的 WORM 剩余项）
|
|||
|
|
WORM_MANIFEST = ".worm-manifest.json" # 归档目录 WORM 模拟清单文件名
|
|||
|
|
WORM_NOTE = ("逻辑 WORM 语义 + 篡改检测模拟：归档文件写入 .worm-manifest.json 密封"
|
|||
|
|
"（sha256 + merkle root 摘要 + 归档时间 + 覆盖范围），常规写路径拒绝改写；"
|
|||
|
|
"物理 WORM 介质（光盘/WORM 盘）为外部验收范围，本实现不模拟硬件级不可覆写。")
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 加密归档模拟（矩阵 104/116 的加密剩余项；真实密钥管理为外部验收）
|
|||
|
|
ENCRYPTION_SCHEME = "aes-256-gcm" # AES-256-GCM 认证加密
|
|||
|
|
ENCRYPTION_KDF = "scrypt" # 口令 → 密钥派生（scrypt）
|
|||
|
|
ENCRYPTION_ENV_KEY = "APS_AUDIT_ENCRYPT_KEY" # 加密密钥/口令 env
|
|||
|
|
ENCRYPTION_ENV_KEY_FILE = "APS_AUDIT_ENCRYPT_KEY_FILE" # 加密密钥文件 env
|
|||
|
|
ENCRYPTED_FORMAT = "aps-audit-encrypted-v1" # 加密归档信封格式版本
|
|||
|
|
ENCRYPTION_NOTE = (f"本地对称加密模拟（aes-256-gcm + scrypt 派生）：归档记录加密落盘，"
|
|||
|
|
f"manifest 记录 ciphertextHash + plaintextHash 双层校验；"
|
|||
|
|
f"真实密钥管理（HSM/KMS、密钥轮换与托管）为外部验收范围，本地密钥来源 "
|
|||
|
|
f"{ENCRYPTION_ENV_KEY}（64 位 hex = 32B 原始密钥，否则视为口令 scrypt 派生）"
|
|||
|
|
f"或 {ENCRYPTION_ENV_KEY_FILE}（密钥文件，规则同上）。")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _file_sha256(path: Path) -> str:
|
|||
|
|
"""计算文件 SHA-256（分块读取，适配大归档文件）。"""
|
|||
|
|
h = hashlib.sha256()
|
|||
|
|
with open(path, "rb") as fh:
|
|||
|
|
for chunk in iter(lambda: fh.read(65536), b""):
|
|||
|
|
h.update(chunk)
|
|||
|
|
return h.hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_jsonl(text: str) -> list[dict[str, Any]]:
|
|||
|
|
"""解析 JSONL 文本 → 记录列表（损坏行标注 _corrupt，不中断）。"""
|
|||
|
|
records: list[dict[str, Any]] = []
|
|||
|
|
for line in text.splitlines():
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
records.append(json.loads(line))
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
records.append({"_corrupt": True})
|
|||
|
|
return records
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _key_material() -> tuple[bytes, str] | None:
|
|||
|
|
"""解析本地加密密钥材料：返回 (material, kind)；kind ∈ {"raw", "passphrase"}。
|
|||
|
|
|
|||
|
|
APS_AUDIT_ENCRYPT_KEY（或密钥文件内容）为 64 位 hex → 32B 原始密钥（kind="raw"），
|
|||
|
|
否则视为口令（kind="passphrase"，scrypt 派生）。未配置/密钥文件缺失 → None，
|
|||
|
|
由调用方决定 fail-closed 或明文归档。
|
|||
|
|
"""
|
|||
|
|
raw = os.environ.get(ENCRYPTION_ENV_KEY)
|
|||
|
|
if raw is None:
|
|||
|
|
key_file = os.environ.get(ENCRYPTION_ENV_KEY_FILE)
|
|||
|
|
if key_file:
|
|||
|
|
try:
|
|||
|
|
raw = Path(key_file).read_text(encoding="utf-8").strip()
|
|||
|
|
except OSError:
|
|||
|
|
return None
|
|||
|
|
if not raw:
|
|||
|
|
return None
|
|||
|
|
raw = raw.strip()
|
|||
|
|
if len(raw) == 64:
|
|||
|
|
try:
|
|||
|
|
return bytes.fromhex(raw), "raw"
|
|||
|
|
except ValueError:
|
|||
|
|
pass
|
|||
|
|
return raw.encode("utf-8"), "passphrase"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _derive_key(material: bytes, salt: bytes, use_scrypt: bool) -> bytes:
|
|||
|
|
"""密钥派生：raw 材料直接使用（须 32B）；口令 scrypt（n=2^14, r=8, p=1）。"""
|
|||
|
|
if not use_scrypt:
|
|||
|
|
if len(material) != 32:
|
|||
|
|
raise ValueError("raw 密钥必须是 32 字节（64 位 hex）")
|
|||
|
|
return material
|
|||
|
|
return hashlib.scrypt(material, salt=salt, n=2 ** 14, r=8, p=1, dklen=32)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _encrypt_bytes(plaintext: bytes, material: tuple[bytes, str]) -> dict[str, Any]:
|
|||
|
|
"""AES-256-GCM 加密归档字节 → 信封 dict（salt/iv/ciphertext base64）。
|
|||
|
|
|
|||
|
|
依赖 cryptography（.venv v49 可用）；库不可用 → RuntimeError（fail-closed，
|
|||
|
|
不静默降级明文）。信封自带 plaintextHash 便于外层双层校验。
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|||
|
|
except ImportError as exc: # pragma: no cover - .venv 已安装
|
|||
|
|
raise RuntimeError("cryptography 库不可用，无法执行加密归档（fail-closed）") from exc
|
|||
|
|
material_bytes, kind = material
|
|||
|
|
salt = os.urandom(16)
|
|||
|
|
iv = os.urandom(12)
|
|||
|
|
use_scrypt = kind == "passphrase"
|
|||
|
|
key = _derive_key(material_bytes, salt, use_scrypt)
|
|||
|
|
payload = AESGCM(key).encrypt(iv, plaintext, None) # ciphertext || tag
|
|||
|
|
return {
|
|||
|
|
"format": ENCRYPTED_FORMAT,
|
|||
|
|
"scheme": ENCRYPTION_SCHEME,
|
|||
|
|
"kdf": ENCRYPTION_KDF if use_scrypt else "none",
|
|||
|
|
"salt": base64.b64encode(salt).decode("ascii"),
|
|||
|
|
"iv": base64.b64encode(iv).decode("ascii"),
|
|||
|
|
"ciphertext": base64.b64encode(payload).decode("ascii"),
|
|||
|
|
"plaintextHash": hashlib.sha256(plaintext).hexdigest(),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _decrypt_bytes(envelope: dict[str, Any], material: bytes) -> bytes:
|
|||
|
|
"""解密信封 → 明文归档字节；密钥缺失/错误/密文或信封篡改 → ValueError（显式失败）。"""
|
|||
|
|
if envelope.get("format") != ENCRYPTED_FORMAT:
|
|||
|
|
raise ValueError(f"未知加密归档格式: {envelope.get('format')!r}")
|
|||
|
|
try:
|
|||
|
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|||
|
|
except ImportError as exc: # pragma: no cover - .venv 已安装
|
|||
|
|
raise RuntimeError("cryptography 库不可用，无法解密归档（fail-closed）") from exc
|
|||
|
|
try:
|
|||
|
|
salt = base64.b64decode(envelope["salt"])
|
|||
|
|
iv = base64.b64decode(envelope["iv"])
|
|||
|
|
payload = base64.b64decode(envelope["ciphertext"])
|
|||
|
|
use_scrypt = envelope.get("kdf") == ENCRYPTION_KDF
|
|||
|
|
key = _derive_key(material, salt, use_scrypt)
|
|||
|
|
plaintext = AESGCM(key).decrypt(iv, payload, None)
|
|||
|
|
except Exception as exc: # noqa: BLE001 - 密钥错误/密文篡改/信封损坏统一显式失败
|
|||
|
|
raise ValueError("归档解密失败（密钥错误或密文被篡改）") from exc
|
|||
|
|
return plaintext
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _read_envelope(path: Path) -> dict[str, Any]:
|
|||
|
|
"""读取加密归档信封（JSON）；损坏 → ValueError（调用方显式失败，不静默）。"""
|
|||
|
|
try:
|
|||
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|||
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|||
|
|
raise ValueError("加密归档信封损坏") from exc
|
|||
|
|
if not isinstance(data, dict):
|
|||
|
|
raise ValueError("加密归档信封损坏")
|
|||
|
|
return data
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _atomic_write(path: Path, data: bytes) -> None:
|
|||
|
|
"""原子写文件（临时文件 + os.replace），避免半写。"""
|
|||
|
|
import tempfile as _tf
|
|||
|
|
fd, tmp = _tf.mkstemp(dir=str(path.parent), suffix=".tmp")
|
|||
|
|
try:
|
|||
|
|
with os.fdopen(fd, "wb") as fh:
|
|||
|
|
fh.write(data)
|
|||
|
|
os.replace(tmp, path)
|
|||
|
|
except BaseException:
|
|||
|
|
if os.path.exists(tmp):
|
|||
|
|
os.unlink(tmp)
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
|
|||
|
|
def audit_root(events: list[dict[str, Any]]) -> str:
|
|||
|
|
"""计算审计事件的锚定根（外部锚定证明）。
|
|||
|
|
|
|||
|
|
与 verify_audit_chain 的链式校验互补：root 对每个事件哈希做
|
|||
|
|
双哈希链聚合（Merkle 式），任何事件内容/顺序/数量变化都会改变根。
|
|||
|
|
"""
|
|||
|
|
acc = bytes.fromhex(GENESIS_ROOT) # 起点：创世根 digest
|
|||
|
|
for ev in events:
|
|||
|
|
ev_hash = str(ev.get("hash") or "").encode("utf-8")
|
|||
|
|
acc = hashlib.sha256(acc + ev_hash).digest()
|
|||
|
|
return acc.hex()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _default_ledger_dir() -> str:
|
|||
|
|
"""默认账本目录：server/data/audit-ledger（可按 APS_AUDIT_LEDGER_DIR 覆盖）。"""
|
|||
|
|
env = os.environ.get("APS_AUDIT_LEDGER_DIR")
|
|||
|
|
if env:
|
|||
|
|
return env
|
|||
|
|
try:
|
|||
|
|
from server.aps_home import path_under_data
|
|||
|
|
return str(path_under_data("audit-ledger"))
|
|||
|
|
except Exception:
|
|||
|
|
return os.path.join("server", "data", "audit-ledger")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _safe_scope(value: str) -> str:
|
|||
|
|
cleaned = "".join(c for c in (value or "default") if c.isalnum() or c in "-_")
|
|||
|
|
return cleaned[:64] or "default"
|
|||
|
|
|
|||
|
|
|
|||
|
|
class AnchorLedger:
|
|||
|
|
"""独立 append-only 审计账本（按租户/项目隔离，JSONL）。
|
|||
|
|
|
|||
|
|
每行一条锚定记录：{eventId, root, count, at, reason}。
|
|||
|
|
记录只追加；删除/改写历史行会导致校验失败（append-only 语义）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, tenant_uuid: str = "platform", world_key: str = "default",
|
|||
|
|
ledger_dir: str | None = None) -> None:
|
|||
|
|
self.tenant_uuid = _safe_scope(tenant_uuid)
|
|||
|
|
self.world_key = _safe_scope(world_key)
|
|||
|
|
base = Path(ledger_dir or _default_ledger_dir())
|
|||
|
|
self.path = base / self.tenant_uuid / f"{self.world_key}.jsonl"
|
|||
|
|
self._lock = threading.Lock()
|
|||
|
|
|
|||
|
|
def _ensure_dir(self) -> None:
|
|||
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
|
|||
|
|
def append(self, events: list[dict[str, Any]], *, reason: str = "manual") -> dict[str, Any]:
|
|||
|
|
"""追加一条锚定记录（原子：写临时文件后重命名，避免半行）。"""
|
|||
|
|
root = audit_root(events)
|
|||
|
|
count = len(events)
|
|||
|
|
last_event_id = str((events[-1].get("id") if events else None) or "GENESIS")
|
|||
|
|
record = {
|
|||
|
|
"eventId": last_event_id,
|
|||
|
|
"root": root,
|
|||
|
|
"count": count,
|
|||
|
|
"at": _now_iso(),
|
|||
|
|
"reason": reason,
|
|||
|
|
}
|
|||
|
|
with self._lock:
|
|||
|
|
self._ensure_dir()
|
|||
|
|
self._guard_archive_sealed(self.path) # WORM 模拟：密封归档件只读
|
|||
|
|
# append-only 追加（进程内锁 + 跨进程文件锁；fsync 保证落盘）
|
|||
|
|
from server.agent_core.audit_filelock import locked_append
|
|||
|
|
with locked_append(self.path) as dst:
|
|||
|
|
dst.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
|||
|
|
return record
|
|||
|
|
|
|||
|
|
def read(self) -> list[dict[str, Any]]:
|
|||
|
|
"""读取全部锚定记录（损坏行 → 校验失败语义由 verify 处理）。"""
|
|||
|
|
if not self.path.exists():
|
|||
|
|
return []
|
|||
|
|
with self._lock:
|
|||
|
|
try:
|
|||
|
|
text = self.path.read_text(encoding="utf-8")
|
|||
|
|
except OSError:
|
|||
|
|
return []
|
|||
|
|
records: list[dict[str, Any]] = []
|
|||
|
|
for line in text.splitlines():
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
records.append(json.loads(line))
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
records.append({"_corrupt": True})
|
|||
|
|
return records
|
|||
|
|
|
|||
|
|
def archive_old(self, keep_last: int | None = None,
|
|||
|
|
*, encrypt: bool | None = None) -> dict[str, Any]:
|
|||
|
|
"""清理策略（矩阵 116 行）：把超过保留窗口的旧锚定记录移到独立归档目录。
|
|||
|
|
|
|||
|
|
append-only 语义保持：归档是分级存储而非删除——旧记录移入
|
|||
|
|
<dir>/archive/<yyyy-mm>.jsonl，主文件保留最近 keep_last 条；
|
|||
|
|
归档文件同样可读、可校验，证据不丢失。
|
|||
|
|
归档后文件密封为 WORM 模拟（.worm-manifest.json：sha256 + merkle root 摘要
|
|||
|
|
+ 归档时间 + 覆盖范围；文件置只读），常规写路径拒绝改写归档件。
|
|||
|
|
加密归档（矩阵 104/116 加密部分）：encrypt=True 强制加密 / False 强制明文 /
|
|||
|
|
None 自动（配置 APS_AUDIT_ENCRYPT_KEY 或 APS_AUDIT_ENCRYPT_KEY_FILE 即加密）；
|
|||
|
|
归档文件替换为 aes-256-gcm 信封，manifest 记录 ciphertextHash + plaintextHash
|
|||
|
|
双层校验；未配置密钥而 encrypt=True → ValueError（fail-closed，不静默降级）。
|
|||
|
|
keep_last: 保留条数（默认取 APS_AUDIT_LEDGER_KEEP_LAST，缺省 10000）
|
|||
|
|
Returns: {"archived": n, "kept": m, "archive": path, "worm": {...},
|
|||
|
|
"encryption": {...}}
|
|||
|
|
"""
|
|||
|
|
import os as _os
|
|||
|
|
import tempfile
|
|||
|
|
if keep_last is None:
|
|||
|
|
try:
|
|||
|
|
keep_last = int(_os.environ.get("APS_AUDIT_LEDGER_KEEP_LAST") or "10000")
|
|||
|
|
except ValueError:
|
|||
|
|
keep_last = 10000
|
|||
|
|
if keep_last < 0:
|
|||
|
|
raise ValueError("keep_last must be >= 0")
|
|||
|
|
records = self.read()
|
|||
|
|
if len(records) <= keep_last:
|
|||
|
|
return {"archived": 0, "kept": len(records), "archive": None,
|
|||
|
|
"encryption": {"enabled": False, "note": ENCRYPTION_NOTE}}
|
|||
|
|
overflow = records[:-keep_last]
|
|||
|
|
kept = records[-keep_last:]
|
|||
|
|
key_material = _key_material()
|
|||
|
|
encrypt_on = bool(key_material) if encrypt is None else bool(encrypt)
|
|||
|
|
if encrypt_on and key_material is None:
|
|||
|
|
raise ValueError(
|
|||
|
|
f"加密归档需要密钥：设置 {ENCRYPTION_ENV_KEY}（64 位 hex 或口令）"
|
|||
|
|
f"或 {ENCRYPTION_ENV_KEY_FILE}（密钥文件），或传 encrypt=False")
|
|||
|
|
archive_dir = self.path.parent / "archive"
|
|||
|
|
archive_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
month = (kept[0].get("at") or "unknown")[:7] if kept else "unknown"
|
|||
|
|
# WORM 模拟：同月归档文件已密封/已存在 → 改用 <month>.<n>.jsonl，不改写旧件
|
|||
|
|
manifest = self._read_manifest() or {}
|
|||
|
|
sealed = set((manifest.get("files") or {}).keys())
|
|||
|
|
archive_path = archive_dir / f"{month}.jsonl"
|
|||
|
|
seq = 2
|
|||
|
|
while (archive_path.name in sealed) or archive_path.exists():
|
|||
|
|
archive_path = archive_dir / f"{month}.{seq}.jsonl"
|
|||
|
|
seq += 1
|
|||
|
|
from server.agent_core.audit_filelock import locked_append
|
|||
|
|
with locked_append(archive_path) as fh:
|
|||
|
|
for rec in overflow:
|
|||
|
|
fh.write(json.dumps(rec, ensure_ascii=False, sort_keys=True) + "\n")
|
|||
|
|
# 重写主文件为保留窗口（原子替换；先写临时文件再 os.replace）
|
|||
|
|
fd, tmp = tempfile.mkstemp(dir=str(self.path.parent), suffix=".tmp")
|
|||
|
|
try:
|
|||
|
|
with _os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|||
|
|
for rec in kept:
|
|||
|
|
fh.write(json.dumps(rec, ensure_ascii=False, sort_keys=True) + "\n")
|
|||
|
|
_os.replace(tmp, self.path)
|
|||
|
|
except BaseException:
|
|||
|
|
if _os.path.exists(tmp):
|
|||
|
|
_os.unlink(tmp)
|
|||
|
|
raise
|
|||
|
|
result = {"archived": len(overflow), "kept": len(kept), "archive": str(archive_path)}
|
|||
|
|
if overflow: # WORM 模拟：密封 + 只读 + 清单
|
|||
|
|
entry = self._seal_archive(archive_path, overflow, _now_iso(),
|
|||
|
|
key_material=key_material if encrypt_on else None)
|
|||
|
|
result["worm"] = {"mode": "worm-simulated", "sealed": True,
|
|||
|
|
"file": archive_path.name, "entry": entry}
|
|||
|
|
result["encryption"] = {
|
|||
|
|
"enabled": encrypt_on,
|
|||
|
|
"scheme": ENCRYPTION_SCHEME if encrypt_on else None,
|
|||
|
|
"file": archive_path.name if encrypt_on else None,
|
|||
|
|
"note": ENCRYPTION_NOTE,
|
|||
|
|
}
|
|||
|
|
else:
|
|||
|
|
result["worm"] = {"mode": "worm-simulated", "sealed": False}
|
|||
|
|
result["encryption"] = {"enabled": False, "note": ENCRYPTION_NOTE}
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def archive_dir(self) -> Path:
|
|||
|
|
"""归档目录：<ledger_dir>/<tenant>/archive（WORM 模拟密封区）。"""
|
|||
|
|
return self.path.parent / "archive"
|
|||
|
|
|
|||
|
|
def _manifest_path(self) -> Path:
|
|||
|
|
"""WORM 清单路径（归档目录内 .worm-manifest.json）。"""
|
|||
|
|
return self.archive_dir / WORM_MANIFEST
|
|||
|
|
|
|||
|
|
def _read_manifest(self) -> dict[str, Any] | None:
|
|||
|
|
"""读取 WORM 清单；缺失/损坏返回 None（verify_archive 显式报告）。"""
|
|||
|
|
mp = self._manifest_path()
|
|||
|
|
if not mp.exists():
|
|||
|
|
return None
|
|||
|
|
try:
|
|||
|
|
data = json.loads(mp.read_text(encoding="utf-8"))
|
|||
|
|
return data if isinstance(data, dict) else None
|
|||
|
|
except (OSError, json.JSONDecodeError):
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def _write_manifest(self, manifest: dict[str, Any]) -> None:
|
|||
|
|
"""原子写 WORM 清单（临时文件 + os.replace，避免半写损坏）。"""
|
|||
|
|
import tempfile as _tf
|
|||
|
|
self.archive_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
fd, tmp = _tf.mkstemp(dir=str(self.archive_dir), suffix=".tmp")
|
|||
|
|
try:
|
|||
|
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|||
|
|
json.dump(manifest, fh, ensure_ascii=False, sort_keys=True, indent=2)
|
|||
|
|
os.replace(tmp, self._manifest_path())
|
|||
|
|
except BaseException:
|
|||
|
|
if os.path.exists(tmp):
|
|||
|
|
os.unlink(tmp)
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
def _guard_archive_sealed(self, target: Path) -> None:
|
|||
|
|
"""WORM 模拟：常规写路径拒绝改写已密封的归档文件（manifest 在册 → 抛错）。"""
|
|||
|
|
try:
|
|||
|
|
inside = target.resolve().parent == self.archive_dir.resolve()
|
|||
|
|
except OSError:
|
|||
|
|
inside = False
|
|||
|
|
if not inside:
|
|||
|
|
return
|
|||
|
|
manifest = self._read_manifest()
|
|||
|
|
if manifest and target.name in (manifest.get("files") or {}):
|
|||
|
|
raise ValueError(
|
|||
|
|
f"WORM 模拟拒绝：归档文件 {target.name} 已密封（manifest 在册），不可改写")
|
|||
|
|
|
|||
|
|
def _seal_archive(self, archive_path: Path, records: list[dict[str, Any]],
|
|||
|
|
archived_at: str,
|
|||
|
|
key_material: tuple[bytes, str] | None = None) -> dict[str, Any]:
|
|||
|
|
"""密封单个归档文件：sha256 + merkle root + 归档时间 + 覆盖范围 → 清单。
|
|||
|
|
|
|||
|
|
可选加密归档（key_material 非 None）：归档文件内容替换为 aes-256-gcm 信封，
|
|||
|
|
manifest 记录 ciphertextHash（密文文件哈希）+ plaintextHash（解密明文哈希）
|
|||
|
|
双层校验 + 算法/版本标记；密钥缺失/错误/篡改由 verify_archive 显式报告。
|
|||
|
|
密封后归档文件置只读（模拟 WORM 介质不可覆写）；清单登记为不可变承诺，
|
|||
|
|
此后 archive_old 对同月文件不再追加（自动改用 <month>.<n>.jsonl 新文件）。
|
|||
|
|
"""
|
|||
|
|
from server.agent_core.audit_merkle import merkle_root # 惰性导入（避免环）
|
|||
|
|
manifest = self._read_manifest() or {
|
|||
|
|
"mode": "worm-simulated",
|
|||
|
|
"tenant": self.tenant_uuid,
|
|||
|
|
"worldKey": self.world_key,
|
|||
|
|
"ledger": self.path.name,
|
|||
|
|
"note": WORM_NOTE,
|
|||
|
|
}
|
|||
|
|
files = manifest.setdefault("files", {})
|
|||
|
|
plaintext_bytes = archive_path.read_bytes() # archive_old 已写 JSONL
|
|||
|
|
if key_material is not None:
|
|||
|
|
envelope = _encrypt_bytes(plaintext_bytes, key_material)
|
|||
|
|
_atomic_write(archive_path, json.dumps(
|
|||
|
|
envelope, ensure_ascii=False, sort_keys=True).encode("utf-8"))
|
|||
|
|
ciphertext_sha = _file_sha256(archive_path)
|
|||
|
|
plaintext_sha = envelope["plaintextHash"]
|
|||
|
|
encryption_meta = {
|
|||
|
|
"enabled": True,
|
|||
|
|
"format": envelope["format"],
|
|||
|
|
"scheme": envelope["scheme"],
|
|||
|
|
"kdf": envelope["kdf"],
|
|||
|
|
"salt": envelope["salt"],
|
|||
|
|
"iv": envelope["iv"],
|
|||
|
|
"ciphertextHash": ciphertext_sha,
|
|||
|
|
"plaintextHash": plaintext_sha,
|
|||
|
|
}
|
|||
|
|
else:
|
|||
|
|
ciphertext_sha = plaintext_sha = _file_sha256(archive_path)
|
|||
|
|
encryption_meta = {"enabled": False, "format": None, "scheme": None}
|
|||
|
|
root = merkle_root(records)
|
|||
|
|
ats = [r.get("at") or "" for r in records]
|
|||
|
|
ids = [str(r.get("eventId") or "") for r in records if r.get("eventId")]
|
|||
|
|
try:
|
|||
|
|
os.chmod(archive_path, 0o444) # 只读位（Windows/类 Unix）
|
|||
|
|
readonly = True
|
|||
|
|
except OSError:
|
|||
|
|
readonly = False
|
|||
|
|
entry = {
|
|||
|
|
"sha256": ciphertext_sha,
|
|||
|
|
"ciphertextHash": ciphertext_sha,
|
|||
|
|
"plaintextHash": plaintext_sha,
|
|||
|
|
"merkleRoot": root,
|
|||
|
|
"archivedAt": archived_at,
|
|||
|
|
"coverage": {
|
|||
|
|
"firstAt": ats[0] if ats else None,
|
|||
|
|
"lastAt": ats[-1] if ats else None,
|
|||
|
|
"firstEventId": ids[0] if ids else None,
|
|||
|
|
"lastEventId": ids[-1] if ids else None,
|
|||
|
|
"records": len(records),
|
|||
|
|
},
|
|||
|
|
"readonly": readonly,
|
|||
|
|
"encryption": encryption_meta,
|
|||
|
|
}
|
|||
|
|
files[archive_path.name] = entry
|
|||
|
|
manifest["updatedAt"] = archived_at
|
|||
|
|
self._write_manifest(manifest)
|
|||
|
|
return entry
|
|||
|
|
|
|||
|
|
def _read_jsonl(self, path: Path) -> list[dict[str, Any]]:
|
|||
|
|
"""读取 JSONL 记录（损坏行标注 _corrupt，不中断；供归档 merkle 复算）。
|
|||
|
|
|
|||
|
|
仅用于明文归档；加密归档由 verify_archive 先解密再解析（_parse_jsonl）。
|
|||
|
|
"""
|
|||
|
|
if not path.exists():
|
|||
|
|
return []
|
|||
|
|
try:
|
|||
|
|
text = path.read_text(encoding="utf-8")
|
|||
|
|
except OSError:
|
|||
|
|
return []
|
|||
|
|
return _parse_jsonl(text)
|
|||
|
|
|
|||
|
|
def verify_archive(self, key: bytes | None = None) -> dict[str, Any]:
|
|||
|
|
"""WORM 归档完整性验证：逐文件密文哈希 + 解密 + 明文哈希 + merkle root 复算。
|
|||
|
|
|
|||
|
|
加密归档：ciphertextHash 与文件哈希不一致 → tampered（密文层篡改）；
|
|||
|
|
解密（密钥缺失 → decryptFailed "encryption-key-missing"；密钥错误/信封损坏 →
|
|||
|
|
decryptFailed "decryption-failed"，均显式失败不静默）；plaintextHash 不一致 →
|
|||
|
|
tampered（明文层篡改）；merkle 复算不一致 → merkleMismatch（语义层异常）。
|
|||
|
|
明文归档行为不变（仅 sha256 + merkle 校验）。
|
|||
|
|
key: 可选密钥材料 bytes（32B 密钥或口令）；缺省从 env/密钥文件解析。
|
|||
|
|
返回结构兼容网关 /api/gov/archive/verify（P0 只读）。
|
|||
|
|
"""
|
|||
|
|
manifest = self._read_manifest()
|
|||
|
|
key_material: bytes | None
|
|||
|
|
if key is not None:
|
|||
|
|
key_material = key
|
|||
|
|
else:
|
|||
|
|
material = _key_material()
|
|||
|
|
key_material = material[0] if material else None
|
|||
|
|
summary = {
|
|||
|
|
"files": 0, "verified": 0, "tampered": 0, "missing": 0,
|
|||
|
|
"merkleMismatch": 0, "unsealed": 0, "noManifest": manifest is None,
|
|||
|
|
"decryptFailed": 0,
|
|||
|
|
}
|
|||
|
|
tampered: list[dict[str, Any]] = []
|
|||
|
|
missing: list[dict[str, Any]] = []
|
|||
|
|
merkle_mismatch: list[dict[str, Any]] = []
|
|||
|
|
decrypt_failed: list[dict[str, Any]] = []
|
|||
|
|
unsealed: list[str] = []
|
|||
|
|
encrypted_files = 0
|
|||
|
|
archive_dir = self.archive_dir
|
|||
|
|
if archive_dir.exists():
|
|||
|
|
from server.agent_core.audit_merkle import merkle_root # 惰性导入（避免环）
|
|||
|
|
files = (manifest or {}).get("files") or {}
|
|||
|
|
for name, meta in files.items():
|
|||
|
|
summary["files"] += 1
|
|||
|
|
path = archive_dir / name
|
|||
|
|
if not path.exists():
|
|||
|
|
summary["missing"] += 1
|
|||
|
|
missing.append({"file": name, "expected": meta.get("sha256"),
|
|||
|
|
"reason": "missing"})
|
|||
|
|
continue
|
|||
|
|
actual = _file_sha256(path)
|
|||
|
|
enc = meta.get("encryption") or {}
|
|||
|
|
if enc.get("enabled"):
|
|||
|
|
encrypted_files += 1
|
|||
|
|
if actual != meta.get("ciphertextHash"):
|
|||
|
|
summary["tampered"] += 1
|
|||
|
|
tampered.append({"file": name,
|
|||
|
|
"expected": meta.get("ciphertextHash"),
|
|||
|
|
"actual": actual,
|
|||
|
|
"reason": "ciphertext-hash-mismatch"})
|
|||
|
|
continue
|
|||
|
|
if key_material is None:
|
|||
|
|
summary["decryptFailed"] += 1
|
|||
|
|
decrypt_failed.append({"file": name,
|
|||
|
|
"reason": "encryption-key-missing",
|
|||
|
|
"note": ENCRYPTION_NOTE})
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
plaintext = _decrypt_bytes(_read_envelope(path), key_material)
|
|||
|
|
except ValueError as exc:
|
|||
|
|
summary["decryptFailed"] += 1
|
|||
|
|
decrypt_failed.append({"file": name, "reason": "decryption-failed",
|
|||
|
|
"detail": str(exc)})
|
|||
|
|
continue
|
|||
|
|
if hashlib.sha256(plaintext).hexdigest() != meta.get("plaintextHash"):
|
|||
|
|
summary["tampered"] += 1
|
|||
|
|
tampered.append({"file": name,
|
|||
|
|
"expected": meta.get("plaintextHash"),
|
|||
|
|
"actual": hashlib.sha256(plaintext).hexdigest(),
|
|||
|
|
"reason": "plaintext-hash-mismatch"})
|
|||
|
|
continue
|
|||
|
|
root = merkle_root(_parse_jsonl(plaintext.decode("utf-8")))
|
|||
|
|
if root != meta.get("merkleRoot"):
|
|||
|
|
summary["merkleMismatch"] += 1
|
|||
|
|
merkle_mismatch.append({"file": name,
|
|||
|
|
"expected": meta.get("merkleRoot"),
|
|||
|
|
"actual": root,
|
|||
|
|
"reason": "merkle-root-mismatch"})
|
|||
|
|
continue
|
|||
|
|
summary["verified"] += 1
|
|||
|
|
continue
|
|||
|
|
# 明文归档（既有路径）
|
|||
|
|
if actual != meta.get("sha256"):
|
|||
|
|
summary["tampered"] += 1
|
|||
|
|
tampered.append({"file": name, "expected": meta.get("sha256"),
|
|||
|
|
"actual": actual, "reason": "sha256-mismatch"})
|
|||
|
|
continue
|
|||
|
|
root = merkle_root(self._read_jsonl(path)) # 记录级语义校验
|
|||
|
|
if root != meta.get("merkleRoot"):
|
|||
|
|
summary["merkleMismatch"] += 1
|
|||
|
|
merkle_mismatch.append({"file": name,
|
|||
|
|
"expected": meta.get("merkleRoot"),
|
|||
|
|
"actual": root,
|
|||
|
|
"reason": "merkle-root-mismatch"})
|
|||
|
|
continue
|
|||
|
|
summary["verified"] += 1
|
|||
|
|
for pth in sorted(archive_dir.iterdir()): # 未登记文件显式提示
|
|||
|
|
if (pth.is_file() and not pth.name.startswith(".")
|
|||
|
|
and pth.name not in files):
|
|||
|
|
summary["unsealed"] += 1
|
|||
|
|
unsealed.append(pth.name)
|
|||
|
|
# 无清单且无文件 = 空态（无归档可验），完整性判为空真；有文件无清单则 files>0 → 不可验
|
|||
|
|
ok = (summary["tampered"] == 0 and summary["missing"] == 0
|
|||
|
|
and summary["merkleMismatch"] == 0 and summary["decryptFailed"] == 0
|
|||
|
|
and (manifest is not None or summary["files"] == 0))
|
|||
|
|
return {
|
|||
|
|
"mode": "worm-simulated",
|
|||
|
|
"ok": ok,
|
|||
|
|
"manifest": manifest,
|
|||
|
|
"summary": summary,
|
|||
|
|
"tampered": tampered,
|
|||
|
|
"missing": missing,
|
|||
|
|
"merkleMismatch": merkle_mismatch,
|
|||
|
|
"decryptFailed": decrypt_failed,
|
|||
|
|
"unsealed": unsealed,
|
|||
|
|
"encryption": {
|
|||
|
|
"enabled": encrypted_files > 0,
|
|||
|
|
"count": encrypted_files,
|
|||
|
|
"scheme": ENCRYPTION_SCHEME if encrypted_files else None,
|
|||
|
|
"note": ENCRYPTION_NOTE,
|
|||
|
|
},
|
|||
|
|
"note": WORM_NOTE,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def status(self, events: list[dict[str, Any]]) -> dict[str, Any]:
|
|||
|
|
"""当前锚定状态：最新记录 vs 当前事件根；无记录时返回待锚定根。"""
|
|||
|
|
records = self.read()
|
|||
|
|
current_root = audit_root(events)
|
|||
|
|
if not records:
|
|||
|
|
return {
|
|||
|
|
"anchored": False,
|
|||
|
|
"ok": False,
|
|||
|
|
"currentRoot": current_root,
|
|||
|
|
"currentCount": len(events),
|
|||
|
|
"records": 0,
|
|||
|
|
"latest": None,
|
|||
|
|
"reason": "not-anchored",
|
|||
|
|
}
|
|||
|
|
latest = records[-1]
|
|||
|
|
if latest.get("_corrupt"):
|
|||
|
|
return {
|
|||
|
|
"anchored": True,
|
|||
|
|
"ok": False,
|
|||
|
|
"currentRoot": current_root,
|
|||
|
|
"currentCount": len(events),
|
|||
|
|
"records": len(records),
|
|||
|
|
"latest": None,
|
|||
|
|
"reason": "corrupt-ledger",
|
|||
|
|
}
|
|||
|
|
match = latest.get("root") == current_root and latest.get("count") == len(events)
|
|||
|
|
return {
|
|||
|
|
"anchored": True,
|
|||
|
|
"ok": match,
|
|||
|
|
"currentRoot": current_root,
|
|||
|
|
"currentCount": len(events),
|
|||
|
|
"records": len(records),
|
|||
|
|
"latest": {
|
|||
|
|
"root": latest.get("root"),
|
|||
|
|
"count": latest.get("count"),
|
|||
|
|
"eventId": latest.get("eventId"),
|
|||
|
|
"at": latest.get("at"),
|
|||
|
|
},
|
|||
|
|
"reason": None if match else "mismatch",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def verify(self, events: list[dict[str, Any]]) -> dict[str, Any]:
|
|||
|
|
"""校验：锚定存在且最新记录与当前事件根一致（篡改检测）。"""
|
|||
|
|
return self.status(events)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _now_iso() -> str:
|
|||
|
|
from server.timeutil import fmt_dt
|
|||
|
|
from datetime import datetime
|
|||
|
|
return fmt_dt(datetime.now())
|