aps-agent/server/knowledge/ingest.py

412 lines
15 KiB
Python
Raw Permalink Normal View History

# ============================================================
# 知识文档导入与切块(moduleId: knowledge-ingest, 可重生 ✅)
# plan.md §8.2:PDF/docx/md/txt → chunk → 知识资产(版本化+审批)
# ============================================================
from __future__ import annotations
import os
import re
from typing import Any
# 切块目标长度(字符)与重叠
_CHUNK_SIZE = 500
_CHUNK_OVERLAP = 50
def _ext(path: str) -> str:
return os.path.splitext(path)[1].lower()
def extract_text_from_bytes(filename: str, raw: bytes) -> list[dict[str, Any]]:
"""按扩展名解析文件为带页/段元数据的文本块列表(尚未切 chunk)。
Returns: [{text, page?, heading?}, ...]
"""
name = filename or "upload.txt"
ext = _ext(name)
if ext in (".md", ".txt", ".markdown"):
text = raw.decode("utf-8", errors="replace")
return _split_markdown_sections(text)
if ext == ".pdf":
return _extract_pdf(raw)
if ext in (".docx",):
return _extract_docx(raw)
if ext in (".xlsx", ".xlsm"):
return _extract_xlsx(raw)
# 兜底当纯文本
try:
return [{"text": raw.decode("utf-8"), "page": 1}]
except UnicodeDecodeError as exc:
raise ValueError(f"不支持的文件类型或编码:{name}") from exc
def extract_text_from_path(path: str) -> list[dict[str, Any]]:
"""从本地路径读取并解析。"""
if not os.path.isfile(path):
raise FileNotFoundError(f"文件不存在:{path}")
with open(path, "rb") as f:
raw = f.read()
return extract_text_from_bytes(os.path.basename(path), raw)
def _split_markdown_sections(text: str) -> list[dict[str, Any]]:
parts: list[dict[str, Any]] = []
current_heading = ""
buf: list[str] = []
for line in text.splitlines():
if re.match(r"^#{1,3}\s+", line):
if buf:
parts.append({"text": "\n".join(buf).strip(), "heading": current_heading or None})
buf = []
current_heading = re.sub(r"^#{1,3}\s+", "", line).strip()
buf.append(line)
else:
buf.append(line)
if buf:
parts.append({"text": "\n".join(buf).strip(), "heading": current_heading or None})
return [p for p in parts if p["text"]]
def _extract_pdf(raw: bytes) -> list[dict[str, Any]]:
try:
import pdfplumber
except ImportError as exc:
raise ValueError("解析 PDF 需要安装 pdfplumber:pip install pdfplumber") from exc
from io import BytesIO
pages: list[dict[str, Any]] = []
with pdfplumber.open(BytesIO(raw)) as pdf:
for i, page in enumerate(pdf.pages, start=1):
t = (page.extract_text() or "").strip()
if t:
pages.append({"text": t, "page": i})
if not pages:
raise ValueError("PDF 未提取到文本(可能是扫描件)")
return pages
def _extract_docx(raw: bytes) -> list[dict[str, Any]]:
try:
from docx import Document
except ImportError as exc:
raise ValueError("解析 docx 需要安装 python-docx:pip install python-docx") from exc
from io import BytesIO
doc = Document(BytesIO(raw))
paras = [p.text.strip() for p in doc.paragraphs if p.text and p.text.strip()]
if not paras:
raise ValueError("docx 未提取到文本")
return [{"text": "\n".join(paras), "page": 1}]
def _extract_xlsx(raw: bytes) -> list[dict[str, Any]]:
from io import BytesIO
from openpyxl import load_workbook
wb = load_workbook(BytesIO(raw), data_only=True, read_only=True)
parts: list[dict[str, Any]] = []
for sheet in wb.worksheets:
rows = []
for row in sheet.iter_rows(values_only=True):
cells = [str(c) for c in row if c is not None and str(c).strip()]
if cells:
rows.append(" | ".join(cells))
if rows:
parts.append({"text": f"# {sheet.title}\n" + "\n".join(rows), "heading": sheet.title})
if not parts:
raise ValueError("xlsx 无有效单元格")
return parts
def chunk_sections(sections: list[dict[str, Any]], *,
size: int = _CHUNK_SIZE, overlap: int = _CHUNK_OVERLAP) -> list[dict[str, Any]]:
"""把段落/页切成固定长度 chunk(带重叠)。"""
chunks: list[dict[str, Any]] = []
idx = 0
for sec in sections:
text = (sec.get("text") or "").strip()
if not text:
continue
heading = sec.get("heading")
page = sec.get("page")
if len(text) <= size:
chunks.append({
"chunkId": f"c{idx:04d}", "text": text,
"heading": heading, "page": page, "seq": idx,
})
idx += 1
continue
start = 0
while start < len(text):
end = min(start + size, len(text))
piece = text[start:end].strip()
if piece:
chunks.append({
"chunkId": f"c{idx:04d}", "text": piece,
"heading": heading, "page": page, "seq": idx,
})
idx += 1
if end >= len(text):
break
start = max(end - overlap, start + 1)
return chunks
def preview_ingest(filename: str, raw: bytes | None = None, path: str | None = None,
*, kind: str = "sop", title: str | None = None) -> dict[str, Any]:
"""预览导入:解析+切块,不写库。"""
if path:
sections = extract_text_from_path(path)
fname = os.path.basename(path)
elif raw is not None:
sections = extract_text_from_bytes(filename, raw)
fname = filename
else:
raise ValueError("需要提供 path 或 raw")
chunks = chunk_sections(sections)
title = title or os.path.splitext(fname)[0]
return {
"filename": fname,
"title": title,
"kind": kind,
"sectionCount": len(sections),
"chunkCount": len(chunks),
"preview": [{"seq": c["seq"], "heading": c.get("heading"), "page": c.get("page"),
"chars": len(c["text"]), "snippet": c["text"][:80]} for c in chunks[:8]],
"chunks": chunks,
}
def confirmation_for_knowledge_import(preview: dict[str, Any]) -> tuple[str, list[str]]:
"""P2 确认卡文案。"""
title = f"导入知识文档《{preview.get('title')}》"
lines = [
f"文件:{preview.get('filename')}",
f"类型:{preview.get('kind')}",
f"切块:{preview.get('chunkCount')} 段(来自 {preview.get('sectionCount')} 节/页)",
"入库后可检索;默认已审批(approved=true)。",
]
return title, lines
def apply_knowledge_import(preview: dict[str, Any], *, tags: list[str] | None = None,
store: Any = None) -> dict[str, Any]:
"""把预览结果写入知识库(带 chunks)。"""
from server.knowledge.assets import get_knowledge
kb = store or get_knowledge()
chunks = preview.get("chunks") or []
content = "\n\n".join(
(f"【{c.get('heading') or '段落'}】\n" if c.get("heading") else "") + c["text"]
for c in chunks
)
if not content.strip():
raise ValueError("无正文可入库")
asset = kb.add_with_chunks(
kind=str(preview.get("kind") or "sop"),
title=str(preview.get("title") or preview.get("filename") or "导入文档"),
content=content[:8000], # 资产摘要正文(检索仍优先 chunk)
chunks=chunks,
tags=tags or ["导入", str(preview.get("filename") or "")],
source=str(preview.get("filename") or ""),
approved=True,
)
return {"assetId": asset["assetId"], "title": asset["title"],
"version": asset["version"], "chunkCount": len(asset.get("chunks") or [])}
def bind_world_to_identity(world: dict[str, Any], *, sources: list[str] | None = None) -> dict[str, Any]:
"""把当前登录用户/租户写到世界元数据(分析/导入后绑定归属)。"""
from datetime import datetime
from server.auth.context import get_identity
from server.timeutil import fmt_dt
identity = get_identity()
meta = dict(world.get("meta") or {})
meta.update({
"ownerUserId": identity.user_id,
"ownerUsername": identity.username,
"ownerFullname": identity.fullname,
"tenantUuid": identity.tenant_uuid,
"boundAt": fmt_dt(datetime.now()),
"sources": list(sources or meta.get("sources") or []),
})
world["meta"] = meta
return meta
def _knowledge_cell(value: Any) -> str:
if value is None:
return ""
if isinstance(value, (list, tuple, set)):
value = "、".join(str(item) for item in value if item not in (None, ""))
return str(value).replace("\r", " ").replace("\n", " ").strip()
def _append_rows_section(
sections: list[dict[str, Any]],
*,
heading: str,
rows: list[dict[str, Any]],
columns: list[tuple[str, str]],
) -> None:
if not rows:
return
lines = [f"# {heading}", f"记录数:{len(rows)}"]
for row in rows:
fields = [
f"{label}={_knowledge_cell(row.get(key))}"
for key, label in columns
if _knowledge_cell(row.get(key))
]
if fields:
lines.append("- " + ";".join(fields))
sections.append({"heading": heading, "text": "\n".join(lines)})
def _project_source_sections(paths: list[str]) -> list[dict[str, Any]]:
"""保留源文件全部可读字段;标准化失败的列也能进入知识库检索。"""
sections: list[dict[str, Any]] = []
seen: set[str] = set()
for raw_path in paths[:32]:
path = os.path.abspath(str(raw_path or ""))
if not path or path in seen or not os.path.isfile(path):
continue
seen.add(path)
if os.path.getsize(path) > 25 * 1024 * 1024:
continue
name = os.path.basename(path)
try:
extracted = extract_text_from_path(path)
except Exception:
if _ext(path) not in (".csv", ".txt"):
continue
try:
with open(path, "rb") as source:
raw = source.read()
except OSError:
continue
text = ""
for encoding in ("utf-8-sig", "gb18030"):
try:
text = raw.decode(encoding)
break
except UnicodeDecodeError:
continue
extracted = [{"text": text, "heading": name}] if text.strip() else []
for section in extracted:
body = str(section.get("text") or "").strip()
if not body:
continue
heading = str(section.get("heading") or "原始内容")
sections.append({
"heading": f"源文件·{name}·{heading}",
"page": section.get("page"),
"text": f"# 源文件 {name}\n## {heading}\n{body}",
})
return sections
def ingest_project_analyze_report(
deep: dict[str, Any],
*,
world: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""项目分析结果 → 当前租户知识库(可检索)+ 绑定登录用户。
- 知识库按 tenant_uuid 隔离(get_knowledge)
- 资产标签含用户/项目/来源,便于问答引用
"""
from server.auth.context import get_identity
from server.knowledge.embedding import index_units
from server.knowledge.assets import get_knowledge
identity = get_identity()
if world is not None:
bind_world_to_identity(world, sources=list(deep.get("sources") or []))
proj = str(deep.get("projectName") or "当前项目")
summary = deep.get("summary") or {}
sections: list[dict[str, Any]] = []
md = (deep.get("markdown") or "").strip()
if md:
sections.append({"text": md, "heading": f"项目分析·{proj}"})
# 原始表格作为数据知识保存,未知列不会因主数据映射失败而丢失。
sections.extend(_project_source_sections(list(deep.get("sourcePaths") or [])))
# 卡片只展示抽样,知识库保存项目世界中的完整标准化数据集。
source = world or {}
datasets: list[tuple[str, list[dict[str, Any]], list[tuple[str, str]]]] = [
(f"{proj} 物料主数据", list(source.get("flexMaterials") or deep.get("materials") or []), [
("code", "物料编码"), ("name", "物料名称"), ("type", "类型"),
("unit", "单位"), ("stock", "库存"), ("sourcingType", "来源"),
]),
(f"{proj} BOM", list(source.get("flexBom") or []), [
("productCode", "父项"), ("materialCode", "子项"),
("quantity", "用量"), ("isKey", "关键件"),
]),
(f"{proj} 工艺路线", list(source.get("flexRoutings") or deep.get("routings") or []), [
("productCode", "产品"), ("seq", "顺序"), ("operationCode", "工序编码"),
("operationName", "工序名称"), ("stdTimePerUnit", "标准工时"),
("sourcingType", "来源"),
]),
(f"{proj} 设备台账", list(source.get("flexEquipment") or deep.get("equipment") or []), [
("code", "设备编码"), ("name", "设备名称"), ("status", "状态"),
("financeCode", "财务编号"), ("internalCode", "厂内编号"),
("assetCode", "固定资产编号"), ("factoryCode", "出厂编号"),
("capabilities", "能力"), ("zone", "区域"), ("availabilityRate", "可用率"),
]),
(f"{proj} 项目订单", list(source.get("flexOrders") or deep.get("orders") or []), [
("orderNo", "订单号"), ("productCode", "产品"), ("productName", "产品名称"),
("quantity", "数量"), ("dueDate", "交期"), ("status", "状态"),
("customerName", "客户"),
]),
(f"{proj} 工序字典", list(source.get("flexOperations") or []), [
("code", "工序编码"), ("name", "工序名称"), ("type", "类型"),
("sourcingType", "来源"), ("changeoverMin", "换型时间"),
]),
(f"{proj} 生产区域", list(source.get("flexZones") or []), [
("code", "区域编码"), ("name", "区域名称"),
]),
]
for heading, rows, columns in datasets:
_append_rows_section(sections, heading=heading, rows=rows, columns=columns)
plan = deep.get("plan") or []
if plan:
sections.append({
"heading": "补数计划",
"text": f"# {proj} 补数 Plan\n" + "\n".join(f"- {p}" for p in plan[:20]),
})
if not sections:
return {"skipped": True, "reason": "无分析正文可入库"}
chunks = chunk_sections(sections)
title = f"项目分析·{proj}"
tags = [
"项目分析", "自动入库", "完整数据集",
f"user:{identity.username}",
f"uid:{identity.user_id}",
f"tenant:{identity.tenant_uuid}",
proj,
]
for src in (deep.get("sources") or [])[:6]:
tags.append(str(src)[:48])
preview = {
"filename": f"project-analyze-{identity.user_id}.md",
"title": title,
"kind": "case",
"sectionCount": len(sections),
"chunkCount": len(chunks),
"chunks": chunks,
}
applied = apply_knowledge_import(preview, tags=tags)
try:
index_units(get_knowledge().iter_search_units())
applied["indexed"] = True
except Exception:
applied["indexed"] = False
applied["ownerUserId"] = identity.user_id
applied["tenantUuid"] = identity.tenant_uuid
applied["summary"] = summary
return applied