1027 lines
39 KiB
Python
1027 lines
39 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import csv
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
import tempfile
|
||
|
|
import uuid
|
||
|
|
from collections.abc import Iterable
|
||
|
|
from copy import deepcopy
|
||
|
|
from datetime import date, datetime, time
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
from zoneinfo import ZoneInfo
|
||
|
|
|
||
|
|
from .config import DATA_DISCLAIMER, GeneratorConfig
|
||
|
|
from .models import DatasetBundle, business_digest, canonical_json
|
||
|
|
from .rag_skills import generate_rag_and_skills
|
||
|
|
from .registry import TABLE_ORDER, TABLE_SPECS
|
||
|
|
from .schemas import build_schema_documents
|
||
|
|
|
||
|
|
_DATASET_GENERATED_AT_DATE = date(2026, 8, 4)
|
||
|
|
_DATASET_GENERATED_AT_BASIS = "DETERMINISTIC_BUILD_DATE"
|
||
|
|
|
||
|
|
_CANONICAL_SCENARIOS: tuple[str, ...] = (
|
||
|
|
"baseline",
|
||
|
|
"material-delay",
|
||
|
|
"crane-failure",
|
||
|
|
"dock-delay",
|
||
|
|
"design-change",
|
||
|
|
"scenario-comparison",
|
||
|
|
)
|
||
|
|
_EXPECTED_ARTIFACTS: tuple[str, ...] = (
|
||
|
|
"baseline-results",
|
||
|
|
"baseline-kpis",
|
||
|
|
"expected-conflicts",
|
||
|
|
"expected-explanations",
|
||
|
|
)
|
||
|
|
_VALIDATION_JSON: tuple[str, ...] = (
|
||
|
|
"validation-report",
|
||
|
|
"referential-integrity",
|
||
|
|
"constraint-validation",
|
||
|
|
)
|
||
|
|
_FLEX_KEYS: tuple[str, ...] = (
|
||
|
|
"flexMaterials",
|
||
|
|
"flexBom",
|
||
|
|
"flexOrders",
|
||
|
|
"flexRoutings",
|
||
|
|
"flexEquipment",
|
||
|
|
"flexScheduleVersions",
|
||
|
|
"flexWorkOrders",
|
||
|
|
"purchaseSuggestions",
|
||
|
|
"purchaseOrders",
|
||
|
|
"outsourceSuggestions",
|
||
|
|
"outsourceOrders",
|
||
|
|
"makeSuggestions",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _json_ready(value: Any) -> Any:
|
||
|
|
if isinstance(value, dict):
|
||
|
|
return {str(key): _json_ready(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))}
|
||
|
|
if isinstance(value, (list, tuple)):
|
||
|
|
return [_json_ready(item) for item in value]
|
||
|
|
if isinstance(value, set):
|
||
|
|
return [_json_ready(item) for item in sorted(value, key=str)]
|
||
|
|
if isinstance(value, (date, datetime)):
|
||
|
|
return value.isoformat()
|
||
|
|
if isinstance(value, Path):
|
||
|
|
return value.as_posix()
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def _json_bytes(value: Any) -> bytes:
|
||
|
|
text = json.dumps(
|
||
|
|
_json_ready(value),
|
||
|
|
ensure_ascii=False,
|
||
|
|
allow_nan=False,
|
||
|
|
indent=2,
|
||
|
|
sort_keys=True,
|
||
|
|
)
|
||
|
|
return (text + "\n").encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def _write_bytes(root: Path, relative: str, payload: bytes) -> None:
|
||
|
|
target = root / Path(relative)
|
||
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
target.write_bytes(payload)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_json(root: Path, relative: str, value: Any) -> int | None:
|
||
|
|
_write_bytes(root, relative, _json_bytes(value))
|
||
|
|
if isinstance(value, list):
|
||
|
|
return len(value)
|
||
|
|
if isinstance(value, dict):
|
||
|
|
return len(value)
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _write_text(root: Path, relative: str, text: str) -> None:
|
||
|
|
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
|
||
|
|
if not normalized.endswith("\n"):
|
||
|
|
normalized += "\n"
|
||
|
|
_write_bytes(root, relative, normalized.encode("utf-8"))
|
||
|
|
|
||
|
|
|
||
|
|
def _csv_cell(value: Any) -> str:
|
||
|
|
if value is None:
|
||
|
|
return ""
|
||
|
|
if isinstance(value, bool):
|
||
|
|
return "true" if value else "false"
|
||
|
|
if isinstance(value, (dict, list, tuple, set)):
|
||
|
|
return canonical_json(_json_ready(value))
|
||
|
|
if isinstance(value, (date, datetime)):
|
||
|
|
return value.isoformat()
|
||
|
|
return str(value)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_csv(root: Path, relative: str, table_name: str, rows: list[dict[str, Any]]) -> int:
|
||
|
|
spec = TABLE_SPECS[table_name]
|
||
|
|
extra = sorted({str(key) for row in rows for key in row if key not in spec.required})
|
||
|
|
fieldnames = [*spec.required, *extra]
|
||
|
|
target = root / Path(relative)
|
||
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
with target.open("w", encoding="utf-8", newline="") as handle:
|
||
|
|
writer = csv.DictWriter(
|
||
|
|
handle,
|
||
|
|
fieldnames=fieldnames,
|
||
|
|
extrasaction="ignore",
|
||
|
|
lineterminator="\n",
|
||
|
|
)
|
||
|
|
writer.writeheader()
|
||
|
|
for row in rows:
|
||
|
|
writer.writerow({field: _csv_cell(row.get(field)) for field in fieldnames})
|
||
|
|
return len(rows)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_jsonl(root: Path, relative: str, rows: Iterable[dict[str, Any]]) -> int:
|
||
|
|
normalized = list(rows)
|
||
|
|
payload = "".join(canonical_json(_json_ready(row)) + "\n" for row in normalized)
|
||
|
|
_write_bytes(root, relative, payload.encode("utf-8"))
|
||
|
|
return len(normalized)
|
||
|
|
|
||
|
|
|
||
|
|
def _artifact(bundle: DatasetBundle, *keys: str) -> Any | None:
|
||
|
|
for key in keys:
|
||
|
|
if key in bundle.artifacts:
|
||
|
|
return deepcopy(bundle.artifacts[key])
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _placeholder(name: str) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"artifact": name,
|
||
|
|
"datasetType": "SYNTHETIC",
|
||
|
|
"source": "bundle.artifacts",
|
||
|
|
"status": "NOT_GENERATED",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _generated_at(config: GeneratorConfig) -> str:
|
||
|
|
zone = ZoneInfo(config.timezone)
|
||
|
|
return datetime.combine(_DATASET_GENERATED_AT_DATE, time.min, tzinfo=zone).isoformat()
|
||
|
|
|
||
|
|
|
||
|
|
def _readme(bundle: DatasetBundle, config: GeneratorConfig, digest: str) -> str:
|
||
|
|
return f"""# Beihai Shipyard APS Synthetic Dataset
|
||
|
|
|
||
|
|
> {DATA_DISCLAIMER}
|
||
|
|
|
||
|
|
- datasetType: `SYNTHETIC`
|
||
|
|
- organizationScenario: `BEIHAI_SHIPYARD_APS`
|
||
|
|
- generatedFor: `APS development and validation`
|
||
|
|
- timezone: `{config.timezone}`
|
||
|
|
- planningBaseDate: `{config.planning_base_date.isoformat()}`
|
||
|
|
- planningHorizonEnd: `{config.planning_horizon_end.isoformat()}`
|
||
|
|
- randomSeed: `{config.seed}`
|
||
|
|
- datasetProfile: `{config.scale.upper()}`
|
||
|
|
- projectCount: `{config.profile.project_count}`
|
||
|
|
- businessDigest: `{digest}`
|
||
|
|
|
||
|
|
This directory is produced by a deterministic generator. `world.json` is the
|
||
|
|
canonical projection and the 51 CSV files are table projections of the same
|
||
|
|
version. `manifest.json` records byte size, row count, and SHA-256 for every
|
||
|
|
payload file. All projects, people, suppliers, owners, and knowledge assets are
|
||
|
|
synthetic and must not be represented as real ERP/MES/PLM/WMS/SRM/QMS data.
|
||
|
|
|
||
|
|
## Directories
|
||
|
|
|
||
|
|
- `master/`, `projects/`, `engineering/`, `materials/`, `planning/`, `execution/`
|
||
|
|
- `schemas/`: domain and table JSON Schema documents
|
||
|
|
- `scenarios/`: baseline, disruptions, alternatives, and before/after diffs
|
||
|
|
- `rag/`: SYNTHETIC_KNOWLEDGE assets
|
||
|
|
- `skills/`: the exact 20 APS skills and input/output schemas
|
||
|
|
- `expected/`: expected results and explanations
|
||
|
|
- `validation/`: independent validator artifacts
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
def _data_dictionary() -> str:
|
||
|
|
lines = [
|
||
|
|
"# Beihai Shipyard APS Synthetic Data Dictionary",
|
||
|
|
"",
|
||
|
|
f"> {DATA_DISCLAIMER}",
|
||
|
|
"",
|
||
|
|
"All CSV files use UTF-8, ISO 8601 dates, and lowerCamelCase fields.",
|
||
|
|
"",
|
||
|
|
"| Table | Path | Primary key | Required fields | Foreign keys |",
|
||
|
|
"|---|---|---|---|---|",
|
||
|
|
]
|
||
|
|
for name in TABLE_ORDER:
|
||
|
|
spec = TABLE_SPECS[name]
|
||
|
|
foreign_keys = "; ".join(
|
||
|
|
f"{fk.field}->{fk.target_table}.{fk.target_field}"
|
||
|
|
+ (" (nullable)" if fk.nullable else "")
|
||
|
|
for fk in spec.foreign_keys
|
||
|
|
) or "-"
|
||
|
|
lines.append(
|
||
|
|
f"| `{name}` | `{spec.path}` | `{spec.primary_key}` | "
|
||
|
|
f"`{', '.join(spec.required)}` | {foreign_keys} |"
|
||
|
|
)
|
||
|
|
lines.extend(
|
||
|
|
(
|
||
|
|
"",
|
||
|
|
"## Frozen semantics",
|
||
|
|
"",
|
||
|
|
"- sourcingMode: MAKE / BUY / OUTSOURCE / OWNER_SUPPLIED",
|
||
|
|
"- fulfillmentMode: STOCK / TRANSFER / PLANNED_RECEIPT / NEW_SUPPLY / DESIGN_PENDING",
|
||
|
|
"- relationType: FS / SS / FF / SF",
|
||
|
|
"- duration: hour; weight: tonne; length: meter; currency: CNY",
|
||
|
|
"- priority: 1-10, where 10 is highest",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return "\n".join(lines) + "\n"
|
||
|
|
|
||
|
|
|
||
|
|
def _lookup(rows: list[dict[str, Any]], *fields: str) -> dict[str, dict[str, Any]]:
|
||
|
|
result: dict[str, dict[str, Any]] = {}
|
||
|
|
for row in rows:
|
||
|
|
for field in fields:
|
||
|
|
value = row.get(field)
|
||
|
|
if value not in (None, ""):
|
||
|
|
result[str(value)] = row
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _projection_overrides(bundle: DatasetBundle) -> dict[str, Any]:
|
||
|
|
projections: dict[str, Any] = {}
|
||
|
|
for artifact_key in ("world-projection", "worldProjection", "flex-projections", "flexProjections"):
|
||
|
|
value = bundle.artifacts.get(artifact_key)
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for key in _FLEX_KEYS:
|
||
|
|
if key in value:
|
||
|
|
projections[key] = deepcopy(value[key])
|
||
|
|
value = bundle.artifacts.get("world")
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for key in _FLEX_KEYS:
|
||
|
|
if key in value:
|
||
|
|
projections[key] = deepcopy(value[key])
|
||
|
|
return projections
|
||
|
|
|
||
|
|
|
||
|
|
def _build_flex_projections(bundle: DatasetBundle) -> dict[str, Any]:
|
||
|
|
projections = _projection_overrides(bundle)
|
||
|
|
materials = bundle.rows("materials")
|
||
|
|
projects = bundle.rows("ship-projects")
|
||
|
|
project_index = _lookup(projects, "projectId", "projectCode")
|
||
|
|
if "flexMaterials" not in projections:
|
||
|
|
flex_materials: list[dict[str, Any]] = []
|
||
|
|
for row in materials:
|
||
|
|
explicit_sourcing = (
|
||
|
|
row.get("sourcingMode")
|
||
|
|
or row.get("procurementType")
|
||
|
|
or row.get("sourcingType")
|
||
|
|
)
|
||
|
|
flex_materials.append(
|
||
|
|
{
|
||
|
|
"id": row.get("materialId"),
|
||
|
|
"code": row.get("materialCode", row.get("code", row.get("materialId"))),
|
||
|
|
"name": row.get("name", row.get("materialName", row.get("materialId"))),
|
||
|
|
"type": row.get("materialType", "RAW_MATERIAL"),
|
||
|
|
"unit": row.get("unit", "PCS"),
|
||
|
|
"category": row.get("materialGroup", row.get("category")),
|
||
|
|
"sourcingMode": explicit_sourcing,
|
||
|
|
"procurementType": row.get("procurementType") or explicit_sourcing,
|
||
|
|
"sourcingType": explicit_sourcing,
|
||
|
|
"sourcingTypeSource": "EXPLICIT",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
flex_materials.extend(
|
||
|
|
{
|
||
|
|
"id": row.get("projectId"),
|
||
|
|
"code": row.get("projectCode", row.get("projectId")),
|
||
|
|
"name": row.get("name", row.get("projectCode", row.get("projectId"))),
|
||
|
|
"type": "FINISHED_PRODUCT",
|
||
|
|
"unit": "VESSEL",
|
||
|
|
"category": row.get("shipType", "SHIP_PROJECT"),
|
||
|
|
"sourcingMode": "MAKE",
|
||
|
|
"procurementType": "MAKE",
|
||
|
|
"sourcingType": "MAKE",
|
||
|
|
"sourcingTypeSource": "EXPLICIT",
|
||
|
|
}
|
||
|
|
for row in projects
|
||
|
|
)
|
||
|
|
projections["flexMaterials"] = flex_materials
|
||
|
|
|
||
|
|
mbom = bundle.rows("mbom")
|
||
|
|
if mbom and "flexBom" not in projections:
|
||
|
|
projections["flexBom"] = [
|
||
|
|
{
|
||
|
|
"id": row.get("mbomLineId", row.get("bomLineId")),
|
||
|
|
"parentCode": row.get("parentMaterialId", row.get("parentCode")),
|
||
|
|
"childCode": row.get("materialId", row.get("childMaterialId", row.get("childCode"))),
|
||
|
|
"qty": row.get("quantity", row.get("qty", 0)),
|
||
|
|
"unit": row.get("unit", "PCS"),
|
||
|
|
}
|
||
|
|
for row in mbom
|
||
|
|
]
|
||
|
|
|
||
|
|
production_orders = bundle.rows("production-orders")
|
||
|
|
material_index = _lookup(materials, "materialId", "materialCode", "code")
|
||
|
|
if production_orders and "flexOrders" not in projections:
|
||
|
|
flex_orders: list[dict[str, Any]] = []
|
||
|
|
for row in production_orders:
|
||
|
|
material = material_index.get(str(row.get("materialId")), {})
|
||
|
|
project = project_index.get(str(row.get("projectId")), {})
|
||
|
|
order_no = row.get("productionOrderCode", row.get("orderNo", row.get("productionOrderId")))
|
||
|
|
flex_orders.append(
|
||
|
|
{
|
||
|
|
"id": row.get("productionOrderId"),
|
||
|
|
"orderNo": order_no,
|
||
|
|
"projectId": row.get("projectId"),
|
||
|
|
"materialCode": material.get(
|
||
|
|
"materialCode",
|
||
|
|
material.get(
|
||
|
|
"code",
|
||
|
|
project.get("projectCode", row.get("projectCode", row.get("productionOrderId"))),
|
||
|
|
),
|
||
|
|
),
|
||
|
|
"qty": row.get("quantity", row.get("qty", 1)),
|
||
|
|
"dueDate": row.get("dueDate", row.get("needDate")),
|
||
|
|
"status": row.get("status", "PLANNED"),
|
||
|
|
"routingId": row.get("routingId"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
projections["flexOrders"] = flex_orders
|
||
|
|
|
||
|
|
routing_operations = bundle.rows("routing-operations")
|
||
|
|
operations_by_routing: dict[str, list[dict[str, Any]]] = {}
|
||
|
|
for operation in routing_operations:
|
||
|
|
routing_id = str(operation.get("routingId", ""))
|
||
|
|
operations_by_routing.setdefault(routing_id, []).append(
|
||
|
|
{
|
||
|
|
"operationCode": operation.get("operationCode", operation.get("code")),
|
||
|
|
"sequence": operation.get("sequence", operation.get("sequenceNo")),
|
||
|
|
"durationHours": operation.get("durationHours", operation.get("runHours")),
|
||
|
|
"resourceGroupId": operation.get("resourceGroupId"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
routings = bundle.rows("routings")
|
||
|
|
if routings and "flexRoutings" not in projections:
|
||
|
|
projections["flexRoutings"] = [
|
||
|
|
{
|
||
|
|
"id": row.get("routingId"),
|
||
|
|
"code": row.get("routingCode", row.get("code", row.get("routingId"))),
|
||
|
|
"materialId": row.get("materialId"),
|
||
|
|
"operations": sorted(
|
||
|
|
operations_by_routing.get(str(row.get("routingId")), []),
|
||
|
|
key=canonical_json,
|
||
|
|
),
|
||
|
|
}
|
||
|
|
for row in routings
|
||
|
|
]
|
||
|
|
|
||
|
|
resources = bundle.rows("resources")
|
||
|
|
resource_index = _lookup(resources, "resourceId", "resourceCode", "code")
|
||
|
|
if resources and "flexEquipment" not in projections:
|
||
|
|
projections["flexEquipment"] = [
|
||
|
|
{
|
||
|
|
"id": row.get("resourceId"),
|
||
|
|
"code": row.get("resourceCode", row.get("code", row.get("resourceId"))),
|
||
|
|
"name": row.get("resourceName", row.get("name", row.get("resourceId"))),
|
||
|
|
"zone": row.get("transportZone", row.get("zone")),
|
||
|
|
"resourceType": row.get("resourceType"),
|
||
|
|
"workcenterId": row.get("workcenterId"),
|
||
|
|
"capacity": row.get("capacity", 1),
|
||
|
|
"exclusive": row.get("exclusive", False),
|
||
|
|
}
|
||
|
|
for row in resources
|
||
|
|
]
|
||
|
|
|
||
|
|
zones = bundle.rows("zones")
|
||
|
|
zone_index = _lookup(zones, "zoneId", "zoneCode", "code")
|
||
|
|
if zones and "flexZones" not in projections:
|
||
|
|
projections["flexZones"] = [
|
||
|
|
{
|
||
|
|
"id": row.get("zoneId"),
|
||
|
|
"code": row.get("zoneCode", row.get("code", row.get("zoneId"))),
|
||
|
|
"name": row.get("name", row.get("zoneId")),
|
||
|
|
"type": row.get("zoneType"),
|
||
|
|
}
|
||
|
|
for row in zones
|
||
|
|
]
|
||
|
|
|
||
|
|
versions = bundle.rows("schedule-versions")
|
||
|
|
if versions and "flexScheduleVersions" not in projections:
|
||
|
|
projections["flexScheduleVersions"] = [
|
||
|
|
{
|
||
|
|
"id": row.get("scheduleVersionId"),
|
||
|
|
"versionNo": row.get("versionNo"),
|
||
|
|
"status": row.get("solveStatus", row.get("status")),
|
||
|
|
"scenarioId": row.get("scenarioId"),
|
||
|
|
"algorithm": row.get("algorithm"),
|
||
|
|
"inputDigest": row.get("inputDigest"),
|
||
|
|
"sortMode": row.get("sortMode", "EARLIEST_START"),
|
||
|
|
}
|
||
|
|
for row in versions
|
||
|
|
]
|
||
|
|
|
||
|
|
slots = bundle.rows("schedule-slots")
|
||
|
|
operations = _lookup(bundle.rows("operations"), "operationId")
|
||
|
|
work_orders = _lookup(bundle.rows("work-orders"), "workOrderId", "workOrderCode")
|
||
|
|
work_packages = _lookup(bundle.rows("work-packages"), "workPackageId", "workPackageCode")
|
||
|
|
orders = _lookup(production_orders, "productionOrderId", "productionOrderCode", "orderNo")
|
||
|
|
if slots and "flexWorkOrders" not in projections:
|
||
|
|
def _consumer_datetime(value: Any) -> Any:
|
||
|
|
if value in (None, ""):
|
||
|
|
return value
|
||
|
|
if isinstance(value, datetime):
|
||
|
|
parsed = value
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
parsed = datetime.fromisoformat(str(value))
|
||
|
|
except ValueError:
|
||
|
|
return value
|
||
|
|
return parsed.strftime("%Y-%m-%d %H:%M")
|
||
|
|
|
||
|
|
flex_work_orders: list[dict[str, Any]] = []
|
||
|
|
for slot in slots:
|
||
|
|
operation = operations.get(str(slot.get("operationId")), {})
|
||
|
|
work_order = work_orders.get(
|
||
|
|
str(operation.get("workOrderId", slot.get("workOrderId"))),
|
||
|
|
{},
|
||
|
|
)
|
||
|
|
order = orders.get(
|
||
|
|
str(work_order.get("productionOrderId", operation.get("productionOrderId"))),
|
||
|
|
{},
|
||
|
|
)
|
||
|
|
project = project_index.get(
|
||
|
|
str(order.get("projectId", work_order.get("projectId", operation.get("projectId")))),
|
||
|
|
{},
|
||
|
|
)
|
||
|
|
work_package = work_packages.get(
|
||
|
|
str(operation.get("workPackageId", work_order.get("workPackageId"))),
|
||
|
|
{},
|
||
|
|
)
|
||
|
|
zone = zone_index.get(str(work_package.get("zoneId")), {})
|
||
|
|
resource = resource_index.get(str(slot.get("resourceId")), {})
|
||
|
|
flex_order_no = order.get(
|
||
|
|
"productionOrderCode",
|
||
|
|
order.get("orderNo", order.get("productionOrderId")),
|
||
|
|
)
|
||
|
|
product_code = project.get(
|
||
|
|
"projectCode",
|
||
|
|
order.get("projectCode", order.get("materialId", flex_order_no)),
|
||
|
|
)
|
||
|
|
quantity = work_order.get(
|
||
|
|
"quantity",
|
||
|
|
order.get("quantity", order.get("qty", 1)),
|
||
|
|
)
|
||
|
|
sequence = operation.get(
|
||
|
|
"sequence",
|
||
|
|
operation.get("sequenceNo", operation.get("operationPosition", 0)),
|
||
|
|
)
|
||
|
|
duration_hours = float(
|
||
|
|
slot.get("durationHours", operation.get("durationHours", 0)) or 0
|
||
|
|
)
|
||
|
|
setup_hours = float(operation.get("setupHours", 0) or 0)
|
||
|
|
flex_work_orders.append(
|
||
|
|
{
|
||
|
|
"id": slot.get("scheduleSlotId"),
|
||
|
|
"versionId": slot.get("scheduleVersionId"),
|
||
|
|
"workOrderId": operation.get("workOrderId", slot.get("workOrderId")),
|
||
|
|
"flexOrderNo": flex_order_no,
|
||
|
|
"orderNo": work_order.get(
|
||
|
|
"workOrderCode",
|
||
|
|
work_order.get("orderNo", operation.get("workOrderId")),
|
||
|
|
),
|
||
|
|
"productCode": product_code,
|
||
|
|
"quantity": quantity,
|
||
|
|
"operationCode": operation.get("operationCode", operation.get("code")),
|
||
|
|
"operationName": operation.get(
|
||
|
|
"operationName",
|
||
|
|
operation.get("name", operation.get("operationCode", "UNKNOWN")),
|
||
|
|
),
|
||
|
|
"seq": sequence,
|
||
|
|
"equipmentId": slot.get("resourceId"),
|
||
|
|
"equipmentCode": resource.get(
|
||
|
|
"resourceCode",
|
||
|
|
resource.get("code", slot.get("resourceId")),
|
||
|
|
),
|
||
|
|
"equipmentName": resource.get(
|
||
|
|
"resourceName",
|
||
|
|
resource.get("name", slot.get("resourceId")),
|
||
|
|
),
|
||
|
|
"teamId": slot.get("teamId"),
|
||
|
|
"zone": zone.get(
|
||
|
|
"zoneCode",
|
||
|
|
zone.get("code", resource.get("transportZone", resource.get("zone"))),
|
||
|
|
),
|
||
|
|
"plannedStartTime": _consumer_datetime(slot.get("start")),
|
||
|
|
"plannedEndTime": _consumer_datetime(slot.get("end")),
|
||
|
|
"start": slot.get("start"),
|
||
|
|
"end": slot.get("end"),
|
||
|
|
"frozen": bool(slot.get("frozen", work_order.get("frozen", False))),
|
||
|
|
"qty": quantity,
|
||
|
|
"changeoverMin": round(setup_hours * 60, 2),
|
||
|
|
"moveMin": 0,
|
||
|
|
"runMin": round(duration_hours * 60, 2),
|
||
|
|
"isBottleneck": bool(slot.get("isBottleneck", False)),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
projections["flexWorkOrders"] = flex_work_orders
|
||
|
|
|
||
|
|
purchase = bundle.rows("purchase-suggestions")
|
||
|
|
if purchase:
|
||
|
|
projections.setdefault("purchaseSuggestions", deepcopy(purchase))
|
||
|
|
purchase_suggestions = projections.get("purchaseSuggestions")
|
||
|
|
if isinstance(purchase_suggestions, list) and (
|
||
|
|
not isinstance(projections.get("purchaseOrders"), list)
|
||
|
|
or (not projections.get("purchaseOrders") and purchase_suggestions)
|
||
|
|
):
|
||
|
|
projections["purchaseOrders"] = [
|
||
|
|
{
|
||
|
|
**deepcopy(row),
|
||
|
|
"documentType": "PURCHASE_SUGGESTION",
|
||
|
|
"recordType": "SUGGESTION",
|
||
|
|
"isActualOrder": False,
|
||
|
|
"projectionRole": "CONSUMER_COMPATIBILITY_ALIAS",
|
||
|
|
}
|
||
|
|
for row in purchase_suggestions
|
||
|
|
if isinstance(row, dict)
|
||
|
|
]
|
||
|
|
outsource = bundle.rows("outsource-suggestions")
|
||
|
|
if outsource:
|
||
|
|
projections.setdefault("outsourceSuggestions", deepcopy(outsource))
|
||
|
|
outsource_suggestions = projections.get("outsourceSuggestions")
|
||
|
|
if isinstance(outsource_suggestions, list) and (
|
||
|
|
not isinstance(projections.get("outsourceOrders"), list)
|
||
|
|
or (not projections.get("outsourceOrders") and outsource_suggestions)
|
||
|
|
):
|
||
|
|
projections["outsourceOrders"] = [
|
||
|
|
{
|
||
|
|
**deepcopy(row),
|
||
|
|
"documentType": "OUTSOURCE_SUGGESTION",
|
||
|
|
"recordType": "SUGGESTION",
|
||
|
|
"isActualOrder": False,
|
||
|
|
"projectionRole": "CONSUMER_COMPATIBILITY_ALIAS",
|
||
|
|
}
|
||
|
|
for row in outsource_suggestions
|
||
|
|
if isinstance(row, dict)
|
||
|
|
]
|
||
|
|
if production_orders:
|
||
|
|
projections.setdefault("makeSuggestions", deepcopy(production_orders))
|
||
|
|
return projections
|
||
|
|
|
||
|
|
|
||
|
|
def _build_world(bundle: DatasetBundle, config: GeneratorConfig, digest: str) -> dict[str, Any]:
|
||
|
|
metadata = deepcopy(bundle.metadata)
|
||
|
|
metadata["businessDigest"] = digest
|
||
|
|
metadata["generatedAt"] = _generated_at(config)
|
||
|
|
metadata["generatedAtBasis"] = _DATASET_GENERATED_AT_BASIS
|
||
|
|
world: dict[str, Any] = {
|
||
|
|
"datasetMetadata": metadata,
|
||
|
|
"shipyardTables": {name: deepcopy(bundle.rows(name)) for name in TABLE_ORDER},
|
||
|
|
"planningSourceHash": digest,
|
||
|
|
"inputVersion": metadata.get("schemaVersion"),
|
||
|
|
"scenarioId": config.scenario,
|
||
|
|
"algorithmEvidence": _artifact(bundle, "algorithmEvidence", "algorithm-evidence") or {},
|
||
|
|
"explanations": _artifact(bundle, "explanations", "expected-explanations") or [],
|
||
|
|
"shipyardKnowledgeAssets": _artifact(bundle, "knowledge-assets") or [],
|
||
|
|
"shipyardSkillRegistry": _artifact(bundle, "skill-registry") or {},
|
||
|
|
}
|
||
|
|
world.update(_build_flex_projections(bundle))
|
||
|
|
return world
|
||
|
|
|
||
|
|
|
||
|
|
def _scenario_artifact(bundle: DatasetBundle, name: str) -> Any:
|
||
|
|
candidates = (name, name.replace("-", "_"), name.replace("-", ""))
|
||
|
|
value = _artifact(bundle, *candidates)
|
||
|
|
if value is not None:
|
||
|
|
return value
|
||
|
|
scenarios = bundle.artifacts.get("scenarios")
|
||
|
|
if isinstance(scenarios, dict):
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate in scenarios:
|
||
|
|
return deepcopy(scenarios[candidate])
|
||
|
|
return _placeholder(name)
|
||
|
|
|
||
|
|
|
||
|
|
def _safe_event_id(value: Any, fallback: str) -> str:
|
||
|
|
candidate = re.sub(r"[^A-Za-z0-9_-]+", "-", str(value or fallback)).strip("-")
|
||
|
|
return candidate or fallback
|
||
|
|
|
||
|
|
|
||
|
|
def _write_artifacts(root: Path, bundle: DatasetBundle, row_counts: dict[str, int | None]) -> None:
|
||
|
|
knowledge_assets = _artifact(bundle, "knowledge-assets") or []
|
||
|
|
row_counts["rag/knowledge-assets.jsonl"] = _write_jsonl(
|
||
|
|
root,
|
||
|
|
"rag/knowledge-assets.jsonl",
|
||
|
|
knowledge_assets,
|
||
|
|
)
|
||
|
|
markdown = _artifact(bundle, "rag-markdown") or {}
|
||
|
|
for name in (
|
||
|
|
"shipbuilding-rules.md",
|
||
|
|
"scheduling-rules.md",
|
||
|
|
"quality-rules.md",
|
||
|
|
"historical-lessons.md",
|
||
|
|
):
|
||
|
|
_write_text(root, f"rag/{name}", str(markdown.get(name, f"# {name}\n\n{DATA_DISCLAIMER}\n")))
|
||
|
|
row_counts[f"rag/{name}"] = None
|
||
|
|
|
||
|
|
registry = _artifact(bundle, "skill-registry") or {"skills": []}
|
||
|
|
row_counts["skills/skill-registry.json"] = _write_json(
|
||
|
|
root,
|
||
|
|
"skills/skill-registry.json",
|
||
|
|
registry,
|
||
|
|
)
|
||
|
|
skill_schemas = _artifact(bundle, "skill-schemas") or {}
|
||
|
|
for skill_id in sorted(skill_schemas):
|
||
|
|
relative = f"skills/skill-schemas/{skill_id}.schema.json"
|
||
|
|
row_counts[relative] = _write_json(root, relative, skill_schemas[skill_id])
|
||
|
|
|
||
|
|
for name, document in build_schema_documents().items():
|
||
|
|
relative = f"schemas/{name}"
|
||
|
|
row_counts[relative] = _write_json(root, relative, document)
|
||
|
|
|
||
|
|
for scenario_name in _CANONICAL_SCENARIOS:
|
||
|
|
relative = f"scenarios/{scenario_name}.json"
|
||
|
|
row_counts[relative] = _write_json(
|
||
|
|
root,
|
||
|
|
relative,
|
||
|
|
_scenario_artifact(bundle, scenario_name),
|
||
|
|
)
|
||
|
|
|
||
|
|
events = _artifact(bundle, "scenario-events", "scenarioEvents", "events") or []
|
||
|
|
if isinstance(events, dict):
|
||
|
|
event_rows = [
|
||
|
|
({"eventId": key, **value} if isinstance(value, dict) else {"eventId": key, "input": value})
|
||
|
|
for key, value in sorted(events.items())
|
||
|
|
]
|
||
|
|
else:
|
||
|
|
event_rows = list(events) if isinstance(events, list) else []
|
||
|
|
for index, event in enumerate(event_rows, start=1):
|
||
|
|
if not isinstance(event, dict):
|
||
|
|
event = {"input": event}
|
||
|
|
event_id = _safe_event_id(event.get("eventId"), f"EVENT-{index:02d}")
|
||
|
|
for phase in ("input", "before", "after", "diff"):
|
||
|
|
relative = f"scenarios/events/{event_id}/{phase}.json"
|
||
|
|
row_counts[relative] = _write_json(
|
||
|
|
root,
|
||
|
|
relative,
|
||
|
|
deepcopy(event.get(phase, _placeholder(f"{event_id}-{phase}"))),
|
||
|
|
)
|
||
|
|
|
||
|
|
alternatives = _artifact(
|
||
|
|
bundle,
|
||
|
|
"schedule-alternatives",
|
||
|
|
"scheduleAlternatives",
|
||
|
|
"alternatives",
|
||
|
|
)
|
||
|
|
row_counts["planning/schedule-alternatives.json"] = _write_json(
|
||
|
|
root,
|
||
|
|
"planning/schedule-alternatives.json",
|
||
|
|
alternatives if alternatives is not None else _placeholder("schedule-alternatives"),
|
||
|
|
)
|
||
|
|
|
||
|
|
for name in _EXPECTED_ARTIFACTS:
|
||
|
|
value = _artifact(bundle, name, name.replace("-", "_"))
|
||
|
|
relative = f"expected/{name}.json"
|
||
|
|
row_counts[relative] = _write_json(
|
||
|
|
root,
|
||
|
|
relative,
|
||
|
|
value if value is not None else _placeholder(name),
|
||
|
|
)
|
||
|
|
|
||
|
|
validation_report = _artifact(bundle, "validation-report", "validation_report")
|
||
|
|
if not isinstance(validation_report, dict):
|
||
|
|
raise TypeError(
|
||
|
|
"validation-report artifact is required before export; "
|
||
|
|
"refusing to write synthetic validation placeholders"
|
||
|
|
)
|
||
|
|
|
||
|
|
referential_metric_names = {
|
||
|
|
"missingTableCount",
|
||
|
|
"duplicatePrimaryKeyCount",
|
||
|
|
"requiredFieldErrorCount",
|
||
|
|
"foreignKeyErrorCount",
|
||
|
|
"unresolvedEvidenceRefCount",
|
||
|
|
}
|
||
|
|
referential_summary = validation_report.get("referentialIntegrity")
|
||
|
|
if not isinstance(referential_summary, dict):
|
||
|
|
referential_summary = {
|
||
|
|
name: validation_report.get("metrics", {}).get(name, 0)
|
||
|
|
for name in sorted(referential_metric_names)
|
||
|
|
}
|
||
|
|
referential_summary["valid"] = not any(
|
||
|
|
int(value or 0) for key, value in referential_summary.items() if key != "valid"
|
||
|
|
)
|
||
|
|
violations = validation_report.get("violations")
|
||
|
|
violation_rows = [row for row in violations if isinstance(row, dict)] if isinstance(violations, list) else []
|
||
|
|
referential_violations = [
|
||
|
|
deepcopy(row)
|
||
|
|
for row in violation_rows
|
||
|
|
if str(row.get("metric")) in referential_metric_names
|
||
|
|
]
|
||
|
|
referential_payload = _artifact(bundle, "referential-integrity", "referential_integrity")
|
||
|
|
if not isinstance(referential_payload, dict):
|
||
|
|
referential_payload = {
|
||
|
|
"datasetType": validation_report.get("datasetType", "SYNTHETIC"),
|
||
|
|
"status": "PASS" if referential_summary.get("valid") else "FAIL",
|
||
|
|
"valid": bool(referential_summary.get("valid")),
|
||
|
|
"summary": deepcopy(referential_summary),
|
||
|
|
"checks": {
|
||
|
|
key: value
|
||
|
|
for key, value in sorted((validation_report.get("checks") or {}).items())
|
||
|
|
if key
|
||
|
|
in {
|
||
|
|
"evidenceRefCheckCount",
|
||
|
|
"foreignKeyCheckCount",
|
||
|
|
"rowSchemaCheckCount",
|
||
|
|
"tableSpecCount",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"metrics": {
|
||
|
|
key: value
|
||
|
|
for key, value in sorted((validation_report.get("metrics") or {}).items())
|
||
|
|
if key in referential_metric_names
|
||
|
|
},
|
||
|
|
"violations": referential_violations,
|
||
|
|
}
|
||
|
|
|
||
|
|
constraint_summary = validation_report.get("constraintValidation")
|
||
|
|
if not isinstance(constraint_summary, dict):
|
||
|
|
constraint_summary = {
|
||
|
|
"valid": False,
|
||
|
|
"hardViolationCount": len(
|
||
|
|
[row for row in violation_rows if row not in referential_violations]
|
||
|
|
),
|
||
|
|
}
|
||
|
|
constraint_payload = _artifact(bundle, "constraint-validation", "constraint_validation")
|
||
|
|
if not isinstance(constraint_payload, dict):
|
||
|
|
constraint_violations = [
|
||
|
|
deepcopy(row)
|
||
|
|
for row in violation_rows
|
||
|
|
if str(row.get("metric")) not in referential_metric_names
|
||
|
|
]
|
||
|
|
constraint_payload = {
|
||
|
|
"datasetType": validation_report.get("datasetType", "SYNTHETIC"),
|
||
|
|
"status": "PASS" if constraint_summary.get("valid") else "FAIL",
|
||
|
|
"valid": bool(constraint_summary.get("valid")),
|
||
|
|
"solveStatus": validation_report.get("solveStatus"),
|
||
|
|
"summary": deepcopy(constraint_summary),
|
||
|
|
"checks": deepcopy(validation_report.get("checks") or {}),
|
||
|
|
"metrics": {
|
||
|
|
key: value
|
||
|
|
for key, value in sorted((validation_report.get("metrics") or {}).items())
|
||
|
|
if key.endswith("ViolationCount") and key not in referential_metric_names
|
||
|
|
},
|
||
|
|
"violations": constraint_violations,
|
||
|
|
}
|
||
|
|
|
||
|
|
validation_payloads = {
|
||
|
|
"validation-report": validation_report,
|
||
|
|
"referential-integrity": referential_payload,
|
||
|
|
"constraint-validation": constraint_payload,
|
||
|
|
}
|
||
|
|
for name in _VALIDATION_JSON:
|
||
|
|
relative = f"validation/{name}.json"
|
||
|
|
row_counts[relative] = _write_json(root, relative, validation_payloads[name])
|
||
|
|
|
||
|
|
data_quality = _artifact(bundle, "data-quality-report", "data_quality_report")
|
||
|
|
if not isinstance(data_quality, str):
|
||
|
|
issue_counts = validation_report.get("issueCounts") or {}
|
||
|
|
checks = validation_report.get("checks") or {}
|
||
|
|
lines = [
|
||
|
|
"# Data Quality Report",
|
||
|
|
"",
|
||
|
|
f"> {DATA_DISCLAIMER}",
|
||
|
|
"",
|
||
|
|
f"- Status: {validation_report.get('status', 'FAIL')}",
|
||
|
|
f"- Valid: {str(bool(validation_report.get('valid'))).lower()}",
|
||
|
|
f"- Solve status: {validation_report.get('solveStatus', 'UNKNOWN')}",
|
||
|
|
f"- Blocking issue count: {len(validation_report.get('blockingIssues') or [])}",
|
||
|
|
f"- Violation count: {len(violation_rows)}",
|
||
|
|
"",
|
||
|
|
"## Validation checks",
|
||
|
|
"",
|
||
|
|
]
|
||
|
|
lines.extend(f"- {key}: {value}" for key, value in sorted(checks.items()))
|
||
|
|
lines.extend(["", "## Issue counts", ""])
|
||
|
|
if issue_counts:
|
||
|
|
lines.extend(f"- {key}: {value}" for key, value in sorted(issue_counts.items()))
|
||
|
|
else:
|
||
|
|
lines.append("- none: 0")
|
||
|
|
blocking_issues = validation_report.get("blockingIssues") or []
|
||
|
|
lines.extend(["", "## Blocking issues", ""])
|
||
|
|
if blocking_issues:
|
||
|
|
lines.extend(f"- {issue}" for issue in blocking_issues)
|
||
|
|
else:
|
||
|
|
lines.append("- none")
|
||
|
|
data_quality = "\n".join(lines) + "\n"
|
||
|
|
_write_text(root, "validation/data-quality-report.md", data_quality)
|
||
|
|
row_counts["validation/data-quality-report.md"] = None
|
||
|
|
|
||
|
|
execution_quality = _artifact(bundle, "execution-quality", "execution_quality")
|
||
|
|
rework_operations = bundle.rows("rework-operations")
|
||
|
|
rework_slots = bundle.rows("rework-schedule-slots")
|
||
|
|
if isinstance(execution_quality, dict):
|
||
|
|
if not rework_operations and isinstance(execution_quality.get("reworkOperations"), list):
|
||
|
|
rework_operations = execution_quality["reworkOperations"]
|
||
|
|
if not rework_slots and isinstance(execution_quality.get("reworkScheduleSlots"), list):
|
||
|
|
rework_slots = execution_quality["reworkScheduleSlots"]
|
||
|
|
if rework_operations:
|
||
|
|
row_counts["execution/rework-operations.json"] = _write_json(
|
||
|
|
root,
|
||
|
|
"execution/rework-operations.json",
|
||
|
|
rework_operations,
|
||
|
|
)
|
||
|
|
if rework_slots:
|
||
|
|
row_counts["execution/rework-schedule-slots.json"] = _write_json(
|
||
|
|
root,
|
||
|
|
"execution/rework-schedule-slots.json",
|
||
|
|
rework_slots,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _sha256(path: Path) -> str:
|
||
|
|
digest = hashlib.sha256()
|
||
|
|
with path.open("rb") as handle:
|
||
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||
|
|
digest.update(chunk)
|
||
|
|
return digest.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def _kind(path: str) -> str:
|
||
|
|
suffix = Path(path).suffix.lower()
|
||
|
|
return {
|
||
|
|
".csv": "CSV",
|
||
|
|
".json": "JSON",
|
||
|
|
".jsonl": "JSONL",
|
||
|
|
".md": "MARKDOWN",
|
||
|
|
}.get(suffix, "FILE")
|
||
|
|
|
||
|
|
|
||
|
|
def _previous_manifest(output: Path) -> dict[str, Any]:
|
||
|
|
path = output / "manifest.json"
|
||
|
|
if not path.is_file():
|
||
|
|
return {}
|
||
|
|
try:
|
||
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||
|
|
except (OSError, json.JSONDecodeError):
|
||
|
|
return {}
|
||
|
|
return value if isinstance(value, dict) else {}
|
||
|
|
|
||
|
|
|
||
|
|
def _manifest_files(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||
|
|
result: dict[str, dict[str, Any]] = {}
|
||
|
|
for item in manifest.get("files", []):
|
||
|
|
if isinstance(item, dict) and isinstance(item.get("path"), str):
|
||
|
|
result[item["path"]] = item
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _inventory(
|
||
|
|
root: Path,
|
||
|
|
row_counts: dict[str, int | None],
|
||
|
|
) -> list[dict[str, Any]]:
|
||
|
|
files: list[dict[str, Any]] = []
|
||
|
|
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||
|
|
relative = path.relative_to(root).as_posix()
|
||
|
|
if relative == "manifest.json":
|
||
|
|
continue
|
||
|
|
files.append(
|
||
|
|
{
|
||
|
|
"path": relative,
|
||
|
|
"kind": _kind(relative),
|
||
|
|
"rowCount": row_counts.get(relative),
|
||
|
|
"bytes": path.stat().st_size,
|
||
|
|
"sha256": _sha256(path),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
return files
|
||
|
|
|
||
|
|
|
||
|
|
def _reuse_unchanged(
|
||
|
|
output: Path,
|
||
|
|
staging: Path,
|
||
|
|
previous: dict[str, Any],
|
||
|
|
files: list[dict[str, Any]],
|
||
|
|
) -> list[str]:
|
||
|
|
previous_files = _manifest_files(previous)
|
||
|
|
skipped: list[str] = []
|
||
|
|
for item in files:
|
||
|
|
relative = str(item["path"])
|
||
|
|
old = previous_files.get(relative)
|
||
|
|
old_path = output / Path(relative)
|
||
|
|
if (
|
||
|
|
old
|
||
|
|
and old.get("sha256") == item.get("sha256")
|
||
|
|
and old_path.is_file()
|
||
|
|
and _sha256(old_path) == item.get("sha256")
|
||
|
|
):
|
||
|
|
shutil.copy2(old_path, staging / Path(relative))
|
||
|
|
skipped.append(relative)
|
||
|
|
return sorted(skipped)
|
||
|
|
|
||
|
|
|
||
|
|
def _is_child(path: Path, parent: Path) -> bool:
|
||
|
|
resolved = path.resolve(strict=False)
|
||
|
|
resolved_parent = parent.resolve(strict=False)
|
||
|
|
return resolved != resolved_parent and resolved_parent in resolved.parents
|
||
|
|
|
||
|
|
|
||
|
|
def _safe_rmtree(path: Path, parent: Path) -> None:
|
||
|
|
if path.exists():
|
||
|
|
if not _is_child(path, parent):
|
||
|
|
raise ValueError(f"refusing to remove path outside output parent: {path}")
|
||
|
|
shutil.rmtree(path)
|
||
|
|
|
||
|
|
|
||
|
|
def _atomic_publish(staging: Path, output: Path) -> None:
|
||
|
|
parent = output.parent.resolve(strict=False)
|
||
|
|
if not _is_child(staging, parent) or not _is_child(output, parent):
|
||
|
|
raise ValueError("output and staging directories must be children of the same parent")
|
||
|
|
if output.exists() and not output.is_dir():
|
||
|
|
raise ValueError(f"output path exists and is not a directory: {output}")
|
||
|
|
if not output.exists():
|
||
|
|
os.replace(staging, output)
|
||
|
|
return
|
||
|
|
backup = parent / f".{output.name}.backup-{uuid.uuid4().hex}"
|
||
|
|
os.replace(output, backup)
|
||
|
|
try:
|
||
|
|
os.replace(staging, output)
|
||
|
|
except BaseException:
|
||
|
|
if not output.exists() and backup.exists():
|
||
|
|
os.replace(backup, output)
|
||
|
|
raise
|
||
|
|
_safe_rmtree(backup, parent)
|
||
|
|
|
||
|
|
|
||
|
|
def export_bundle(
|
||
|
|
bundle: DatasetBundle,
|
||
|
|
config: GeneratorConfig,
|
||
|
|
output: Path,
|
||
|
|
*,
|
||
|
|
incremental: bool = False,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Export one deterministic dataset using an atomic sibling-directory replace."""
|
||
|
|
output = Path(output).expanduser().resolve(strict=False)
|
||
|
|
parent = output.parent
|
||
|
|
if output == parent or not output.name:
|
||
|
|
raise ValueError("output must name a dataset directory")
|
||
|
|
parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
previous = _previous_manifest(output) if incremental else {}
|
||
|
|
|
||
|
|
working = bundle.sorted_copy()
|
||
|
|
working = generate_rag_and_skills(working, config).sorted_copy()
|
||
|
|
digest = business_digest(working)
|
||
|
|
working.metadata["businessDigest"] = digest
|
||
|
|
working.metadata["generatedAt"] = _generated_at(config)
|
||
|
|
working.metadata["generatedAtBasis"] = _DATASET_GENERATED_AT_BASIS
|
||
|
|
|
||
|
|
staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.tmp-", dir=str(parent)))
|
||
|
|
row_counts: dict[str, int | None] = {}
|
||
|
|
try:
|
||
|
|
for table_name in TABLE_ORDER:
|
||
|
|
spec = TABLE_SPECS[table_name]
|
||
|
|
rows = sorted(working.rows(table_name), key=canonical_json)
|
||
|
|
row_counts[spec.path] = _write_csv(
|
||
|
|
staging,
|
||
|
|
spec.path,
|
||
|
|
table_name,
|
||
|
|
rows,
|
||
|
|
)
|
||
|
|
|
||
|
|
world = _build_world(working, config, digest)
|
||
|
|
row_counts["world.json"] = _write_json(staging, "world.json", world)
|
||
|
|
_write_text(staging, "README.md", _readme(working, config, digest))
|
||
|
|
row_counts["README.md"] = None
|
||
|
|
_write_text(staging, "data-dictionary.md", _data_dictionary())
|
||
|
|
row_counts["data-dictionary.md"] = None
|
||
|
|
_write_artifacts(staging, working, row_counts)
|
||
|
|
|
||
|
|
files = _inventory(staging, row_counts)
|
||
|
|
skipped = _reuse_unchanged(output, staging, previous, files) if previous else []
|
||
|
|
current_paths = {str(item["path"]) for item in files}
|
||
|
|
tombstones = sorted(set(_manifest_files(previous)) - current_paths)
|
||
|
|
table_counts = {name: len(working.rows(name)) for name in TABLE_ORDER}
|
||
|
|
manifest = {
|
||
|
|
**deepcopy(working.metadata),
|
||
|
|
"datasetType": "SYNTHETIC",
|
||
|
|
"organizationScenario": "BEIHAI_SHIPYARD_APS",
|
||
|
|
"generatedAt": _generated_at(config),
|
||
|
|
"businessDigest": digest,
|
||
|
|
"sourceHash": digest,
|
||
|
|
"encoding": "UTF-8",
|
||
|
|
"manifestSelfIncluded": False,
|
||
|
|
"fileCount": len(files),
|
||
|
|
"csvFileCount": len(TABLE_ORDER),
|
||
|
|
"tableRowCounts": table_counts,
|
||
|
|
"files": files,
|
||
|
|
"incremental": {
|
||
|
|
"enabled": bool(incremental),
|
||
|
|
"skippedFiles": skipped,
|
||
|
|
"tombstones": tombstones,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
_write_json(staging, "manifest.json", manifest)
|
||
|
|
_atomic_publish(staging, output)
|
||
|
|
return manifest
|
||
|
|
except BaseException:
|
||
|
|
_safe_rmtree(staging, parent)
|
||
|
|
raise
|