from __future__ import annotations import csv import hashlib import json import os import re import shutil import tempfile import uuid from collections.abc import Iterable from copy import deepcopy from datetime import date, datetime, time from pathlib import Path from typing import Any from zoneinfo import ZoneInfo from .config import DATA_DISCLAIMER, GeneratorConfig from .models import DatasetBundle, business_digest, canonical_json from .rag_skills import generate_rag_and_skills from .registry import TABLE_ORDER, TABLE_SPECS from .schemas import build_schema_documents _DATASET_GENERATED_AT_DATE = date(2026, 8, 4) _DATASET_GENERATED_AT_BASIS = "DETERMINISTIC_BUILD_DATE" _CANONICAL_SCENARIOS: tuple[str, ...] = ( "baseline", "material-delay", "crane-failure", "dock-delay", "design-change", "scenario-comparison", ) _EXPECTED_ARTIFACTS: tuple[str, ...] = ( "baseline-results", "baseline-kpis", "expected-conflicts", "expected-explanations", ) _VALIDATION_JSON: tuple[str, ...] = ( "validation-report", "referential-integrity", "constraint-validation", ) _FLEX_KEYS: tuple[str, ...] = ( "flexMaterials", "flexBom", "flexOrders", "flexRoutings", "flexEquipment", "flexScheduleVersions", "flexWorkOrders", "purchaseSuggestions", "purchaseOrders", "outsourceSuggestions", "outsourceOrders", "makeSuggestions", ) def _json_ready(value: Any) -> Any: if isinstance(value, dict): return {str(key): _json_ready(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))} if isinstance(value, (list, tuple)): return [_json_ready(item) for item in value] if isinstance(value, set): return [_json_ready(item) for item in sorted(value, key=str)] if isinstance(value, (date, datetime)): return value.isoformat() if isinstance(value, Path): return value.as_posix() return value def _json_bytes(value: Any) -> bytes: text = json.dumps( _json_ready(value), ensure_ascii=False, allow_nan=False, indent=2, sort_keys=True, ) return (text + "\n").encode("utf-8") def _write_bytes(root: Path, relative: str, payload: bytes) -> None: target = root / Path(relative) target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(payload) def _write_json(root: Path, relative: str, value: Any) -> int | None: _write_bytes(root, relative, _json_bytes(value)) if isinstance(value, list): return len(value) if isinstance(value, dict): return len(value) return None def _write_text(root: Path, relative: str, text: str) -> None: normalized = text.replace("\r\n", "\n").replace("\r", "\n") if not normalized.endswith("\n"): normalized += "\n" _write_bytes(root, relative, normalized.encode("utf-8")) def _csv_cell(value: Any) -> str: if value is None: return "" if isinstance(value, bool): return "true" if value else "false" if isinstance(value, (dict, list, tuple, set)): return canonical_json(_json_ready(value)) if isinstance(value, (date, datetime)): return value.isoformat() return str(value) def _write_csv(root: Path, relative: str, table_name: str, rows: list[dict[str, Any]]) -> int: spec = TABLE_SPECS[table_name] extra = sorted({str(key) for row in rows for key in row if key not in spec.required}) fieldnames = [*spec.required, *extra] target = root / Path(relative) target.parent.mkdir(parents=True, exist_ok=True) with target.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter( handle, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n", ) writer.writeheader() for row in rows: writer.writerow({field: _csv_cell(row.get(field)) for field in fieldnames}) return len(rows) def _write_jsonl(root: Path, relative: str, rows: Iterable[dict[str, Any]]) -> int: normalized = list(rows) payload = "".join(canonical_json(_json_ready(row)) + "\n" for row in normalized) _write_bytes(root, relative, payload.encode("utf-8")) return len(normalized) def _artifact(bundle: DatasetBundle, *keys: str) -> Any | None: for key in keys: if key in bundle.artifacts: return deepcopy(bundle.artifacts[key]) return None def _placeholder(name: str) -> dict[str, Any]: return { "artifact": name, "datasetType": "SYNTHETIC", "source": "bundle.artifacts", "status": "NOT_GENERATED", } def _generated_at(config: GeneratorConfig) -> str: zone = ZoneInfo(config.timezone) return datetime.combine(_DATASET_GENERATED_AT_DATE, time.min, tzinfo=zone).isoformat() def _readme(bundle: DatasetBundle, config: GeneratorConfig, digest: str) -> str: return f"""# Beihai Shipyard APS Synthetic Dataset > {DATA_DISCLAIMER} - datasetType: `SYNTHETIC` - organizationScenario: `BEIHAI_SHIPYARD_APS` - generatedFor: `APS development and validation` - timezone: `{config.timezone}` - planningBaseDate: `{config.planning_base_date.isoformat()}` - planningHorizonEnd: `{config.planning_horizon_end.isoformat()}` - randomSeed: `{config.seed}` - datasetProfile: `{config.scale.upper()}` - projectCount: `{config.profile.project_count}` - businessDigest: `{digest}` This directory is produced by a deterministic generator. `world.json` is the canonical projection and the 51 CSV files are table projections of the same version. `manifest.json` records byte size, row count, and SHA-256 for every payload file. All projects, people, suppliers, owners, and knowledge assets are synthetic and must not be represented as real ERP/MES/PLM/WMS/SRM/QMS data. ## Directories - `master/`, `projects/`, `engineering/`, `materials/`, `planning/`, `execution/` - `schemas/`: domain and table JSON Schema documents - `scenarios/`: baseline, disruptions, alternatives, and before/after diffs - `rag/`: SYNTHETIC_KNOWLEDGE assets - `skills/`: the exact 20 APS skills and input/output schemas - `expected/`: expected results and explanations - `validation/`: independent validator artifacts """ def _data_dictionary() -> str: lines = [ "# Beihai Shipyard APS Synthetic Data Dictionary", "", f"> {DATA_DISCLAIMER}", "", "All CSV files use UTF-8, ISO 8601 dates, and lowerCamelCase fields.", "", "| Table | Path | Primary key | Required fields | Foreign keys |", "|---|---|---|---|---|", ] for name in TABLE_ORDER: spec = TABLE_SPECS[name] foreign_keys = "; ".join( f"{fk.field}->{fk.target_table}.{fk.target_field}" + (" (nullable)" if fk.nullable else "") for fk in spec.foreign_keys ) or "-" lines.append( f"| `{name}` | `{spec.path}` | `{spec.primary_key}` | " f"`{', '.join(spec.required)}` | {foreign_keys} |" ) lines.extend( ( "", "## Frozen semantics", "", "- sourcingMode: MAKE / BUY / OUTSOURCE / OWNER_SUPPLIED", "- fulfillmentMode: STOCK / TRANSFER / PLANNED_RECEIPT / NEW_SUPPLY / DESIGN_PENDING", "- relationType: FS / SS / FF / SF", "- duration: hour; weight: tonne; length: meter; currency: CNY", "- priority: 1-10, where 10 is highest", ) ) return "\n".join(lines) + "\n" def _lookup(rows: list[dict[str, Any]], *fields: str) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} for row in rows: for field in fields: value = row.get(field) if value not in (None, ""): result[str(value)] = row return result def _projection_overrides(bundle: DatasetBundle) -> dict[str, Any]: projections: dict[str, Any] = {} for artifact_key in ("world-projection", "worldProjection", "flex-projections", "flexProjections"): value = bundle.artifacts.get(artifact_key) if isinstance(value, dict): for key in _FLEX_KEYS: if key in value: projections[key] = deepcopy(value[key]) value = bundle.artifacts.get("world") if isinstance(value, dict): for key in _FLEX_KEYS: if key in value: projections[key] = deepcopy(value[key]) return projections def _build_flex_projections(bundle: DatasetBundle) -> dict[str, Any]: projections = _projection_overrides(bundle) materials = bundle.rows("materials") projects = bundle.rows("ship-projects") project_index = _lookup(projects, "projectId", "projectCode") if "flexMaterials" not in projections: flex_materials: list[dict[str, Any]] = [] for row in materials: explicit_sourcing = ( row.get("sourcingMode") or row.get("procurementType") or row.get("sourcingType") ) flex_materials.append( { "id": row.get("materialId"), "code": row.get("materialCode", row.get("code", row.get("materialId"))), "name": row.get("name", row.get("materialName", row.get("materialId"))), "type": row.get("materialType", "RAW_MATERIAL"), "unit": row.get("unit", "PCS"), "category": row.get("materialGroup", row.get("category")), "sourcingMode": explicit_sourcing, "procurementType": row.get("procurementType") or explicit_sourcing, "sourcingType": explicit_sourcing, "sourcingTypeSource": "EXPLICIT", } ) flex_materials.extend( { "id": row.get("projectId"), "code": row.get("projectCode", row.get("projectId")), "name": row.get("name", row.get("projectCode", row.get("projectId"))), "type": "FINISHED_PRODUCT", "unit": "VESSEL", "category": row.get("shipType", "SHIP_PROJECT"), "sourcingMode": "MAKE", "procurementType": "MAKE", "sourcingType": "MAKE", "sourcingTypeSource": "EXPLICIT", } for row in projects ) projections["flexMaterials"] = flex_materials mbom = bundle.rows("mbom") if mbom and "flexBom" not in projections: projections["flexBom"] = [ { "id": row.get("mbomLineId", row.get("bomLineId")), "parentCode": row.get("parentMaterialId", row.get("parentCode")), "childCode": row.get("materialId", row.get("childMaterialId", row.get("childCode"))), "qty": row.get("quantity", row.get("qty", 0)), "unit": row.get("unit", "PCS"), } for row in mbom ] production_orders = bundle.rows("production-orders") material_index = _lookup(materials, "materialId", "materialCode", "code") if production_orders and "flexOrders" not in projections: flex_orders: list[dict[str, Any]] = [] for row in production_orders: material = material_index.get(str(row.get("materialId")), {}) project = project_index.get(str(row.get("projectId")), {}) order_no = row.get("productionOrderCode", row.get("orderNo", row.get("productionOrderId"))) flex_orders.append( { "id": row.get("productionOrderId"), "orderNo": order_no, "projectId": row.get("projectId"), "materialCode": material.get( "materialCode", material.get( "code", project.get("projectCode", row.get("projectCode", row.get("productionOrderId"))), ), ), "qty": row.get("quantity", row.get("qty", 1)), "dueDate": row.get("dueDate", row.get("needDate")), "status": row.get("status", "PLANNED"), "routingId": row.get("routingId"), } ) projections["flexOrders"] = flex_orders routing_operations = bundle.rows("routing-operations") operations_by_routing: dict[str, list[dict[str, Any]]] = {} for operation in routing_operations: routing_id = str(operation.get("routingId", "")) operations_by_routing.setdefault(routing_id, []).append( { "operationCode": operation.get("operationCode", operation.get("code")), "sequence": operation.get("sequence", operation.get("sequenceNo")), "durationHours": operation.get("durationHours", operation.get("runHours")), "resourceGroupId": operation.get("resourceGroupId"), } ) routings = bundle.rows("routings") if routings and "flexRoutings" not in projections: projections["flexRoutings"] = [ { "id": row.get("routingId"), "code": row.get("routingCode", row.get("code", row.get("routingId"))), "materialId": row.get("materialId"), "operations": sorted( operations_by_routing.get(str(row.get("routingId")), []), key=canonical_json, ), } for row in routings ] resources = bundle.rows("resources") resource_index = _lookup(resources, "resourceId", "resourceCode", "code") if resources and "flexEquipment" not in projections: projections["flexEquipment"] = [ { "id": row.get("resourceId"), "code": row.get("resourceCode", row.get("code", row.get("resourceId"))), "name": row.get("resourceName", row.get("name", row.get("resourceId"))), "zone": row.get("transportZone", row.get("zone")), "resourceType": row.get("resourceType"), "workcenterId": row.get("workcenterId"), "capacity": row.get("capacity", 1), "exclusive": row.get("exclusive", False), } for row in resources ] zones = bundle.rows("zones") zone_index = _lookup(zones, "zoneId", "zoneCode", "code") if zones and "flexZones" not in projections: projections["flexZones"] = [ { "id": row.get("zoneId"), "code": row.get("zoneCode", row.get("code", row.get("zoneId"))), "name": row.get("name", row.get("zoneId")), "type": row.get("zoneType"), } for row in zones ] versions = bundle.rows("schedule-versions") if versions and "flexScheduleVersions" not in projections: projections["flexScheduleVersions"] = [ { "id": row.get("scheduleVersionId"), "versionNo": row.get("versionNo"), "status": row.get("solveStatus", row.get("status")), "scenarioId": row.get("scenarioId"), "algorithm": row.get("algorithm"), "inputDigest": row.get("inputDigest"), "sortMode": row.get("sortMode", "EARLIEST_START"), } for row in versions ] slots = bundle.rows("schedule-slots") operations = _lookup(bundle.rows("operations"), "operationId") work_orders = _lookup(bundle.rows("work-orders"), "workOrderId", "workOrderCode") work_packages = _lookup(bundle.rows("work-packages"), "workPackageId", "workPackageCode") orders = _lookup(production_orders, "productionOrderId", "productionOrderCode", "orderNo") if slots and "flexWorkOrders" not in projections: def _consumer_datetime(value: Any) -> Any: if value in (None, ""): return value if isinstance(value, datetime): parsed = value else: try: parsed = datetime.fromisoformat(str(value)) except ValueError: return value return parsed.strftime("%Y-%m-%d %H:%M") flex_work_orders: list[dict[str, Any]] = [] for slot in slots: operation = operations.get(str(slot.get("operationId")), {}) work_order = work_orders.get( str(operation.get("workOrderId", slot.get("workOrderId"))), {}, ) order = orders.get( str(work_order.get("productionOrderId", operation.get("productionOrderId"))), {}, ) project = project_index.get( str(order.get("projectId", work_order.get("projectId", operation.get("projectId")))), {}, ) work_package = work_packages.get( str(operation.get("workPackageId", work_order.get("workPackageId"))), {}, ) zone = zone_index.get(str(work_package.get("zoneId")), {}) resource = resource_index.get(str(slot.get("resourceId")), {}) flex_order_no = order.get( "productionOrderCode", order.get("orderNo", order.get("productionOrderId")), ) product_code = project.get( "projectCode", order.get("projectCode", order.get("materialId", flex_order_no)), ) quantity = work_order.get( "quantity", order.get("quantity", order.get("qty", 1)), ) sequence = operation.get( "sequence", operation.get("sequenceNo", operation.get("operationPosition", 0)), ) duration_hours = float( slot.get("durationHours", operation.get("durationHours", 0)) or 0 ) setup_hours = float(operation.get("setupHours", 0) or 0) flex_work_orders.append( { "id": slot.get("scheduleSlotId"), "versionId": slot.get("scheduleVersionId"), "workOrderId": operation.get("workOrderId", slot.get("workOrderId")), "flexOrderNo": flex_order_no, "orderNo": work_order.get( "workOrderCode", work_order.get("orderNo", operation.get("workOrderId")), ), "productCode": product_code, "quantity": quantity, "operationCode": operation.get("operationCode", operation.get("code")), "operationName": operation.get( "operationName", operation.get("name", operation.get("operationCode", "UNKNOWN")), ), "seq": sequence, "equipmentId": slot.get("resourceId"), "equipmentCode": resource.get( "resourceCode", resource.get("code", slot.get("resourceId")), ), "equipmentName": resource.get( "resourceName", resource.get("name", slot.get("resourceId")), ), "teamId": slot.get("teamId"), "zone": zone.get( "zoneCode", zone.get("code", resource.get("transportZone", resource.get("zone"))), ), "plannedStartTime": _consumer_datetime(slot.get("start")), "plannedEndTime": _consumer_datetime(slot.get("end")), "start": slot.get("start"), "end": slot.get("end"), "frozen": bool(slot.get("frozen", work_order.get("frozen", False))), "qty": quantity, "changeoverMin": round(setup_hours * 60, 2), "moveMin": 0, "runMin": round(duration_hours * 60, 2), "isBottleneck": bool(slot.get("isBottleneck", False)), } ) projections["flexWorkOrders"] = flex_work_orders purchase = bundle.rows("purchase-suggestions") if purchase: projections.setdefault("purchaseSuggestions", deepcopy(purchase)) purchase_suggestions = projections.get("purchaseSuggestions") if isinstance(purchase_suggestions, list) and ( not isinstance(projections.get("purchaseOrders"), list) or (not projections.get("purchaseOrders") and purchase_suggestions) ): projections["purchaseOrders"] = [ { **deepcopy(row), "documentType": "PURCHASE_SUGGESTION", "recordType": "SUGGESTION", "isActualOrder": False, "projectionRole": "CONSUMER_COMPATIBILITY_ALIAS", } for row in purchase_suggestions if isinstance(row, dict) ] outsource = bundle.rows("outsource-suggestions") if outsource: projections.setdefault("outsourceSuggestions", deepcopy(outsource)) outsource_suggestions = projections.get("outsourceSuggestions") if isinstance(outsource_suggestions, list) and ( not isinstance(projections.get("outsourceOrders"), list) or (not projections.get("outsourceOrders") and outsource_suggestions) ): projections["outsourceOrders"] = [ { **deepcopy(row), "documentType": "OUTSOURCE_SUGGESTION", "recordType": "SUGGESTION", "isActualOrder": False, "projectionRole": "CONSUMER_COMPATIBILITY_ALIAS", } for row in outsource_suggestions if isinstance(row, dict) ] if production_orders: projections.setdefault("makeSuggestions", deepcopy(production_orders)) return projections def _build_world(bundle: DatasetBundle, config: GeneratorConfig, digest: str) -> dict[str, Any]: metadata = deepcopy(bundle.metadata) metadata["businessDigest"] = digest metadata["generatedAt"] = _generated_at(config) metadata["generatedAtBasis"] = _DATASET_GENERATED_AT_BASIS world: dict[str, Any] = { "datasetMetadata": metadata, "shipyardTables": {name: deepcopy(bundle.rows(name)) for name in TABLE_ORDER}, "planningSourceHash": digest, "inputVersion": metadata.get("schemaVersion"), "scenarioId": config.scenario, "algorithmEvidence": _artifact(bundle, "algorithmEvidence", "algorithm-evidence") or {}, "explanations": _artifact(bundle, "explanations", "expected-explanations") or [], "shipyardKnowledgeAssets": _artifact(bundle, "knowledge-assets") or [], "shipyardSkillRegistry": _artifact(bundle, "skill-registry") or {}, } world.update(_build_flex_projections(bundle)) return world def _scenario_artifact(bundle: DatasetBundle, name: str) -> Any: candidates = (name, name.replace("-", "_"), name.replace("-", "")) value = _artifact(bundle, *candidates) if value is not None: return value scenarios = bundle.artifacts.get("scenarios") if isinstance(scenarios, dict): for candidate in candidates: if candidate in scenarios: return deepcopy(scenarios[candidate]) return _placeholder(name) def _safe_event_id(value: Any, fallback: str) -> str: candidate = re.sub(r"[^A-Za-z0-9_-]+", "-", str(value or fallback)).strip("-") return candidate or fallback def _write_artifacts(root: Path, bundle: DatasetBundle, row_counts: dict[str, int | None]) -> None: knowledge_assets = _artifact(bundle, "knowledge-assets") or [] row_counts["rag/knowledge-assets.jsonl"] = _write_jsonl( root, "rag/knowledge-assets.jsonl", knowledge_assets, ) markdown = _artifact(bundle, "rag-markdown") or {} for name in ( "shipbuilding-rules.md", "scheduling-rules.md", "quality-rules.md", "historical-lessons.md", ): _write_text(root, f"rag/{name}", str(markdown.get(name, f"# {name}\n\n{DATA_DISCLAIMER}\n"))) row_counts[f"rag/{name}"] = None registry = _artifact(bundle, "skill-registry") or {"skills": []} row_counts["skills/skill-registry.json"] = _write_json( root, "skills/skill-registry.json", registry, ) skill_schemas = _artifact(bundle, "skill-schemas") or {} for skill_id in sorted(skill_schemas): relative = f"skills/skill-schemas/{skill_id}.schema.json" row_counts[relative] = _write_json(root, relative, skill_schemas[skill_id]) for name, document in build_schema_documents().items(): relative = f"schemas/{name}" row_counts[relative] = _write_json(root, relative, document) for scenario_name in _CANONICAL_SCENARIOS: relative = f"scenarios/{scenario_name}.json" row_counts[relative] = _write_json( root, relative, _scenario_artifact(bundle, scenario_name), ) events = _artifact(bundle, "scenario-events", "scenarioEvents", "events") or [] if isinstance(events, dict): event_rows = [ ({"eventId": key, **value} if isinstance(value, dict) else {"eventId": key, "input": value}) for key, value in sorted(events.items()) ] else: event_rows = list(events) if isinstance(events, list) else [] for index, event in enumerate(event_rows, start=1): if not isinstance(event, dict): event = {"input": event} event_id = _safe_event_id(event.get("eventId"), f"EVENT-{index:02d}") for phase in ("input", "before", "after", "diff"): relative = f"scenarios/events/{event_id}/{phase}.json" row_counts[relative] = _write_json( root, relative, deepcopy(event.get(phase, _placeholder(f"{event_id}-{phase}"))), ) alternatives = _artifact( bundle, "schedule-alternatives", "scheduleAlternatives", "alternatives", ) row_counts["planning/schedule-alternatives.json"] = _write_json( root, "planning/schedule-alternatives.json", alternatives if alternatives is not None else _placeholder("schedule-alternatives"), ) for name in _EXPECTED_ARTIFACTS: value = _artifact(bundle, name, name.replace("-", "_")) relative = f"expected/{name}.json" row_counts[relative] = _write_json( root, relative, value if value is not None else _placeholder(name), ) validation_report = _artifact(bundle, "validation-report", "validation_report") if not isinstance(validation_report, dict): raise TypeError( "validation-report artifact is required before export; " "refusing to write synthetic validation placeholders" ) referential_metric_names = { "missingTableCount", "duplicatePrimaryKeyCount", "requiredFieldErrorCount", "foreignKeyErrorCount", "unresolvedEvidenceRefCount", } referential_summary = validation_report.get("referentialIntegrity") if not isinstance(referential_summary, dict): referential_summary = { name: validation_report.get("metrics", {}).get(name, 0) for name in sorted(referential_metric_names) } referential_summary["valid"] = not any( int(value or 0) for key, value in referential_summary.items() if key != "valid" ) violations = validation_report.get("violations") violation_rows = [row for row in violations if isinstance(row, dict)] if isinstance(violations, list) else [] referential_violations = [ deepcopy(row) for row in violation_rows if str(row.get("metric")) in referential_metric_names ] referential_payload = _artifact(bundle, "referential-integrity", "referential_integrity") if not isinstance(referential_payload, dict): referential_payload = { "datasetType": validation_report.get("datasetType", "SYNTHETIC"), "status": "PASS" if referential_summary.get("valid") else "FAIL", "valid": bool(referential_summary.get("valid")), "summary": deepcopy(referential_summary), "checks": { key: value for key, value in sorted((validation_report.get("checks") or {}).items()) if key in { "evidenceRefCheckCount", "foreignKeyCheckCount", "rowSchemaCheckCount", "tableSpecCount", } }, "metrics": { key: value for key, value in sorted((validation_report.get("metrics") or {}).items()) if key in referential_metric_names }, "violations": referential_violations, } constraint_summary = validation_report.get("constraintValidation") if not isinstance(constraint_summary, dict): constraint_summary = { "valid": False, "hardViolationCount": len( [row for row in violation_rows if row not in referential_violations] ), } constraint_payload = _artifact(bundle, "constraint-validation", "constraint_validation") if not isinstance(constraint_payload, dict): constraint_violations = [ deepcopy(row) for row in violation_rows if str(row.get("metric")) not in referential_metric_names ] constraint_payload = { "datasetType": validation_report.get("datasetType", "SYNTHETIC"), "status": "PASS" if constraint_summary.get("valid") else "FAIL", "valid": bool(constraint_summary.get("valid")), "solveStatus": validation_report.get("solveStatus"), "summary": deepcopy(constraint_summary), "checks": deepcopy(validation_report.get("checks") or {}), "metrics": { key: value for key, value in sorted((validation_report.get("metrics") or {}).items()) if key.endswith("ViolationCount") and key not in referential_metric_names }, "violations": constraint_violations, } validation_payloads = { "validation-report": validation_report, "referential-integrity": referential_payload, "constraint-validation": constraint_payload, } for name in _VALIDATION_JSON: relative = f"validation/{name}.json" row_counts[relative] = _write_json(root, relative, validation_payloads[name]) data_quality = _artifact(bundle, "data-quality-report", "data_quality_report") if not isinstance(data_quality, str): issue_counts = validation_report.get("issueCounts") or {} checks = validation_report.get("checks") or {} lines = [ "# Data Quality Report", "", f"> {DATA_DISCLAIMER}", "", f"- Status: {validation_report.get('status', 'FAIL')}", f"- Valid: {str(bool(validation_report.get('valid'))).lower()}", f"- Solve status: {validation_report.get('solveStatus', 'UNKNOWN')}", f"- Blocking issue count: {len(validation_report.get('blockingIssues') or [])}", f"- Violation count: {len(violation_rows)}", "", "## Validation checks", "", ] lines.extend(f"- {key}: {value}" for key, value in sorted(checks.items())) lines.extend(["", "## Issue counts", ""]) if issue_counts: lines.extend(f"- {key}: {value}" for key, value in sorted(issue_counts.items())) else: lines.append("- none: 0") blocking_issues = validation_report.get("blockingIssues") or [] lines.extend(["", "## Blocking issues", ""]) if blocking_issues: lines.extend(f"- {issue}" for issue in blocking_issues) else: lines.append("- none") data_quality = "\n".join(lines) + "\n" _write_text(root, "validation/data-quality-report.md", data_quality) row_counts["validation/data-quality-report.md"] = None execution_quality = _artifact(bundle, "execution-quality", "execution_quality") rework_operations = bundle.rows("rework-operations") rework_slots = bundle.rows("rework-schedule-slots") if isinstance(execution_quality, dict): if not rework_operations and isinstance(execution_quality.get("reworkOperations"), list): rework_operations = execution_quality["reworkOperations"] if not rework_slots and isinstance(execution_quality.get("reworkScheduleSlots"), list): rework_slots = execution_quality["reworkScheduleSlots"] if rework_operations: row_counts["execution/rework-operations.json"] = _write_json( root, "execution/rework-operations.json", rework_operations, ) if rework_slots: row_counts["execution/rework-schedule-slots.json"] = _write_json( root, "execution/rework-schedule-slots.json", rework_slots, ) def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _kind(path: str) -> str: suffix = Path(path).suffix.lower() return { ".csv": "CSV", ".json": "JSON", ".jsonl": "JSONL", ".md": "MARKDOWN", }.get(suffix, "FILE") def _previous_manifest(output: Path) -> dict[str, Any]: path = output / "manifest.json" if not path.is_file(): return {} try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return {} return value if isinstance(value, dict) else {} def _manifest_files(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} for item in manifest.get("files", []): if isinstance(item, dict) and isinstance(item.get("path"), str): result[item["path"]] = item return result def _inventory( root: Path, row_counts: dict[str, int | None], ) -> list[dict[str, Any]]: files: list[dict[str, Any]] = [] for path in sorted(item for item in root.rglob("*") if item.is_file()): relative = path.relative_to(root).as_posix() if relative == "manifest.json": continue files.append( { "path": relative, "kind": _kind(relative), "rowCount": row_counts.get(relative), "bytes": path.stat().st_size, "sha256": _sha256(path), } ) return files def _reuse_unchanged( output: Path, staging: Path, previous: dict[str, Any], files: list[dict[str, Any]], ) -> list[str]: previous_files = _manifest_files(previous) skipped: list[str] = [] for item in files: relative = str(item["path"]) old = previous_files.get(relative) old_path = output / Path(relative) if ( old and old.get("sha256") == item.get("sha256") and old_path.is_file() and _sha256(old_path) == item.get("sha256") ): shutil.copy2(old_path, staging / Path(relative)) skipped.append(relative) return sorted(skipped) def _is_child(path: Path, parent: Path) -> bool: resolved = path.resolve(strict=False) resolved_parent = parent.resolve(strict=False) return resolved != resolved_parent and resolved_parent in resolved.parents def _safe_rmtree(path: Path, parent: Path) -> None: if path.exists(): if not _is_child(path, parent): raise ValueError(f"refusing to remove path outside output parent: {path}") shutil.rmtree(path) def _atomic_publish(staging: Path, output: Path) -> None: parent = output.parent.resolve(strict=False) if not _is_child(staging, parent) or not _is_child(output, parent): raise ValueError("output and staging directories must be children of the same parent") if output.exists() and not output.is_dir(): raise ValueError(f"output path exists and is not a directory: {output}") if not output.exists(): os.replace(staging, output) return backup = parent / f".{output.name}.backup-{uuid.uuid4().hex}" os.replace(output, backup) try: os.replace(staging, output) except BaseException: if not output.exists() and backup.exists(): os.replace(backup, output) raise _safe_rmtree(backup, parent) def export_bundle( bundle: DatasetBundle, config: GeneratorConfig, output: Path, *, incremental: bool = False, ) -> dict[str, Any]: """Export one deterministic dataset using an atomic sibling-directory replace.""" output = Path(output).expanduser().resolve(strict=False) parent = output.parent if output == parent or not output.name: raise ValueError("output must name a dataset directory") parent.mkdir(parents=True, exist_ok=True) previous = _previous_manifest(output) if incremental else {} working = bundle.sorted_copy() working = generate_rag_and_skills(working, config).sorted_copy() digest = business_digest(working) working.metadata["businessDigest"] = digest working.metadata["generatedAt"] = _generated_at(config) working.metadata["generatedAtBasis"] = _DATASET_GENERATED_AT_BASIS staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.tmp-", dir=str(parent))) row_counts: dict[str, int | None] = {} try: for table_name in TABLE_ORDER: spec = TABLE_SPECS[table_name] rows = sorted(working.rows(table_name), key=canonical_json) row_counts[spec.path] = _write_csv( staging, spec.path, table_name, rows, ) world = _build_world(working, config, digest) row_counts["world.json"] = _write_json(staging, "world.json", world) _write_text(staging, "README.md", _readme(working, config, digest)) row_counts["README.md"] = None _write_text(staging, "data-dictionary.md", _data_dictionary()) row_counts["data-dictionary.md"] = None _write_artifacts(staging, working, row_counts) files = _inventory(staging, row_counts) skipped = _reuse_unchanged(output, staging, previous, files) if previous else [] current_paths = {str(item["path"]) for item in files} tombstones = sorted(set(_manifest_files(previous)) - current_paths) table_counts = {name: len(working.rows(name)) for name in TABLE_ORDER} manifest = { **deepcopy(working.metadata), "datasetType": "SYNTHETIC", "organizationScenario": "BEIHAI_SHIPYARD_APS", "generatedAt": _generated_at(config), "businessDigest": digest, "sourceHash": digest, "encoding": "UTF-8", "manifestSelfIncluded": False, "fileCount": len(files), "csvFileCount": len(TABLE_ORDER), "tableRowCounts": table_counts, "files": files, "incremental": { "enabled": bool(incremental), "skippedFiles": skipped, "tombstones": tombstones, }, } _write_json(staging, "manifest.json", manifest) _atomic_publish(staging, output) return manifest except BaseException: _safe_rmtree(staging, parent) raise