66 lines
2.3 KiB
Python
66 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
|
|
def canonical_json(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def stable_id(namespace: str, *parts: Any, prefix: str | None = None, length: int = 16) -> str:
|
|
payload = canonical_json([namespace, *parts]).encode("utf-8")
|
|
digest = hashlib.sha256(payload).hexdigest()[:length].upper()
|
|
return f"{prefix or namespace.upper()}-{digest}"
|
|
|
|
|
|
def stable_hash(value: Any) -> str:
|
|
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class DatasetBundle:
|
|
metadata: dict[str, Any]
|
|
tables: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
|
|
artifacts: dict[str, Any] = field(default_factory=dict)
|
|
diagnostics: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
def ensure_table(self, name: str) -> list[dict[str, Any]]:
|
|
return self.tables.setdefault(name, [])
|
|
|
|
def add_rows(self, name: str, rows: Iterable[dict[str, Any]]) -> None:
|
|
self.ensure_table(name).extend(copy.deepcopy(list(rows)))
|
|
|
|
def set_rows(self, name: str, rows: Iterable[dict[str, Any]]) -> None:
|
|
self.tables[name] = copy.deepcopy(list(rows))
|
|
|
|
def rows(self, name: str) -> list[dict[str, Any]]:
|
|
return self.tables.get(name, [])
|
|
|
|
def clone(self) -> DatasetBundle:
|
|
return copy.deepcopy(self)
|
|
|
|
def sorted_copy(self) -> DatasetBundle:
|
|
clone = self.clone()
|
|
for name, rows in clone.tables.items():
|
|
clone.tables[name] = sorted(rows, key=canonical_json)
|
|
clone.artifacts = {key: clone.artifacts[key] for key in sorted(clone.artifacts)}
|
|
return clone
|
|
|
|
def counts(self) -> dict[str, int]:
|
|
return {name: len(rows) for name, rows in sorted(self.tables.items())}
|
|
|
|
|
|
def business_digest(bundle: DatasetBundle) -> str:
|
|
stable = bundle.sorted_copy()
|
|
metadata = {
|
|
key: value
|
|
for key, value in stable.metadata.items()
|
|
if key not in {"generatedAt", "solveTimeMs", "elapsedSeconds", "peakWorkingSetMiB"}
|
|
}
|
|
return stable_hash({"metadata": metadata, "tables": stable.tables, "artifacts": stable.artifacts})
|