WIP:DataScan refactor to expose data and statistics consistently

.github/workflows/ci-tests.yaml

@@ -29,7 +29,7 @@ jobs:
           pip install -e '.[dev]'
       - name: Install test dependencies
         run: |
-          pip install pytest pytest-cov pytest-snapshot pandas polars ibis-framework[duckdb,mysql,postgres,sqlite]>=9.5.0 chatlas shiny
+          pip install pytest pytest-randomly pytest-cov pytest-snapshot pandas polars ibis-framework[duckdb,mysql,postgres,sqlite]>=9.5.0 chatlas shiny hypothesis
       - name: pytest unit tests
         run: |
           make test

.gitignore

@@ -124,3 +124,4 @@ datasets/
 /*.parquet
 /*.csv
 .ruff_cache
+.swp

Makefile

@@ -1,7 +1,9 @@
 .PHONY: check
 
 test:
-	pytest --cov=pointblank --cov-report=xml
+	pytest --cov=pointblank --cov-report=xml \
+		--randomly-seed=12301998
+
 
 test-update:
 	pytest --snapshot-update

pointblank/_datascan_utils.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from math import floor, log10
+from typing import TYPE_CHECKING
+
+from great_tables.vals import fmt_integer, fmt_number, fmt_scientific
+
+if TYPE_CHECKING:
+    pass
+
+
+def _round_to_sig_figs(value: float, sig_figs: int) -> float:
+    if value == 0:
+        return 0
+    return round(value, sig_figs - int(floor(log10(abs(value)))) - 1)
+
+
+def _compact_integer_fmt(value: float | int) -> str:
+    if value == 0:
+        formatted = "0"
+    elif abs(value) >= 1 and abs(value) < 10_000:
+        formatted = fmt_integer(value, use_seps=False)[0]
+    else:
+        formatted = fmt_scientific(value, decimals=1, exp_style="E1")[0]
+
+    return formatted
+
+
+def _compact_decimal_fmt(value: float | int) -> str:
+    if value == 0:
+        formatted = "0.00"
+    elif abs(value) < 1 and abs(value) >= 0.01:
+        formatted = fmt_number(value, decimals=2)[0]
+    elif abs(value) < 0.01:
+        formatted = fmt_scientific(value, decimals=1, exp_style="E1")[0]
+    elif abs(value) >= 1 and abs(value) < 1000:
+        formatted = fmt_number(value, n_sigfig=3)[0]
+    elif abs(value) >= 1000 and abs(value) < 10_000:
+        formatted = fmt_number(value, decimals=0, use_seps=False)[0]
+    else:
+        formatted = fmt_scientific(value, decimals=1, exp_style="E1")[0]
+
+    return formatted
+
+
+def _compact_0_1_fmt(value: float | int | None) -> str | None:
+    if value is None:
+        return value
+
+    if value == 0:
+        return " 0.00"
+
+    if value == 1:
+        return " 1.00"
+
+    if abs(value) < 1 and abs(value) >= 0.01:
+        return " " + fmt_number(value, decimals=2)[0]
+
+    if abs(value) < 0.01:
+        return "<0.01"
+
+    if abs(value) > 0.99:
+        return ">0.99"
+
+    return fmt_number(value, n_sigfig=3)[0]

pointblank/_utils.py

@@ -2,6 +2,7 @@
 
 import inspect
 import re
+from collections import defaultdict
 from typing import TYPE_CHECKING, Any
 
 import narwhals as nw
@@ -12,9 +13,28 @@
 from pointblank._constants import ASSERTION_TYPE_METHOD_MAP, GENERAL_COLUMN_TYPES
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from pointblank._typing import AbsoluteBounds, Tolerance
 
 
+def transpose_dicts(list_of_dicts: list[dict[str, Any]]) -> dict[str, list[Any]]:
+    if not list_of_dicts:
+        return {}
+
+    # Get all unique keys across all dictionaries
+    all_keys = set()
+    for d in list_of_dicts:
+        all_keys.update(d.keys())
+
+    result = defaultdict(list)
+    for d in list_of_dicts:
+        for key in all_keys:
+            result[key].append(d.get(key))  # None is default for missing keys
+
+    return dict(result)
+
+
 def _derive_single_bound(ref: int, tol: int | float) -> int:
     """Derive a single bound using the reference."""
     if not isinstance(tol, float | int):
@@ -750,3 +770,14 @@ def _format_to_float_value(
     formatted_vals = _get_column_of_values(gt, column_name="x", context="html")
 
     return formatted_vals[0]
+
+
+def _pivot_to_dict(col_dict: Mapping[str, Any]):  # TODO : Type hint and unit test
+    result_dict = {}
+    for col, sub_dict in col_dict.items():
+        for key, value in sub_dict.items():
+            # add columns fields not present
+            if key not in result_dict:
+                result_dict[key] = [None] * len(col_dict)
+            result_dict[key][list(col_dict.keys()).index(col)] = value
+    return result_dict

pointblank/_utils_html.py

@@ -1,9 +1,49 @@
 from __future__ import annotations
 
+from typing import Any
+
+from great_tables import html
+
 from pointblank._constants import TABLE_TYPE_STYLES
 from pointblank._utils import _format_to_integer_value
 
 
+def _fmt_frac(vec) -> list[str | None]:
+    res: list[str | None] = []
+    for x in vec:
+        if x is None:
+            res.append(x)
+            continue
+
+        if x == 0:
+            res.append("0")
+            continue
+
+        if x < 0.01:
+            res.append("<.01")
+            continue
+
+        try:
+            intx: int = int(x)
+        except ValueError:  # generic object, ie. NaN
+            res.append(str(x))
+            continue
+
+        if intx == x:  # can remove trailing 0s w/o loss
+            res.append(str(intx))
+            continue
+
+        res.append(str(round(x, 2)))
+
+    return res
+
+
+def _make_sublabel(major: str, minor: str) -> Any:
+    return html(
+        f'{major!s}<span style="font-size: 0.75em; vertical-align: sub; position: relative; line-height: 0.5em;">{minor!s}</span>'
+    )
+
+
 def _create_table_type_html(
     tbl_type: str | None, tbl_name: str | None, font_size: str = "10px"
 ) -> str:

pointblank/assistant.py

@@ -176,9 +176,7 @@ def assistant(
     if data is not None:
         scan = DataScan(data=data)
 
-        scan_dict = scan.to_dict()
-
-        tbl_type = scan_dict["tbl_type"]
+        tbl_type: str = scan.profile.implementation.name.lower()
         tbl_json = scan.to_json()
 
         if tbl_name is not None:

pointblank/compare.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pointblank import DataScan
+
+if TYPE_CHECKING:
+    from narwhals.typing import IntoFrame
+
+
+class Compare:
+    def __init__(self, a: IntoFrame, b: IntoFrame) -> None:
+        self.a: IntoFrame = a
+        self.b: IntoFrame = b
+
+    def compare(self) -> None:
+        ## Scan both frames
+        self._scana = DataScan(self.a)
+        self._scanb = DataScan(self.b)
+
+        ## Get summary outs
+        summarya = self._scana.summary_data
+        summaryb = self._scana.summary_data
+
+        summarya.columns
+
+        self._scana.profile