categorical histogram: a value named unique or longtail is overwritten by the summary bars #1084

Activity

1. paddymul

   added

   Histograms

   on Oct 6, 2026
2. paddymul commented on Oct 7, 2026

   

   paddymul

   on Oct 7, 2026

   CollaboratorAuthor

   More actions

   Probe script for this issue. Save it as probe_1084.py in a buckaroo checkout and run uv run python probe_1084.py. The output below is from main at e88ace9. Each case prints the histogram from pandas (pd), polars (pl) and xorq (xo): the bar kinds, the sum of the population bars, the sum of the categorical bars and the NA bar, then each bar as name:kind=value, and below that the histogram_bins range, min, max, nan_per and distinct_count.

   probe_1084.py

   """Histogram probe: runs columns through the pandas, polars and xorq stat pipelines and prints each histogram.
   
   Each line is [bar kinds, population sum, categorical sum, NA, duplicate names] then the bars as name:kind=value.
   """
   import logging
   import warnings
   from collections import Counter
   
   logging.disable(logging.CRITICAL)
   warnings.filterwarnings('ignore')
   
   import numpy as np
   import pandas as pd
   import polars as pl
   import pyarrow as pa
   import xorq.api as xo
   
   from buckaroo.customizations.pd_stats_v2 import PD_ANALYSIS_V2
   from buckaroo.customizations.pl_stats_v2 import PL_ANALYSIS_V2
   from buckaroo.customizations.xorq_stats_v2 import XORQ_STATS_V2
   from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
   from buckaroo.pluggable_analysis_framework.xorq_stat_pipeline import XorqStatPipeline
   
   rng = np.random.default_rng(0)
   
   
   def summarize(h):
       if not isinstance(h, list):
           return f'NOT-A-LIST {type(h).__name__}'
       if len(h) == 0:
           return 'EMPTY'
       kinds = Counter(k for b in h for k in b if k != 'name')
       parts = []
       for b in h:
           k = next((k for k in b if k != 'name'), '?')
           v = b.get(k)
           parts.append(f"{b['name']}:{k[:3]}={v:g}" if isinstance(v, (int, float, np.number)) else f"{b['name']}:{k}={v}")
       dups = [n for n, c in Counter(b['name'] for b in h).items() if c > 1]
       pop = sum(b.get('population', 0) for b in h)
       cat = sum(b.get(k, 0) for b in h for k in ('cat_pop', 'unique', 'longtail'))
       na = sum(b.get('NA', 0) for b in h)
       return f"[{dict(kinds)} pop={pop:g} cat={cat:g} NA={na:g}{' DUPNAMES=' + str(dups) if dups else ''}] " + ' | '.join(parts)
   
   
   def bins_s(b):
       if not b:
           return 'bins=[]'
       arr = np.asarray(b, dtype=float)
       return f'bins={len(b)} [{arr[0]:.6g}..{arr[-1]:.6g}]' + ('' if np.isfinite(arr).all() else ' NONFINITE')
   
   
   def run(backend, values, pd_dtype=None, pl_dtype=None, pa_type=None):
       if backend == 'pd':
           sd, errs = StatPipeline(PD_ANALYSIS_V2, unit_test=False).process_df(pd.DataFrame({'a': pd.Series(values, dtype=pd_dtype)}))
       elif backend == 'pl':
           sd, errs = StatPipeline(PL_ANALYSIS_V2, unit_test=False).process_df(pl.DataFrame({'a': pl.Series('a', values, dtype=pl_dtype, strict=False)}))
       else:
           table = xo.memtable(pa.table({'a': pa.array(values, type=pa_type)}), name='t')
           sd, errs = XorqStatPipeline(XORQ_STATS_V2, unit_test=False).process_table(table)
       return sd.get('a', {}), sorted({f'{e.stat_key}:{type(e.error).__name__}' for e in errs})
   
   
   def case(name, values, pd_dtype=None, pl_dtype=None, pa_type=None, backends=('pd', 'pl', 'xo')):
       print(f'\n### {name}')
       for b in backends:
           try:
               c, errs = run(b, values, pd_dtype, pl_dtype, pa_type)
           except Exception as e:
               print(f'  {b}: RAISED {type(e).__name__}: {str(e)[:150]}')
               continue
           print(f"  {b}: {summarize(c.get('histogram'))}")
           print(f"      {bins_s(c.get('histogram_bins'))} min={c.get('min')!r:.30} max={c.get('max')!r:.30} "
                 f"nan_per={c.get('nan_per')} distinct={c.get('distinct_count')}" + (f' ERRS={errs}' if errs else ''))
   
   case('100 unique, 60 longtail, 40 singletons', ['unique'] * 100 + ['longtail'] * 60 + [f'v{i}' for i in range(40)], pa_type=pa.string())
   case('categories named unique/longtail/NA beside nulls', ['unique'] * 50 + ['longtail'] * 30 + ['NA'] * 20 + [f'v{i}' for i in range(20)] + [None] * 5, pa_type=pa.string())
   case('a value named NA beside real nulls', ['NA'] * 30 + ['x'] * 60 + [None] * 10, pa_type=pa.string())
   case('1 and "1" in an object column', [1, '1', 1, '1', 2, '2'], backends=('pd',))

   Output at e88ace9

   ### 100 unique, 60 longtail, 40 singletons
     pd: [{'unique': 1, 'longtail': 1} pop=0 cat=80 NA=0] unique:uni=20 | longtail:lon=60
         bins=[] min=nan max=nan nan_per=0.0 distinct=42
     pl: [{'unique': 1, 'longtail': 1} pop=0 cat=80 NA=0] unique:uni=20 | longtail:lon=60
         bins=[] min=nan max=nan nan_per=0.0 distinct=42
     xo: [{'cat_pop': 10} pop=0 cat=100 NA=0] unique:cat=59.5 | longtail:cat=35.7 | v25:cat=0.6 | v32:cat=0.6 | v33:cat=0.6 | v30:cat=0.6 | v19:cat=0.6 | v27:cat=0.6 | v11:cat=0.6 | v31:cat=0.6
         bins=[] min=None max=None nan_per=0.0 distinct=41
   
   ### categories named unique/longtail/NA beside nulls
     pd: [{'cat_pop': 5, 'unique': 1, 'longtail': 1, 'NA': 1} pop=0 cat=66 NA=4 DUPNAMES=['NA']] NA:cat=16 | v10:cat=1 | v18:cat=1 | v17:cat=1 | v16:cat=1 | unique:uni=16 | longtail:lon=30 | NA:NA=4
         bins=[] min=nan max=nan nan_per=0.04 distinct=23
     pl: [{'cat_pop': 5, 'unique': 1, 'longtail': 1, 'NA': 1} pop=0 cat=66 NA=4 DUPNAMES=['NA']] NA:cat=16 | v0:cat=1 | v1:cat=1 | v2:cat=1 | v3:cat=1 | unique:uni=16 | longtail:lon=30 | NA:NA=4
         bins=[] min=nan max=nan nan_per=0.04 distinct=23
     xo: [{'cat_pop': 10} pop=0 cat=99.9 NA=0] unique:cat=45 | longtail:cat=27 | NA:cat=18 | None:cat=4.5 | v19:cat=0.9 | v12:cat=0.9 | v11:cat=0.9 | v3:cat=0.9 | v14:cat=0.9 | v13:cat=0.9
         bins=[] min=None max=None nan_per=0.04 distinct=23
   
   ### a value named NA beside real nulls
     pd: [{'cat_pop': 2, 'NA': 1} pop=0 cat=90 NA=10 DUPNAMES=['NA']] x:cat=60 | NA:cat=30 | NA:NA=10
         bins=[] min=nan max=nan nan_per=0.1 distinct=2
     pl: [{'cat_pop': 2, 'NA': 1} pop=0 cat=90 NA=10 DUPNAMES=['NA']] x:cat=60 | NA:cat=30 | NA:NA=10
         bins=[] min=nan max=nan nan_per=0.1 distinct=2
     xo: [{'cat_pop': 3} pop=0 cat=100 NA=0] x:cat=60 | NA:cat=30 | None:cat=10
         bins=[] min=None max=None nan_per=0.1 distinct=2
   
   ### 1 and "1" in an object column
     pd: [{'cat_pop': 4, 'unique': 1} pop=0 cat=133 NA=0 DUPNAMES=['1', '2']] 1:cat=33 | 1:cat=33 | 2:cat=17 | 2:cat=17 | unique:uni=33
         bins=[] min=nan max=nan nan_per=0.0 distinct=4