polars stats: NaN in a float column reaches the histogram percentiles (nan tail label, no NA bar) #1080

Activity

1. paddymul

   added

   Histograms

   on Oct 6, 2026
2. paddymul commented on Oct 7, 2026

   

   paddymul

   on Oct 7, 2026

   CollaboratorAuthor

   More actions

   Probe script for this issue. Save it as probe_1080.py in a buckaroo checkout and run uv run python probe_1080.py. The output below is from main at e88ace9. Each case prints the histogram from pandas (pd), polars (pl) and xorq (xo): the bar kinds, the sum of the population bars, the sum of the categorical bars and the NA bar, then each bar as name:kind=value, and below that the histogram_bins range, min, max, nan_per and distinct_count.

   probe_1080.py

   """Histogram probe: runs columns through the pandas, polars and xorq stat pipelines and prints each histogram.
   
   Each line is [bar kinds, population sum, categorical sum, NA, duplicate names] then the bars as name:kind=value.
   """
   import logging
   import warnings
   from collections import Counter
   
   logging.disable(logging.CRITICAL)
   warnings.filterwarnings('ignore')
   
   import numpy as np
   import pandas as pd
   import polars as pl
   import pyarrow as pa
   import xorq.api as xo
   
   from buckaroo.customizations.pd_stats_v2 import PD_ANALYSIS_V2
   from buckaroo.customizations.pl_stats_v2 import PL_ANALYSIS_V2
   from buckaroo.customizations.xorq_stats_v2 import XORQ_STATS_V2
   from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
   from buckaroo.pluggable_analysis_framework.xorq_stat_pipeline import XorqStatPipeline
   
   rng = np.random.default_rng(0)
   
   
   def summarize(h):
       if not isinstance(h, list):
           return f'NOT-A-LIST {type(h).__name__}'
       if len(h) == 0:
           return 'EMPTY'
       kinds = Counter(k for b in h for k in b if k != 'name')
       parts = []
       for b in h:
           k = next((k for k in b if k != 'name'), '?')
           v = b.get(k)
           parts.append(f"{b['name']}:{k[:3]}={v:g}" if isinstance(v, (int, float, np.number)) else f"{b['name']}:{k}={v}")
       dups = [n for n, c in Counter(b['name'] for b in h).items() if c > 1]
       pop = sum(b.get('population', 0) for b in h)
       cat = sum(b.get(k, 0) for b in h for k in ('cat_pop', 'unique', 'longtail'))
       na = sum(b.get('NA', 0) for b in h)
       return f"[{dict(kinds)} pop={pop:g} cat={cat:g} NA={na:g}{' DUPNAMES=' + str(dups) if dups else ''}] " + ' | '.join(parts)
   
   
   def bins_s(b):
       if not b:
           return 'bins=[]'
       arr = np.asarray(b, dtype=float)
       return f'bins={len(b)} [{arr[0]:.6g}..{arr[-1]:.6g}]' + ('' if np.isfinite(arr).all() else ' NONFINITE')
   
   
   def run(backend, values, pd_dtype=None, pl_dtype=None, pa_type=None):
       if backend == 'pd':
           sd, errs = StatPipeline(PD_ANALYSIS_V2, unit_test=False).process_df(pd.DataFrame({'a': pd.Series(values, dtype=pd_dtype)}))
       elif backend == 'pl':
           sd, errs = StatPipeline(PL_ANALYSIS_V2, unit_test=False).process_df(pl.DataFrame({'a': pl.Series('a', values, dtype=pl_dtype, strict=False)}))
       else:
           table = xo.memtable(pa.table({'a': pa.array(values, type=pa_type)}), name='t')
           sd, errs = XorqStatPipeline(XORQ_STATS_V2, unit_test=False).process_table(table)
       return sd.get('a', {}), sorted({f'{e.stat_key}:{type(e.error).__name__}' for e in errs})
   
   
   def case(name, values, pd_dtype=None, pl_dtype=None, pa_type=None, backends=('pd', 'pl', 'xo')):
       print(f'\n### {name}')
       for b in backends:
           try:
               c, errs = run(b, values, pd_dtype, pl_dtype, pa_type)
           except Exception as e:
               print(f'  {b}: RAISED {type(e).__name__}: {str(e)[:150]}')
               continue
           print(f"  {b}: {summarize(c.get('histogram'))}")
           print(f"      {bins_s(c.get('histogram_bins'))} min={c.get('min')!r:.30} max={c.get('max')!r:.30} "
                 f"nan_per={c.get('nan_per')} distinct={c.get('distinct_count')}" + (f' ERRS={errs}' if errs else ''))
   
   # polars keeps NaN distinct from null; pandas treats NaN as null
   case('5% NaN', [float(i) for i in range(95)] + [float('nan')] * 5, backends=('pd', 'pl'))
   case('90% NaN', [float('nan')] * 900 + [float(i) for i in range(100)], backends=('pd', 'pl'))
   case('50% NaN + 6 distinct', [float('nan')] * 600 + [float(i) for i in range(6)] * 100, backends=('pd', 'pl'))
   case('NaN beside an infinity (since #1069 the infinity is filtered, the NaN is not)', [float(i) for i in range(95)] + [float('nan')] * 3 + [float('inf')] * 2, backends=('pd', 'pl'))

   Output at e88ace9

   ### 5% NaN
     pd: [{'tail': 2, 'population': 10, 'NA': 1} pop=103 cat=0 NA=5] 0–0.9:tai=1 | 1–10.2:pop=11 | 10.2–19.4:pop=10 | 19.4–28.6:pop=10 | 28.6–37.8:pop=10 | 37.8–47:pop=10 | 47–56.2:pop=11 | 56.2–65.4:pop=10 | 65.4–74.6:pop=10 | 74.6–83.8:pop=10 | 83.8–93:pop=11 | 93.1–94:tai=1 | NA:NA=5
         bins=11 [1..93] min=np.float64(0.0) max=np.float64(94.0) nan_per=0.05 distinct=95
     pl: [{'tail': 2, 'population': 10} pop=103 cat=0 NA=0] 0–1:tai=1 | 2–11.2:pop=11 | 11.2–20.4:pop=10 | 20.4–29.6:pop=10 | 29.6–38.8:pop=10 | 38.8–48:pop=10 | 48–57.2:pop=11 | 57.2–66.4:pop=10 | 66.4–75.6:pop=10 | 75.6–84.8:pop=10 | 84.8–94:pop=11 | nan–94:tai=1
         bins=11 [2..94] min=0.0 max=94.0 nan_per=0.0 distinct=96
   
   ### 90% NaN
     pd: [{'tail': 2, 'population': 10, 'NA': 1} pop=98 cat=0 NA=90] 0–1:tai=1 | 1–10.7:pop=10 | 10.7–20.4:pop=10 | 20.4–30.1:pop=10 | 30.1–39.8:pop=9 | 39.8–49.5:pop=10 | 49.5–59.2:pop=10 | 59.2–68.9:pop=9 | 68.9–78.6:pop=10 | 78.6–88.3:pop=10 | 88.3–98:pop=10 | 98–99:tai=1 | NA:NA=90
         bins=11 [1..98] min=np.float64(0.0) max=np.float64(99.0) nan_per=0.9 distinct=100
     pl: [{'tail': 2, 'population': 10} pop=99 cat=0 NA=0] 0–10:tai=1 | 11–19.8:pop=10 | 19.8–28.6:pop=10 | 28.6–37.4:pop=10 | 37.4–46.2:pop=10 | 46.2–55:pop=9 | 55–63.8:pop=10 | 63.8–72.6:pop=10 | 72.6–81.4:pop=10 | 81.4–90.2:pop=10 | 90.2–99:pop=10 | nan–99:tai=1
         bins=11 [11..99] min=0.0 max=99.0 nan_per=0.0 distinct=101
   
   ### 50% NaN + 6 distinct
     pd: [{'tail': 2, 'population': 10, 'NA': 1} pop=100 cat=0 NA=50] 0–0:tai=1 | 1–1.3:pop=25 | 1.3–1.6:pop=0 | 1.6–1.9:pop=0 | 1.9–2.2:pop=25 | 2.2–2.5:pop=0 | 2.5–2.8:pop=0 | 2.8–3.1:pop=25 | 3.1–3.4:pop=0 | 3.4–3.7:pop=0 | 3.7–4:pop=25 | 5–5:tai=1 | NA:NA=50
         bins=11 [1..4] min=np.float64(0.0) max=np.float64(5.0) nan_per=0.5 distinct=6
     pl: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] 0–0:tai=1 | 1–1.4:pop=20 | 1.4–1.8:pop=0 | 1.8–2.2:pop=20 | 2.2–2.6:pop=0 | 2.6–3:pop=0 | 3–3.4:pop=20 | 3.4–3.8:pop=0 | 3.8–4.2:pop=20 | 4.2–4.6:pop=0 | 4.6–5:pop=20 | nan–5:tai=1
         bins=11 [1..5] min=0.0 max=5.0 nan_per=0.0 distinct=7
   
   ### NaN beside an infinity (since #1069 the infinity is filtered, the NaN is not)
     pd: [{'tail': 2, 'population': 10, 'NA': 1} pop=103 cat=0 NA=3] 0–0.9:tai=1 | 1–10.2:pop=11 | 10.2–19.4:pop=10 | 19.4–28.6:pop=10 | 28.6–37.8:pop=10 | 37.8–47:pop=10 | 47–56.2:pop=11 | 56.2–65.4:pop=10 | 65.4–74.6:pop=10 | 74.6–83.8:pop=10 | 83.8–93:pop=11 | 93.1–inf:tai=1 | NA:NA=3
         bins=11 [1..93] min=np.float64(0.0) max=np.float64(inf) nan_per=0.03 distinct=96
     pl: [{'tail': 2, 'population': 10} pop=103 cat=0 NA=0] 0–1:tai=1 | 2–11.2:pop=11 | 11.2–20.4:pop=10 | 20.4–29.6:pop=10 | 29.6–38.8:pop=10 | 38.8–48:pop=10 | 48–57.2:pop=11 | 57.2–66.4:pop=10 | 66.4–75.6:pop=10 | 75.6–84.8:pop=10 | 84.8–94:pop=11 | nan–inf:tai=1
         bins=11 [2..94] min=0.0 max=inf nan_per=0.0 distinct=97