histogram labels: a numpy int column holding its dtype's minimum labels the low tail from 0 #1082

Activity

1. paddymul

   added

   Histograms

   on Oct 6, 2026
2. paddymul commented on Oct 7, 2026

   

   paddymul

   on Oct 7, 2026

   CollaboratorAuthor

   More actions

   Probe script for this issue. Save it as probe_1082.py in a buckaroo checkout and run uv run python probe_1082.py. The output below is from main at e88ace9. Each case prints the histogram from pandas (pd), polars (pl) and xorq (xo): the bar kinds, the sum of the population bars, the sum of the categorical bars and the NA bar, then each bar as name:kind=value, and below that the histogram_bins range, min, max, nan_per and distinct_count.

   probe_1082.py

   """Histogram probe: runs columns through the pandas, polars and xorq stat pipelines and prints each histogram.
   
   Each line is [bar kinds, population sum, categorical sum, NA, duplicate names] then the bars as name:kind=value.
   """
   import logging
   import warnings
   from collections import Counter
   
   logging.disable(logging.CRITICAL)
   warnings.filterwarnings('ignore')
   
   import numpy as np
   import pandas as pd
   import polars as pl
   import pyarrow as pa
   import xorq.api as xo
   
   from buckaroo.customizations.pd_stats_v2 import PD_ANALYSIS_V2
   from buckaroo.customizations.pl_stats_v2 import PL_ANALYSIS_V2
   from buckaroo.customizations.xorq_stats_v2 import XORQ_STATS_V2
   from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
   from buckaroo.pluggable_analysis_framework.xorq_stat_pipeline import XorqStatPipeline
   
   rng = np.random.default_rng(0)
   
   
   def summarize(h):
       if not isinstance(h, list):
           return f'NOT-A-LIST {type(h).__name__}'
       if len(h) == 0:
           return 'EMPTY'
       kinds = Counter(k for b in h for k in b if k != 'name')
       parts = []
       for b in h:
           k = next((k for k in b if k != 'name'), '?')
           v = b.get(k)
           parts.append(f"{b['name']}:{k[:3]}={v:g}" if isinstance(v, (int, float, np.number)) else f"{b['name']}:{k}={v}")
       dups = [n for n, c in Counter(b['name'] for b in h).items() if c > 1]
       pop = sum(b.get('population', 0) for b in h)
       cat = sum(b.get(k, 0) for b in h for k in ('cat_pop', 'unique', 'longtail'))
       na = sum(b.get('NA', 0) for b in h)
       return f"[{dict(kinds)} pop={pop:g} cat={cat:g} NA={na:g}{' DUPNAMES=' + str(dups) if dups else ''}] " + ' | '.join(parts)
   
   
   def bins_s(b):
       if not b:
           return 'bins=[]'
       arr = np.asarray(b, dtype=float)
       return f'bins={len(b)} [{arr[0]:.6g}..{arr[-1]:.6g}]' + ('' if np.isfinite(arr).all() else ' NONFINITE')
   
   
   def run(backend, values, pd_dtype=None, pl_dtype=None, pa_type=None):
       if backend == 'pd':
           sd, errs = StatPipeline(PD_ANALYSIS_V2, unit_test=False).process_df(pd.DataFrame({'a': pd.Series(values, dtype=pd_dtype)}))
       elif backend == 'pl':
           sd, errs = StatPipeline(PL_ANALYSIS_V2, unit_test=False).process_df(pl.DataFrame({'a': pl.Series('a', values, dtype=pl_dtype, strict=False)}))
       else:
           table = xo.memtable(pa.table({'a': pa.array(values, type=pa_type)}), name='t')
           sd, errs = XorqStatPipeline(XORQ_STATS_V2, unit_test=False).process_table(table)
       return sd.get('a', {}), sorted({f'{e.stat_key}:{type(e.error).__name__}' for e in errs})
   
   
   def case(name, values, pd_dtype=None, pl_dtype=None, pa_type=None, backends=('pd', 'pl', 'xo')):
       print(f'\n### {name}')
       for b in backends:
           try:
               c, errs = run(b, values, pd_dtype, pl_dtype, pa_type)
           except Exception as e:
               print(f'  {b}: RAISED {type(e).__name__}: {str(e)[:150]}')
               continue
           print(f"  {b}: {summarize(c.get('histogram'))}")
           print(f"      {bins_s(c.get('histogram_bins'))} min={c.get('min')!r:.30} max={c.get('max')!r:.30} "
                 f"nan_per={c.get('nan_per')} distinct={c.get('distinct_count')}" + (f' ERRS={errs}' if errs else ''))
   
   # watch the low tail (first bar): pandas passes min through as a numpy int
   case('int8 -128..127', list(range(-128, 128)), pd_dtype='int8', pl_dtype=pl.Int8, pa_type=pa.int8())
   case('int16 -128..127 (min is not the dtype minimum)', list(range(-128, 128)), pd_dtype='int16', pl_dtype=pl.Int16, pa_type=pa.int16())
   case('int32 -2**31 beside 0..99', [-2**31] + list(range(100)), pd_dtype='int32', pl_dtype=pl.Int32, pa_type=pa.int32())
   case('int64 -2**63 beside 0..99', [-2**63] + list(range(100)), pd_dtype='int64', pl_dtype=pl.Int64, pa_type=pa.int64())
   print()
   print('abs(np.int8(-128)) =', abs(np.int8(-128)), '| abs(np.int64(-2**63)) =', abs(np.int64(-2**63)))

   Output at e88ace9

   ### int8 -128..127
     pd: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] 0<>-125:tai=1 | -125<>-100:pop=10 | -100<>-75:pop=10 | -75<>-50:pop=10 | -50<>-25:pop=10 | -25<>-0:pop=10 | -0<>24:pop=10 | 24–49:pop=10 | 49–74:pop=10 | 74–99:pop=10 | 99–124:pop=10 | 124–127:tai=1
         bins=11 [-125..124] min=np.int8(-128) max=np.int8(127) nan_per=0.0 distinct=256
     pl: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] -128<>-125:tai=1 | -124<>-99:pop=10 | -99<>-75:pop=10 | -75<>-50:pop=10 | -50<>-25:pop=10 | -25<>-0:pop=10 | -0<>24:pop=10 | 24–49:pop=10 | 49–74:pop=10 | 74–98:pop=10 | 98–123:pop=10 | 124–127:tai=1
         bins=11 [-124..123] min=-128 max=127 nan_per=0.0 distinct=256
     xo: [{'population': 10} pop=100.4 cat=0 NA=0] -128<>-102:pop=10.2 | -102<>-77:pop=9.8 | -77<>-52:pop=10.2 | -52<>-26:pop=9.8 | -26<>-0:pop=10.2 | -0<>25:pop=9.8 | 25–50:pop=10.2 | 50–76:pop=9.8 | 76–102:pop=10.2 | 102–127:pop=10.2
         bins=11 [-128..127] min=-128 max=127 nan_per=0.0 distinct=254
   
   ### int16 -128..127 (min is not the dtype minimum)
     pd: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] -128<>-125:tai=1 | -125<>-100:pop=10 | -100<>-75:pop=10 | -75<>-50:pop=10 | -50<>-25:pop=10 | -25<>-0:pop=10 | -0<>24:pop=10 | 24–49:pop=10 | 49–74:pop=10 | 74–99:pop=10 | 99–124:pop=10 | 124–127:tai=1
         bins=11 [-125..124] min=np.int16(-128) max=np.int16(127) nan_per=0.0 distinct=256
     pl: [{'tail': 2, 'population': 10} pop=100 cat=0 NA=0] -128<>-125:tai=1 | -124<>-99:pop=10 | -99<>-75:pop=10 | -75<>-50:pop=10 | -50<>-25:pop=10 | -25<>-0:pop=10 | -0<>24:pop=10 | 24–49:pop=10 | 49–74:pop=10 | 74–98:pop=10 | 98–123:pop=10 | 124–127:tai=1
         bins=11 [-124..123] min=-128 max=127 nan_per=0.0 distinct=256
     xo: [{'population': 10} pop=100.4 cat=0 NA=0] -128<>-102:pop=10.2 | -102<>-77:pop=9.8 | -77<>-52:pop=10.2 | -52<>-26:pop=9.8 | -26<>-0:pop=10.2 | -0<>25:pop=9.8 | 25–50:pop=10.2 | 50–76:pop=9.8 | 76–102:pop=10.2 | 102–127:pop=10.2
         bins=11 [-128..127] min=-128 max=127 nan_per=0.0 distinct=255
   
   ### int32 -2**31 beside 0..99
     pd: [{'tail': 2, 'population': 10} pop=97 cat=0 NA=0] 0–0:tai=1 | 1–10.6:pop=10 | 10.6–20.2:pop=10 | 20.2–29.8:pop=9 | 29.8–39.4:pop=10 | 39.4–49:pop=9 | 49–58.6:pop=10 | 58.6–68.2:pop=10 | 68.2–77.8:pop=9 | 77.8–87.4:pop=10 | 87.4–97:pop=10 | 98–99:tai=1
         bins=11 [1..97] min=np.int32(-2147483648) max=np.int32(99) nan_per=0.0 distinct=101
     pl: [{'tail': 2, 'population': 10} pop=97 cat=0 NA=0] -2.15B<>0:tai=1 | 1–10.6:pop=10 | 10.6–20.2:pop=10 | 20.2–29.8:pop=9 | 29.8–39.4:pop=10 | 39.4–49:pop=9 | 49–58.6:pop=10 | 58.6–68.2:pop=10 | 68.2–77.8:pop=9 | 77.8–87.4:pop=10 | 87.4–97:pop=10 | 98–99:tai=1
         bins=11 [1..97] min=-2147483648 max=99 nan_per=0.0 distinct=101
     xo: [{'population': 2} pop=100 cat=0 NA=0] -2.15B<>-1.93B:pop=1 | -0.21B<>0B:pop=99
         bins=11 [-2.14748e+09..99] min=-2147483648 max=99 nan_per=0.0 distinct=101
   
   ### int64 -2**63 beside 0..99
     pd: [{'tail': 2, 'population': 10} pop=97 cat=0 NA=0] 0–0:tai=1 | 1–10.6:pop=10 | 10.6–20.2:pop=10 | 20.2–29.8:pop=9 | 29.8–39.4:pop=10 | 39.4–49:pop=9 | 49–58.6:pop=10 | 58.6–68.2:pop=10 | 68.2–77.8:pop=9 | 77.8–87.4:pop=10 | 87.4–97:pop=10 | 98–99:tai=1
         bins=11 [1..97] min=np.int64(-9223372036854775808) max=np.int64(99) nan_per=0.0 distinct=101
     pl: [{'tail': 2, 'population': 10} pop=97 cat=0 NA=0] -9223372.04T<>0:tai=1 | 1–10.6:pop=10 | 10.6–20.2:pop=10 | 20.2–29.8:pop=9 | 29.8–39.4:pop=10 | 39.4–49:pop=9 | 49–58.6:pop=10 | 58.6–68.2:pop=10 | 68.2–77.8:pop=9 | 77.8–87.4:pop=10 | 87.4–97:pop=10 | 98–99:tai=1
         bins=11 [1..97] min=-9223372036854775808 max=99 nan_per=0.0 distinct=101
     xo: [{'population': 2} pop=100 cat=0 NA=0] -9223372T<>-8301035T:pop=1 | -922337T<>0T:pop=99
         bins=11 [-9.22337e+18..0] min=-9223372036854775808 max=99 nan_per=0.0 distinct=101
   
   abs(np.int8(-128)) = -128 | abs(np.int64(-2**63)) = -9223372036854775808