From 29e1e42c48de8fb30965706aabb92f358eb8c8f8 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sat, 3 Oct 2026 22:58:11 -0400 Subject: [PATCH 01/18] test(client): failing tests for client hardening under a two-message protocol (rows-first c0a) Tests for the client half of plan 1 phase 0a. A server that sends the summary stats separately, or not at all, leaves the first message without values the client has been assuming. BuckarooView: a change that reached the model before the effect subscribed, one initial_state with metadata decoding once, and out-of-order decodes applying the newer. Pinned rows: a valueless key shows a placeholder with its own row id while df_meta.stats.status is pending, and is omitted when not_computed. The simple tooltip returns nothing for a valueless cell. color_map is silent without bins and restyles when they arrive. An unrelated df_data_dict update keeps the in-flight indicator when the server reports df_meta.stats. One Storybook Playwright test covers pending, not_computed and complete in a real browser, and is added to the Storybook list in scripts/test_playwright_storybook.sh. Co-Authored-By: Claude Sonnet 5.5 --- .../stats-pending-pinned-rows.spec.ts | 55 +++++ .../BuckarooInfiniteWidget.flash.test.tsx | 209 +++++++++++++++++- .../DFViewerParts/gridUtils.test.ts | 69 +++++- .../src/server/BuckarooView.test.tsx | 134 ++++++++++- .../StatsPendingPinnedRows.stories.tsx | 130 +++++++++++ scripts/test_playwright_storybook.sh | 1 + 6 files changed, 590 insertions(+), 8 deletions(-) create mode 100644 packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts create mode 100644 packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx diff --git a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts new file mode 100644 index 000000000..88c60f4a4 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts @@ -0,0 +1,55 @@ +/** + * Playwright test for the client states of the two-message protocol + * (rows-first c0a). + * + * While summary stats have not arrived, `df_meta.stats.status` decides what + * the pinned area shows. The StatsPendingPinnedRows story flips the status and + * supplies the stats on "complete": + * - pending: every valueless pinned key shows a placeholder row with its own + * row id, and hovering a valueless cell does not throw + * - not_computed: valueless pinned keys are omitted + * - complete: the values appear and the color-mapped column restyles + */ +import { test, expect } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-dfviewer-statspendingpinnedrows--primary&globals=&args="; + +test("pinned rows follow df_meta.stats.status: placeholders, omitted, then values and colors", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + const bodyCellA = page.locator('.ag-center-cols-container .ag-row[row-index="0"] [col-id="a"]'); + const backgroundOf = (loc: typeof bodyCellA) => loc.evaluate((el) => getComputedStyle(el).backgroundColor); + + // pending: one placeholder per pinned key, each with its own row id. + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(bodyCellA).toHaveText("1"); + const backgroundBeforeBins = await backgroundOf(bodyCellA); + + // Hovering a valueless pinned cell shows no tooltip and throws nothing. + await page.locator('.ag-floating-top .ag-cell[col-id="a"]').first().hover(); + await page.waitForTimeout(500); + expect(pageErrors).toEqual([]); + + // not_computed: the valueless keys are omitted. + await page.getByTestId("status-not_computed").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + + // complete: the values appear, and the color-mapped column restyles now + // that its histogram bins exist. + await page.getByTestId("status-complete").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + await expect.poll(() => backgroundOf(bodyCellA), { timeout: 10_000 }).not.toBe(backgroundBeforeBins); + expect(pageErrors).toEqual([]); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index a117e9816..51a14cb41 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -7,7 +7,7 @@ * Tests assert CURRENT behavior on main (Option A in docs/rerender-test-plan.md). * Tests tagged "[captures current flash]" are tracking pain, not validating it. */ -import { render } from "@testing-library/react"; +import { render, act } from "@testing-library/react"; import { BuckarooInfiniteWidget } from "./BuckarooWidgetInfinite"; import { KeyAwareSmartRowCache } from "./DFViewerParts/SmartRowCache"; import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridSpy"; @@ -21,8 +21,14 @@ jest.mock("ag-grid-react", () => jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); // StatusBar also instantiates AgGridReact; stub it so the spy only counts the data grid. +// The stub records its props so the in-flight tests can read inFlight and drive +// setBuckarooState the way the real status bar does. +let mockStatusBarProps: any = null; jest.mock("./StatusBar", () => ({ - StatusBar: () =>
, + StatusBar: (props: any) => { + mockStatusBarProps = props; + return
; + }, })); // DFViewerInfinite-prop capture for identity-stability assertion. @@ -656,3 +662,202 @@ describe("BuckarooInfiniteWidget — flash matrix (current behavior)", () => { }); }); }); + +// Rows-first c0a: the grid has to behave while summary stats are pending, not +// computed, or arrive after the first paint. df_meta.stats.status drives it; +// absent df_meta.stats means "complete", as servers without the field send. +describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", () => { + const withStats = (status: string): DFMeta => ({ ...baseDfMeta, stats: { status } } as DFMeta); + + const pinnedConfig: DFViewerConfig = { + ...baseConfig, + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "histogram", displayer_args: { displayer: "obj" } }, + ], + }; + const pinnedDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: pinnedConfig, summary_stats_key: "summary_stats" }, + }; + + const widgetProps = (over: Record = {}) => ({ + df_data_dict: { summary_stats: [] as any[] }, + df_display_args: pinnedDisplayArgs, + df_meta: baseDfMeta, + operations: [], + on_operations: jest.fn(), + operation_results: {} as any, + command_config: { argspecs: {}, defaultArgs: {} }, + buckaroo_state: initialState, + on_buckaroo_state: jest.fn(), + buckaroo_options: baseOptions, + src: mkSrc(), + ...over, + }); + + const lastPinnedRows = (): any[] => { + const sets = getSpyCalls().setGridOption.filter(([k]) => k === "pinnedTopRowData"); + return sets[sets.length - 1][1] as any[]; + }; + const pinnedRowIds = (rows: any[]): string[] => { + const { gridOptions, context } = getSpyCalls().lastProps; + return rows.map((data) => gridOptions.getRowId({ data, level: 0, rowPinned: "top", context })); + }; + + describe("pinned rows with no value", () => { + it("pending: shows one placeholder per valueless key, each with its own row id", () => { + render(); + const rows = lastPinnedRows(); + // Each placeholder keeps its key as the row label, so the pinned area + // holds its height and the rows are told apart. + expect(rows.map((r) => r?.index)).toEqual(["dtype", "histogram"]); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: a key that has a value renders it and only the other key gets a placeholder", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + const rows = lastPinnedRows(); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]?.index).toBe("histogram"); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: placeholders turn into the real rows when stats arrive", () => { + const props = widgetProps({ df_meta: withStats("pending") }); + const { rerender } = render(); + const stats = [ + { index: "dtype", a: "int64" }, + { index: "histogram", a: [{ name: "1-5", population: 100 }] }, + ]; + rerender( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("not_computed: omits the valueless keys", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("not_computed: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("no df_meta.stats (an older server) behaves as complete: valueless keys stay undefined rows", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render(); + const rows = lastPinnedRows(); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]).toEqual({}); + }); + }); + + describe("color_map columns", () => { + const colorConfig: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + }; + const colorDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: colorConfig, summary_stats_key: "summary_stats" }, + }; + const colorProps = (stats: any[]) => + widgetProps({ df_display_args: colorDisplayArgs, df_data_dict: { summary_stats: stats } }); + const forcedRefreshes = () => getSpyCalls().refreshCells.filter((p: any) => p.force === true); + + it("restyles the color-mapped columns when histogram bins arrive after the first render", () => { + const props = colorProps([{ index: "dtype", a: "int64", b: "int64" }]); + const { rerender } = render(); + expect(forcedRefreshes()).toHaveLength(0); + + rerender( + , + ); + const refreshes = forcedRefreshes(); + expect(refreshes).toHaveLength(1); + expect(refreshes[0].columns).toEqual(["a"]); + }); + + it("does not refresh when the new stats leave the bins unchanged", () => { + const props = colorProps([{ index: "histogram_bins", a: [1, 2, 3, 4, 5] }]); + const { rerender } = render(); + rerender( + , + ); + expect(forcedRefreshes()).toHaveLength(0); + }); + }); + + describe("in-flight indicator", () => { + const inFlightAttr = () => document.querySelector('[data-testid="status-bar-stub"]')!.getAttribute("data-inflight"); + const dispatchSearch = () => + act(() => { + mockStatusBarProps.setBuckarooState({ ...initialState, quick_command_args: { search: ["x"] } }); + }); + const props = (meta: DFMeta, dict: Record) => widgetProps({ df_meta: meta, df_data_dict: dict }); + + it("a df_data_dict update alone does not clear it when the server reports df_meta.stats", () => { + const meta = withStats("pending"); + const first = props(meta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + // A stats-only update: a new df_data_dict, the same df_meta. + rerender(); + expect(inFlightAttr()).toBe("true"); + }); + + it("clears when a frame with a new df_meta and df_data_dict arrives", () => { + const first = props(withStats("pending"), { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + + rerender( + , + ); + expect(inFlightAttr()).toBe("false"); + }); + + it("without df_meta.stats a new df_data_dict still clears it, as before", () => { + const first = props(baseDfMeta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + rerender(); + expect(inFlightAttr()).toBe("false"); + }); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts index 01697ba2e..944ceddbc 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts @@ -15,7 +15,8 @@ import { import * as _ from "lodash-es"; import { DFData, DFViewerConfig, NormalColumnConfig, MultiIndexColumnConfig, PinnedRowConfig, ColumnConfig, FormatterArgs } from "./DFWhole"; import { getFormatter, getFloatFormatter, getCompactNumberFormatter, formatDuration, formatIsoDuration, getDurationFormatter } from './Displayer'; -import { ColDef, ICellRendererParams, ValueFormatterParams } from 'ag-grid-community'; +import { CellClassParams, ColDef, ICellRendererParams, ITooltipParams, ValueFormatterParams } from 'ag-grid-community'; +import { getSimpleTooltip } from './SeriesSummaryTooltip'; describe("testing utility functions in gridUtils ", () => { // mostly sanity checks to help develop gridUtils @@ -732,6 +733,70 @@ describe("testing multi index organiztion ", () => { expect(children.length).toBe(2); }); +}); + +// Rows-first c0a: while summary stats are pending or not computed, pinned +// cells have no value, and color_map has no histogram bins to read. +describe("pinned cells without stats values (rows-first c0a)", () => { + it("the simple tooltip returns nothing for a valueless pinned cell instead of throwing", () => { + const tooltip = getSimpleTooltip("a"); + // A pinned row with no value for the column: only the row label exists. + expect(() => tooltip({ data: { index: "dtype" } } as ITooltipParams)).not.toThrow(); + expect(tooltip({ data: { index: "dtype" } } as ITooltipParams)).toBeUndefined(); + // A null cell and a row that has not loaded (data undefined) are also valueless. + expect(() => tooltip({ data: { index: "mean", a: null } } as ITooltipParams)).not.toThrow(); + expect(() => tooltip({ data: undefined } as unknown as ITooltipParams)).not.toThrow(); + }); + + it("the simple tooltip still renders a cell that has a value", () => { + const tooltip = getSimpleTooltip("a"); + const el = tooltip({ data: { index: 0, a: 5 } } as ITooltipParams); + expect(el).toBeDefined(); + expect((el as any).props.children).toBe("5"); + }); + + describe("color_map without histogram bins", () => { + const config: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + }, + ], + }; + const cellStyleFor = (context: any, value: any = 3) => { + const colDef = dfToAgrid(config)[0] as ColDef; + const cellStyle = colDef.cellStyle as (p: CellClassParams) => Record; + return cellStyle({ context, data: { index: 0, a: value }, value, node: { rowPinned: undefined } } as unknown as CellClassParams); + }; - + let logSpy: jest.SpyInstance; + beforeEach(() => { logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); }); + afterEach(() => { logSpy.mockRestore(); }); + + it("returns the neutral style without logging when the stats column has no entry", () => { + expect(cellStyleFor({ histogram_stats: {} })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the entry has no histogram_bins", () => { + expect(cellStyleFor({ histogram_stats: { a: { histogram_log_bins: [1, 2] } } })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the context carries no histogram_stats", () => { + expect(cellStyleFor({})).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("colors the cell once bins exist", () => { + const style = cellStyleFor({ histogram_stats: { a: { histogram_bins: [1, 2, 3, 4, 5] } } }); + expect(style.backgroundColor).not.toBe("inherit"); + expect(style.backgroundColor).toBeDefined(); + }); + }); }); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx index 8a38aa6cf..796e1db75 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx @@ -9,18 +9,34 @@ import { render, cleanup, act } from "@testing-library/react"; import { BuckarooView } from "./BuckarooView"; import type { IModel } from "./IModel"; +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; // Stub the heavy widget surfaces — this test exercises the injection // wiring, not AG-Grid. The widget components instantiate AgGridReact which // is fragile under jsdom; the stub keeps the test focused on the model // contract. +// +// The viewer stub records the props BuckarooView hands it, so the rows-first +// tests below can see which df_meta / df_data_dict reached the widget. +const mockViewerProps: any[] = []; jest.mock("../components/BuckarooWidgetInfinite", () => ({ BuckarooInfiniteWidget: () =>
, - DFViewerInfiniteDS: () =>
, + DFViewerInfiniteDS: (props: any) => { + mockViewerProps.push(props); + return
; + }, getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), })); -function makeFakeModel(): { model: IModel; events: Map>; sent: any[] } { +// Wrap decodeDFDataDict in a jest.fn that defaults to the real decoder, so the +// existing tests run unchanged and the rows-first tests can count and delay +// decodes. +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { ...actual, decodeDFDataDict: jest.fn(actual.decodeDFDataDict) }; +}); + +function makeFakeModel(): { model: IModel; events: Map>; sent: any[]; state: Record } { const events = new Map>(); const state: Record = {}; const sent: any[] = []; @@ -35,10 +51,13 @@ function makeFakeModel(): { model: IModel; events: Map>; s }, off: (e, h) => { events.get(e)?.delete(h); }, }; - return { model, events, sent }; + return { model, events, sent, state }; } -afterEach(() => cleanup()); +afterEach(() => { + mockViewerProps.length = 0; + cleanup(); +}); describe("BuckarooView (injectable IModel — #759)", () => { it("renders the viewer widget when given a fake IModel + initialState — no WebSocket needed", async () => { @@ -102,3 +121,110 @@ describe("BuckarooView (injectable IModel — #759)", () => { expect(onMetadata).toHaveBeenCalledWith({ path: "/data/sales.parquet", rows: 42 }, "tell me about sales"); }); }); + +// Rows-first c0a: a second message can follow the first at any moment, so the +// view has to cope with changes that land while it is still wiring itself up. +describe("BuckarooView two-message hardening (rows-first c0a)", () => { + const mockDecode = decodeDFDataDict as jest.Mock; + const realDecode = jest.requireActual("../components/DFViewerParts/resolveDFData").decodeDFDataDict; + + const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, + }; + const metaWith = (total_rows: number) => ({ total_rows, columns: 1, filtered_rows: total_rows, rows_shown: total_rows }); + const emit = (events: Map>, name: string, ...args: unknown[]) => { + for (const h of Array.from(events.get(name) ?? [])) h(...args); + }; + const lastProps = () => mockViewerProps[mockViewerProps.length - 1]; + + beforeEach(() => { + mockDecode.mockReset(); + mockDecode.mockImplementation(realDecode); + }); + + it("applies a change that reached the model before the effect subscribed", async () => { + // initialState is what the host held when it built the view. A second + // initial_state then landed on the model while React was committing, + // so its change:* events had no listener yet. + const { model, state } = makeFakeModel(); + state.df_meta = metaWith(99); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_meta.total_rows).toBe(99); + }); + + it("decodes a df_data_dict that reached the model before the effect subscribed", async () => { + const { model, state } = makeFakeModel(); + const raw = { format: "mock", id: "N" }; + state.df_data_dict = { main: raw }; + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "N" }]); + }); + + it("decodes one initial_state that carries metadata once, and renders its df_data_dict once", async () => { + const { model, events, state } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + const dictsBefore = new Set(mockViewerProps.map((p) => p.df_data_dict)); + mockDecode.mockClear(); + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + + // The order WebSocketModel emits for a full frame: one change per key, + // then "metadata". The model state already holds the new values. + const frame = { + df_meta: metaWith(7), + df_data_dict: { main: { format: "mock", id: "F" } }, + df_display_args: displayArgs, + metadata: { path: "/data/f.parquet", rows: 7 }, + }; + Object.assign(state, frame); + await act(async () => { + emit(events, "change:df_meta", frame.df_meta); + emit(events, "change:df_data_dict", frame.df_data_dict); + emit(events, "change:df_display_args", frame.df_display_args); + emit(events, "metadata", frame.metadata, undefined); + }); + + expect(mockDecode).toHaveBeenCalledTimes(1); + const dictsAfter = new Set(mockViewerProps.map((p) => p.df_data_dict)); + expect(dictsAfter.size - dictsBefore.size).toBe(1); + expect(lastProps().df_meta.total_rows).toBe(7); + }); + + it("applies the newer df_data_dict when decodes complete out of order", async () => { + const { model, events } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + + const finish: Record void> = {}; + mockDecode.mockImplementation( + (dict: any) => + new Promise((resolve) => { + finish[dict.main.id] = () => resolve({ main: [{ index: 0, from: dict.main.id }] }); + }), + ); + await act(async () => { + emit(events, "change:df_data_dict", { main: { format: "mock", id: "older" } }); + emit(events, "change:df_data_dict", { main: { format: "mock", id: "newer" } }); + }); + // The newer decode finishes first, then the stale one. + await act(async () => { finish["newer"](); }); + await act(async () => { finish["older"](); }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "newer" }]); + }); +}); diff --git a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx new file mode 100644 index 000000000..caa5628c1 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx @@ -0,0 +1,130 @@ +/** + * Story for the two-message protocol's client states (rows-first c0a). + * + * The server sends the first message once the schema and row count are known, + * and the summary stats later or never. `df_meta.stats.status` says which: + * + * - "pending" pinned keys with no value show a placeholder row + * - "not_computed" pinned keys with no value are omitted + * - "complete" stats are present; also the meaning of a missing + * `df_meta.stats`, as servers without the field send + * + * Column `a` is color-mapped and has a simple tooltip, so the story also shows + * that cells restyle when the histogram bins arrive and that hovering a + * valueless pinned cell does nothing. Used by stats-pending-pinned-rows.spec.ts. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import { DFViewerInfiniteDS } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { DFMeta } from "../components/WidgetTypes"; + +type Status = "pending" | "not_computed" | "complete"; +const STATUSES: Status[] = ["pending", "not_computed", "complete"]; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, + { index: 3, a: 4, b: "w" }, + { index: 4, a: 5, b: "v" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 3, b: "N/A" }, + { index: "histogram_bins", a: [0, 1, 2, 3, 4, 5], b: [] }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + tooltip_config: { tooltip_type: "simple", val_column: "a" }, + }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const StatsPendingPinnedRowsInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { status }, + }) as DFMeta, + [status], + ); + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : ([] as DFData), + empty: [] as DFData, + }), + [status], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + df_meta.stats.status = {status} +
+
+ +
+
+ ); +}; + +const meta = { + title: "Buckaroo/DFViewer/StatsPendingPinnedRows", + component: StatsPendingPinnedRowsInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; diff --git a/scripts/test_playwright_storybook.sh b/scripts/test_playwright_storybook.sh index c0064c16d..6d835ab2a 100755 --- a/scripts/test_playwright_storybook.sh +++ b/scripts/test_playwright_storybook.sh @@ -95,6 +95,7 @@ STORYBOOK_TESTS=( "pw-tests/transcript-replayer.spec.ts" "pw-tests/outside-params.spec.ts" "pw-tests/fitcontent-height.spec.ts" + "pw-tests/stats-pending-pinned-rows.spec.ts" # "pw-tests/example.spec.ts" # Has pre-existing failures, excluded for now ) From 405f4ded5f3a767e39cfd542abe461c4b54f8417 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sat, 3 Oct 2026 23:10:59 -0400 Subject: [PATCH 02/18] fix(client): harden the client for a two-message protocol (rows-first c0a) BuckarooView reads model.get() for every key once its listeners are in place, so a change:* emitted before the effect ran reaches React. All df_data_dict values go through one decoder (makeLatestDictDecoder) that skips a dict it has already seen and applies only the newest decode, so an initial_state with metadata decodes once and out-of-order decodes cannot overwrite a newer one. standalone.tsx gets the same decoder and catch-up. df_meta.stats.status reaches the grid as stats_status. While pending, a required pinned key with no value becomes a placeholder row that carries its key as index (unique row id, empty value cells). When not_computed it is omitted. A missing df_meta.stats, or any other status, keeps the old behaviour. The simple tooltip returns nothing for a cell with no value or no row data. color_map no longer logs when bins are missing, and the grid refreshes the color-mapped columns when their bins change after the first render. That needs RenderApiModule, which was not registered, so api.refreshCells logged AG Grid error 200 and did nothing. BuckarooInfiniteWidget keeps inFlight when only df_data_dict changed and the server reports df_meta.stats; the answer to the dispatched change is a frame with a new df_meta. Without df_meta.stats the rule is unchanged. The Playwright story test no longer hovers a valueless pinned cell: with the old tooltip it still passed, since AG Grid does not call the tooltip for an empty value. Co-Authored-By: Claude Sonnet 5.5 --- .../stats-pending-pinned-rows.spec.ts | 7 +- .../src/components/BuckarooWidgetInfinite.tsx | 19 +++++- .../DFViewerParts/DFViewerInfinite.tsx | 45 ++++++++++++- .../DFViewerParts/SeriesSummaryTooltip.tsx | 11 +++- .../src/components/DFViewerParts/Styler.tsx | 5 +- .../src/components/DFViewerParts/gridUtils.ts | 32 +++++++-- .../src/components/WidgetTypes.tsx | 18 +++++ packages/buckaroo-js-core/src/index.ts | 3 + .../src/server/BuckarooView.tsx | 60 ++++++++++++----- .../src/server/latestDictDecoder.ts | 34 ++++++++++ .../StatsPendingPinnedRows.stories.tsx | 6 +- packages/js/standalone.tsx | 65 ++++++++++++++++--- 12 files changed, 253 insertions(+), 52 deletions(-) create mode 100644 packages/buckaroo-js-core/src/server/latestDictDecoder.ts diff --git a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts index 88c60f4a4..91c8344e6 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts @@ -6,7 +6,7 @@ * the pinned area shows. The StatsPendingPinnedRows story flips the status and * supplies the stats on "complete": * - pending: every valueless pinned key shows a placeholder row with its own - * row id, and hovering a valueless cell does not throw + * row id * - not_computed: valueless pinned keys are omitted * - complete: the values appear and the color-mapped column restyles */ @@ -36,11 +36,6 @@ test("pinned rows follow df_meta.stats.status: placeholders, omitted, then value await expect(bodyCellA).toHaveText("1"); const backgroundBeforeBins = await backgroundOf(bodyCellA); - // Hovering a valueless pinned cell shows no tooltip and throws nothing. - await page.locator('.ag-floating-top .ag-cell[col-id="a"]').first().hover(); - await page.waitForTimeout(500); - expect(pageErrors).toEqual([]); - // not_computed: the valueless keys are omitted. await page.getByTestId("status-not_computed").click(); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index d126a87c7..07e3a7b50 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -7,7 +7,7 @@ import { DFData } from "./DFViewerParts/DFWhole"; import { StatusBar } from "./StatusBar"; import { BuckarooState } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, getStatsStatus } from "./WidgetTypes"; import { CommandConfigT } from "./CommandUtils"; import { Operation } from "./OperationUtils"; import { @@ -290,13 +290,23 @@ export function BuckarooInfiniteWidget({ // so we don't accidentally clear on the same dict the user was looking // at when they fired the change. Cleared on the *next* dict reference. const inFlightAtDictRef = useRef(null); + // The df_meta at that moment. A server that reports df_meta.stats can + // send df_data_dict on its own (a stats update for an earlier state), + // and df_meta only comes with a full frame, so for those servers a + // new df_meta is also needed before the answer to the change counts + // as arrived. Servers without df_meta.stats keep the dict-only rule. + const inFlightAtMetaRef = useRef(null); useEffect(() => { if (inFlight && inFlightAtDictRef.current !== null && df_data_dict !== inFlightAtDictRef.current) { + if (df_meta === inFlightAtMetaRef.current && df_meta.stats !== undefined) { + return; + } setInFlight(false); inFlightAtDictRef.current = null; + inFlightAtMetaRef.current = null; } - }, [df_data_dict, inFlight]); + }, [df_data_dict, df_meta, inFlight]); const wrappedOnBuckarooState = useCallback< React.Dispatch> @@ -311,10 +321,11 @@ export function BuckarooInfiniteWidget({ !== JSON.stringify(buckaroo_state.quick_command_args); if (dataflowChanged) { inFlightAtDictRef.current = df_data_dict; + inFlightAtMetaRef.current = df_meta; setInFlight(true); } on_buckaroo_state(next); - }, [buckaroo_state, df_data_dict, on_buckaroo_state]); + }, [buckaroo_state, df_data_dict, df_meta, on_buckaroo_state]); return (
{buckaroo_state.show_commands ? ( @@ -447,6 +459,7 @@ export function DFViewerInfiniteDS({ activeCol={activeCol} setActiveCol={setActiveCol} error_info={""} + stats_status={getStatsStatus(df_meta)} />
diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx index 7c8fa079e..4f4465ccb 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx @@ -7,7 +7,8 @@ import { import * as _ from "lodash-es"; import { DFData, DFDataRow, DFViewerConfig, SDFT } from "./DFWhole"; -import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT } from "./gridUtils"; +import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT, getFieldVal } from "./gridUtils"; +import type { StatsStatus } from "../WidgetTypes"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { @@ -21,6 +22,7 @@ import { CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -45,6 +47,9 @@ ModuleRegistry.registerModules([ CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + // api.refreshCells lives here. Without it the call logs AG Grid error 200 + // and does nothing. + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -150,6 +155,7 @@ export function DFViewerInfinite({ max_rows_in_configs, view_name, data_key, + stats_status, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -173,6 +179,10 @@ export function DFViewerInfinite({ // a rowId, even though their `index` values overlap (row 0 in main is a // different record than row 0 in summary). data_key?: string; + // df_meta.stats.status. While "pending" a pinned key with no value shows a + // placeholder row; when "not_computed" it is omitted. Undefined behaves + // as "complete". + stats_status?: StatsStatus; }) { /* The idea is to do some pre-setup here for @@ -233,6 +243,7 @@ export function DFViewerInfinite({ effectiveScheme={effectiveScheme} view_name={view_name} data_key={data_key} + stats_status={stats_status} />
) @@ -250,6 +261,7 @@ export function DFViewerInfiniteInner({ effectiveScheme, view_name, data_key, + stats_status, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -266,6 +278,7 @@ export function DFViewerInfiniteInner({ effectiveScheme?: 'light' | 'dark'; view_name?: string; data_key?: string; + stats_status?: StatsStatus; }) { /* const lastProps = useRef(null); @@ -344,8 +357,8 @@ export function DFViewerInfiniteInner({ // Always re-extract; upstream may mutate summary in-place without changing identity // Memoize to ensure it updates when summary_stats_data changes const topRowData = useMemo( - () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : []) as DFDataRow[], - [summary_stats_data, pinned_rows] + () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : [], stats_status) as DFDataRow[], + [summary_stats_data, pinned_rows, stats_status] ); // Pinned rows are extracted and ready @@ -442,6 +455,32 @@ export function DFViewerInfiniteInner({ // ignore until grid ready } }, [pinnedSig]); + + // color_map reads histogram_bins from the grid context when a cell is + // painted, so cells that rendered before the bins arrived keep the + // neutral style. Repaint the color-mapped columns when their bins + // change after the first render. Bins that are already there at mount + // paint correctly, so the first run only records the signature. + const colorMapCols = useMemo( + () => df_viewer_config.column_config.flatMap((cc) => + cc.color_map_config?.color_rule === "color_map" + ? [{ field: getFieldVal(cc), statsCol: cc.color_map_config.val_column }] + : []), + [df_viewer_config.column_config], + ); + const colorMapSig = useMemo(() => { + if (colorMapCols.length === 0) return ""; + const stats = extractSDFT(summary_stats_data); + return JSON.stringify(colorMapCols.map( + ({ field, statsCol }) => [field, statsCol === undefined ? undefined : stats[statsCol]?.histogram_bins])); + }, [colorMapCols, summary_stats_data]); + const colorMapSigRef = useRef(colorMapSig); + useEffect(() => { + if (colorMapSigRef.current === colorMapSig) return; + colorMapSigRef.current = colorMapSig; + if (colorMapCols.length === 0) return; + gridRef.current?.api?.refreshCells({ force: true, columns: colorMapCols.map((c) => c.field) }); + }, [colorMapSig, colorMapCols]); // Force update rowData when Raw data changes const rawDataSig = useMemo(() => { diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx index 907fef642..bfbfad3aa 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx @@ -24,11 +24,16 @@ export const getSimpleTooltip = (tooltipField:string) => { // This should be possible with the tooltipValueGetter, but that // wasn't working for some reason - if (props.data.index === "histogram") { + if (props.data?.index === "histogram") { return; } - const val = props.data[tooltipField].toString() - return
{val}
; + // A pinned row whose stats have not arrived has no value for the + // column, and a row that has not loaded has no data: show nothing. + const raw = props.data?.[tooltipField]; + if (raw === undefined || raw === null) { + return; + } + return
{raw.toString()}
; }; return simpleTooltip; } diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx index ef83da6ae..fcdd0baff 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx @@ -64,13 +64,14 @@ export function colorMap(cmr: ColorMapRules) { const summarys = params.context?.histogram_stats; const statsCol = cmr.val_column; // || col_name; + // No bins is the normal state until the summary stats arrive, so + // fall back to the neutral style quietly; the grid repaints the + // column when they do. if (statsCol === undefined || summarys === undefined){ - console.log("66 couldn't find stats_col") return baseReturn; } const summary_stats_cell = summarys[statsCol]; if (summary_stats_cell === undefined || summary_stats_cell.histogram_bins === undefined ) { - console.log("69 couldn't find summary_stats"); return baseReturn } const histogram_edges = summary_stats_cell.histogram_bins; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts index b963123e4..c9b29af66 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts @@ -38,6 +38,7 @@ import { getFormatterFromArgs, getCellRenderer, objFormatter, getFormatter } fro import { CSSProperties, Dispatch, SetStateAction } from "react"; import { CommandConfigT } from "../CommandUtils"; import { KeyAwareSmartRowCache, PayloadArgs } from "./SmartRowCache"; +import type { StatsStatus } from "../WidgetTypes"; // for now colDef stuff with less than 3 implementantions should stay in this file @@ -97,13 +98,33 @@ export function stripOptionalPinnedKey(key: string): string { return isOptionalPinnedKey(key) ? key.slice(1) : key; } -export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[]) { +// Marks a pinned row that stands in for stats that have not arrived. The row +// carries its key as `index`, so it keeps its label and gets a row id of its +// own, and the cell renderer selector leaves its value cells empty. +export const PENDING_STAT_ROW_KEY = "__stat_pending"; + +const PendingStatCell = () => null; +const pendingStatRenderer: CellRendererSelectorResult = { component: PendingStatCell }; + +// `statsStatus` is df_meta.stats.status. A required key with no value is, by +// status: +// "pending" a placeholder row, which holds the pinned area's height +// "not_computed" omitted, since no value is coming +// anything else undefined, as it was before df_meta.stats existed +export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[], statsStatus?: StatsStatus) { const result: (DFData[number] | undefined)[] = []; for (const cfg of prc) { const raw = cfg.primary_key_val; - const found = _.find(sdf, { index: stripOptionalPinnedKey(raw) }); - if (found === undefined && isOptionalPinnedKey(raw)) { - continue; + const key = stripOptionalPinnedKey(raw); + const found = _.find(sdf, { index: key }); + if (found === undefined) { + if (isOptionalPinnedKey(raw) || statsStatus === "not_computed") { + continue; + } + if (statsStatus === "pending") { + result.push({ index: key, [PENDING_STAT_ROW_KEY]: true }); + continue; + } } result.push(found); } @@ -340,6 +361,9 @@ export function getCellRendererSelector(pinned_rows: PinnedRowConfig[], column_c if (pk === undefined) { return anyRenderer; // default renderer } + if (_.get(params.node.data, PENDING_STAT_ROW_KEY) === true && params.column?.getColId() !== "index") { + return pendingStatRenderer; // a stat that has not arrived: leave the cell empty + } const maybePrc: PinnedRowConfig | undefined = _.find( pinned_rows, (cfg) => stripOptionalPinnedKey(cfg.primary_key_val) === pk, diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index e7f63015c..d15baac8e 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -1,11 +1,29 @@ +// Where the summary stats stand, as the server reports it in df_meta.stats. +// "pending": a stats_update is expected. "not_computed": none will be sent +// unless the user asks. A missing df_meta.stats means "complete", which is +// what servers that predate the field send. +export type StatsStatus = "complete" | "pending" | "not_computed" | "error"; + +export interface DFMetaStats { + status: StatsStatus; + tier?: string; + reason?: string; + gen?: number; +} + export interface DFMeta { // static, total_rows: number; columns: number; filtered_rows: number; rows_shown: number; + // Absent when the server predates the two-message protocol. + stats?: DFMetaStats; } +export const getStatsStatus = (meta: DFMeta | undefined): StatsStatus => + meta?.stats?.status ?? "complete"; + export interface BuckarooOptions { sampled: string[]; cleaning_method: string[]; diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 21e422a46..f26c4860d 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -18,6 +18,7 @@ import { BuckarooStaticTable } from './components/BuckarooStaticTable'; import { BuckarooServerView, buckarooWsUrl } from './server/BuckarooServerView'; import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; +import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -60,6 +61,7 @@ export default { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, }; // Named exports for direct imports @@ -89,6 +91,7 @@ export { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 3ede8298e..9c86f5bee 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -1,15 +1,15 @@ import * as React from "react"; import { BuckarooInfiniteWidget, DFViewerInfiniteDS, getKeySmartRowCache } from "../components/BuckarooWidgetInfinite"; -import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { DFMeta, BuckarooState, BuckarooOptions } from "../components/WidgetTypes"; import { CommandConfigT } from "../components/CommandUtils"; import { Operation } from "../components/OperationUtils"; import { OperationResult, baseOperationResults } from "../components/DependentTabs"; -import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFData } from "../components/DFViewerParts/DFWhole"; import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; +import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -161,6 +161,22 @@ export function BuckarooView({ const onMetadataRef = React.useRef(onMetadata); React.useEffect(() => { onMetadataRef.current = onMetadata; }, [onMetadata]); + // Every df_data_dict that reaches the view (the seed, change:df_data_dict, + // the metadata handler, the catch-up on subscribe) goes through one + // decoder, so a frame decodes once and the newest decode wins. The seed is + // already applied when it needs no resolution. + const dictDecoderRef = React.useRef<((raw: RawDFDataDict) => void) | null>(null); + if (dictDecoderRef.current === null) { + dictDecoderRef.current = makeLatestDictDecoder( + (d) => { + setDfDataDict(d as Record); + setDataReady(true); + }, + initialNeedsResolution ? undefined : (initialState.df_data_dict as RawDFDataDict), + ); + } + const loadDfDataDict = dictDecoderRef.current; + // Resolve any parquet-encoded payloads in df_data_dict. Pre-resolved // dicts (e.g. when BuckarooServerView already ran decodeDFDataDict) // pass through unchanged, so this is cheap in the common case. Skip @@ -168,19 +184,13 @@ export function BuckarooView({ // for the BuckarooServerView path. React.useEffect(() => { if (!initialNeedsResolution) return; - let cancelled = false; - const dict = initialState.df_data_dict as Record | undefined; + const dict = initialState.df_data_dict as RawDFDataDict; if (!dict) { setDataReady(true); return; } - decodeDFDataDict(dict).then((d) => { - if (cancelled) return; - setDfDataDict(d as Record); - setDataReady(true); - }); - return () => { cancelled = true; }; - }, [initialState, initialNeedsResolution]); + loadDfDataDict(dict); + }, [initialState, initialNeedsResolution, loadDfDataDict]); // Fire onMetadata for the initial payload, matching BuckarooServerView's // pre-split behavior. @@ -202,8 +212,7 @@ export function BuckarooView({ const onMeta = (metadata: BuckarooServerMetadata, prompt?: string) => { onMetadataRef.current?.(metadata, prompt); setDfMeta((model.get("df_meta") as DFMeta | undefined) ?? { ...DEFAULT_DF_META, total_rows: metadata?.rows ?? 0 }); - decodeDFDataDict((model.get("df_data_dict") as Record | undefined) ?? {}) - .then((d) => setDfDataDict(d as Record)); + loadDfDataDict(model.get("df_data_dict") as RawDFDataDict); setDfDisplayArgs((model.get("df_display_args") as Record | undefined) ?? {}); setBuckarooStateLocal((model.get("buckaroo_state") as BuckarooState | undefined) ?? DEFAULT_BUCKAROO_STATE); setBuckarooOptions((model.get("buckaroo_options") as BuckarooOptions | undefined) ?? DEFAULT_BUCKAROO_OPTIONS); @@ -212,9 +221,7 @@ export function BuckarooView({ setOperations((model.get("operations") as Operation[] | undefined) ?? []); }; const onDfMeta = (v: DFMeta) => setDfMeta(v); - const onDfDataDict = (v: Record) => { - decodeDFDataDict(v).then((d) => setDfDataDict(d as Record)); - }; + const onDfDataDict = (v: RawDFDataDict) => loadDfDataDict(v); const onDfDisplayArgs = (v: Record) => setDfDisplayArgs(v); const onBState = (v: BuckarooState) => setBuckarooStateLocal(v); const onBOpts = (v: BuckarooOptions) => setBuckarooOptions(v); @@ -232,6 +239,25 @@ export function BuckarooView({ model.on("change:operation_results", onOpRes); model.on("change:operations", onOps); + // A change:* emitted after the model was built and before this effect + // ran had no listener. The model holds the latest value of every key, + // so read each one now that the handlers are in place. A value that is + // the one already held is a no-op (same reference). + const catchUp: Array<[string, (v: any) => void]> = [ + ["df_meta", onDfMeta], + ["df_data_dict", onDfDataDict], + ["df_display_args", onDfDisplayArgs], + ["buckaroo_state", onBState], + ["buckaroo_options", onBOpts], + ["command_config", onCmdCfg], + ["operation_results", onOpRes], + ["operations", onOps], + ]; + for (const [key, apply] of catchUp) { + const v = model.get(key); + if (v !== undefined) apply(v); + } + return () => { model.off("metadata", onMeta); model.off("change:df_meta", onDfMeta); @@ -243,7 +269,7 @@ export function BuckarooView({ model.off("change:operation_results", onOpRes); model.off("change:operations", onOps); }; - }, [model]); + }, [model, loadDfDataDict]); const onBuckarooState = React.useCallback>>((newState) => { const resolved = typeof newState === "function" diff --git a/packages/buckaroo-js-core/src/server/latestDictDecoder.ts b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts new file mode 100644 index 000000000..16c4c3ef9 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts @@ -0,0 +1,34 @@ +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; + +export type RawDFDataDict = Record | undefined | null; + +/** + * Decode df_data_dict values arriving from a model and hand `apply` only the + * newest result. + * + * A full `initial_state` reaches a view twice, as `change:df_data_dict` and + * then as `metadata`, and both handlers read the same dict off the model. The + * returned function ignores a dict it has already seen (same reference), so + * that frame decodes once. Decoding is async, so a slow decode of an earlier + * dict can finish after a later one; each call takes a token, and a result is + * applied only if no later call has started. + * + * `seen` is a dict the caller has already applied (the view's seed), so the + * first read of that same dict off the model does not decode it again. + */ +export function makeLatestDictDecoder( + apply: (decoded: Record) => void, + seen?: RawDFDataDict, +): (raw: RawDFDataDict) => void { + let latest = 0; + let lastRaw: RawDFDataDict = seen; + return (raw) => { + if (raw === lastRaw) return; + lastRaw = raw; + const mine = ++latest; + decodeDFDataDict(raw).then((decoded) => { + if (mine === latest) apply(decoded); + }); + }; +} diff --git a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx index caa5628c1..22e3e0c44 100644 --- a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx @@ -9,9 +9,8 @@ * - "complete" stats are present; also the meaning of a missing * `df_meta.stats`, as servers without the field send * - * Column `a` is color-mapped and has a simple tooltip, so the story also shows - * that cells restyle when the histogram bins arrive and that hovering a - * valueless pinned cell does nothing. Used by stats-pending-pinned-rows.spec.ts. + * Column `a` is color-mapped, so the story also shows that its cells restyle + * when the histogram bins arrive. Used by stats-pending-pinned-rows.spec.ts. */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; @@ -45,7 +44,6 @@ const viewerConfig: DFViewerConfig = { header_name: "a", displayer_args: { displayer: "obj" }, color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, - tooltip_config: { tooltip_type: "simple", val_column: "a" }, }, { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, ], diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index 715312832..37f191c31 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -75,28 +75,57 @@ function updateFilenameDisplay(metadata: any, prompt?: string) { } } +/** What the model held for `keys` when a component first rendered. */ +function snapshotModel(model: WebSocketModel, keys: string[]): Record { + return Object.fromEntries(keys.map((k) => [k, model.get(k)])); +} + +/** A change:* emitted between the first render and the effect that registers + * the listeners had no listener. The model holds the latest value of every + * key, so apply each one that differs from what the first render read. */ +function catchUpOnModel( + model: WebSocketModel, + seed: Record, + appliers: Record void>, +) { + for (const [key, apply] of Object.entries(appliers)) { + const v = model.get(key); + if (v !== undefined && v !== seed[key]) apply(v); + } +} + function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { const [dfMeta, setDfMeta] = React.useState(model.get("df_meta") || { total_rows: 0 }); const [dfDataDict, setDfDataDict] = React.useState(model.get("df_data_dict") || {}); const [dfDisplayArgs, setDfDisplayArgs] = React.useState(patchDisplayArgsHeight(model.get("df_display_args") || {})); + const seed = React.useRef(snapshotModel(model, ["df_meta", "df_data_dict", "df_display_args"])); + // A full frame reaches the view as change:df_data_dict and again as + // metadata; this decodes each dict once and applies only the newest decode. + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: metadata.rows || 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); }; model.on("metadata", onMeta); const onDfMeta = (v: any) => setDfMeta(v); - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); model.on("change:df_meta", onDfMeta); model.on("change:df_data_dict", onDfDataDict); model.on("change:df_display_args", onDfDisplayArgs); + catchUpOnModel(model, seed.current, { + df_meta: onDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + }); // Catch up on metadata that arrived before useEffect registered listeners const existingMeta = model.get("metadata"); @@ -110,7 +139,7 @@ function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { model.off("change:df_data_dict", onDfDataDict); model.off("change:df_display_args", onDfDisplayArgs); }; - }, [model]); + }, [model, loadDict]); if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
@@ -140,12 +169,20 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { const [commandConfig, setCommandConfig] = React.useState(model.get("command_config") || {}); const [operationResults, setOperationResults] = React.useState(model.get("operation_results") || {}); const [operations, setOperations] = React.useState(model.get("operations") || []); + const seed = React.useRef(snapshotModel(model, [ + "df_meta", "df_data_dict", "df_display_args", "buckaroo_state", "buckaroo_options", + "command_config", "operation_results", "operations", + ])); + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); setBuckarooState(model.get("buckaroo_state") || {}); setBuckarooOptions(model.get("buckaroo_options") || {}); @@ -168,9 +205,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { }; // df_data_dict needs async pre-resolution of parquet_b64 values - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); model.on("change:df_data_dict", onDfDataDict); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); @@ -185,6 +220,16 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { ["operation_results", onChange("operation_results", setOperationResults)], ["operations", onChange("operations", setOperations)], ]; + catchUpOnModel(model, seed.current, { + df_meta: setDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + buckaroo_state: setBuckarooState, + buckaroo_options: setBuckarooOptions, + command_config: setCommandConfig, + operation_results: setOperationResults, + operations: setOperations, + }); return () => { model.off("metadata", onMeta); @@ -193,7 +238,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.off(`change:${key}`, handler); } }; - }, [model]); + }, [model, loadDict]); const onBuckarooState = React.useCallback((newState: any) => { // newState may be a value or a setter function From fe046acc7907af09754aa19eb7b5f91410c68459 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:22:28 -0400 Subject: [PATCH 03/18] test(client): failing tests for the stats_update channel (rows-first c2) Jest tests, driven through a WebSocketModel with a fake socket, for the client half of the stats wire: a stats_update is key-merged into all_stats and dropped when its stats_gen is not the expected one, the expected gen follows df_meta.stats.gen on every applied initial_state, a merge in flight is discarded or redone when a frame replaces the dict, and stats_aborted moves the status. Plus withStatsCapability and BuckarooServerView putting ?caps=stats_update on the WebSocket URL, and a server Playwright test that the standalone page does the same. StatsChannel.ts is a stub (identity withStatsCapability) so the tests fail on assertions rather than on a missing module. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 16 + .../server/BuckarooServerView.caps.test.tsx | 84 ++++++ .../src/server/StatsChannel.test.ts | 278 ++++++++++++++++++ .../src/server/StatsChannel.ts | 6 + 4 files changed, 384 insertions(+) create mode 100644 packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx create mode 100644 packages/buckaroo-js-core/src/server/StatsChannel.test.ts create mode 100644 packages/buckaroo-js-core/src/server/StatsChannel.ts diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index ad9634a05..3327d908a 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -485,6 +485,22 @@ test.describe('WebSocket data flow', () => { expect(await getCellText(page, COL.name, 0)).toBe('Alice'); }); + test('standalone page advertises stats_update on its WebSocket URL', async ({ page, request }) => { + const session = `ws-caps-${Date.now()}`; + await loadSession(request, session, csvPath); + + // The server reads capabilities from ?caps= at open, before the client + // sends anything, so the page has to put them on the URL it connects to. + const socketUrls: string[] = []; + page.on('websocket', (socket) => socketUrls.push(socket.url())); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + const wsUrls = socketUrls.filter((u) => u.includes(`/ws/${session}`)); + expect(wsUrls).toHaveLength(1); + expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update'); + }); + test('WebSocket receives data for scrolled rows', async ({ page, request }) => { // Create a larger dataset (100 rows) to force infinite scrolling const rows = []; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx new file mode 100644 index 000000000..21e8b8421 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx @@ -0,0 +1,84 @@ +/** + * BuckarooServerView — capability advertisement (rows-first c2). + * + * The server records a client's capabilities from `?caps=` on the WebSocket + * URL, because it sends the first message before the client says anything. A + * client that merges `stats_update` must put it there, whatever URL the host + * passes in. + */ +import { render, cleanup, waitFor } from "@testing-library/react"; +import { BuckarooServerView } from "./BuckarooServerView"; + +const capturedViewProps: any[] = []; + +jest.mock("./BuckarooView", () => ({ + BuckarooView: (props: any) => { + capturedViewProps.push(props); + return
; + }, + pickMode: (m: unknown) => (m === "buckaroo" ? "buckaroo" : "viewer"), +})); + +jest.mock("./WebSocketModel", () => ({ + WebSocketModel: class { constructor(_ws: any, _state: any) {} }, +})); + +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + binaryType = "arraybuffer"; + onopen: (() => void) | null = null; + onerror: ((e: any) => void) | null = null; + private listeners: Record void>> = {}; + constructor(public url: string) { + FakeWebSocket.instances.push(this); + setTimeout(() => { + this.onopen?.(); + setTimeout(() => { + const initial = { + type: "initial_state", + df_meta: { total_rows: 4, columns: 2, filtered_rows: 4, rows_shown: 4 }, + df_data_dict: {}, + df_display_args: {}, + mode: "viewer", + }; + this.listeners["message"]?.forEach((h) => h({ data: JSON.stringify(initial) } as any)); + }, 0); + }, 0); + } + addEventListener(ev: string, h: (e: any) => void) { + (this.listeners[ev] ??= new Set()).add(h); + } + removeEventListener(ev: string, h: (e: any) => void) { + this.listeners[ev]?.delete(h); + } + close() {} +} + +const origWebSocket = (globalThis as any).WebSocket; + +beforeAll(() => { + (globalThis as any).WebSocket = FakeWebSocket; +}); +afterAll(() => { + (globalThis as any).WebSocket = origWebSocket; +}); +afterEach(() => { + capturedViewProps.length = 0; + FakeWebSocket.instances.length = 0; + cleanup(); +}); + +describe("BuckarooServerView advertises stats_update", () => { + it("opens the socket with ?caps=stats_update", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?caps=stats_update"); + }); + + it("keeps the query string the host passed", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?token=abc&caps=stats_update"); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts new file mode 100644 index 000000000..70a8f7c8d --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -0,0 +1,278 @@ +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A capable client receives a stats-free `initial_state` (df_meta.stats.status + * "pending"), asks for the stats and merges the `stats_update` that answers, + * keyed by `stats_gen`. These tests drive a WebSocketModel with a fake socket, + * the way the server's frames reach it. + */ +import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; + +// The real decoder, except that an envelope carrying `hold` waits on `gate`, +// so a test can deliver a frame while a decode is in flight. +let gate: Promise = Promise.resolve(); +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { + ...actual, + decodeDFData: jest.fn(async (env: any, buffers?: DataView[]) => { + if (env && env.hold) await gate; + return actual.decodeDFData(env, buffers); + }), + }; +}); + +class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } +} + +// Lets every promise continuation (the payload decode) run. +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +const row = (stat: string, cells: Record) => ({ index: stat, level_0: stat, ...cells }); + +// What the schema tier ships: identity rows only. +const schemaStats = () => [ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 3, c: 3 }), +]; + +const metaFor = (stats?: Record) => ({ + total_rows: 3, columns: 3, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); + +const frame = (gen: number | undefined, allStats: any = schemaStats()) => ({ + type: "initial_state", + df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, +}); + +// The server's payload is a wide DFEnvelope; the json format decodes to the +// same row shape without a parquet fixture. +const update = (gen: number, rows: any[], extra: object = {}) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final: true, + payload: { format: "json", layout: "wide", data: rows }, + elapsed_ms: 12.5, + ...extra, +}); + +function makeModel(gen: number | undefined = 3, allStats: any = schemaStats()) { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, + }); + const events: { event: string; value: any }[] = []; + for (const key of ["df_data_dict", "df_meta"]) { + model.on(`change:${key}`, (value: any) => events.push({ event: `change:${key}`, value })); + } + return { ws, model, events }; +} + +describe("withStatsCapability", () => { + it("adds ?caps=stats_update to a bare URL", () => { + expect(withStatsCapability("ws://localhost:8700/ws/sales")).toBe("ws://localhost:8700/ws/sales?caps=stats_update"); + }); + + it("appends to an existing query string", () => { + expect(withStatsCapability("ws://h/ws/s?token=abc")).toBe("ws://h/ws/s?token=abc&caps=stats_update"); + }); + + it("extends an existing caps value", () => { + expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update"); + }); + + it("keeps the fragment last", () => { + expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update#frag"); + }); +}); + +describe("stats_update merge", () => { + it("key-merges the payload's columns into all_stats and keeps the other columns", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [ + row("length", { a: 3, b: 5 }), + row("mean", { a: 2, b: 4.5 }), + ])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 5, c: 3 }), + row("mean", { a: 2, b: 4.5 }), + ]); + }); + + it("does not let a null in the payload erase a value already merged", async () => { + const { ws, model } = makeModel(3); + // The wide pivot pads a stat a column did not carry with null. + ws.deliver(update(3, [row("dtype", { a: null, b: "float32" })])); + await settle(); + const dtype = model.get("df_data_dict").all_stats.find((r: any) => r.index === "dtype"); + expect(dtype).toEqual(row("dtype", { a: "int64", b: "float32", c: "object" })); + }); + + it("assigns a new df_data_dict and leaves the previous objects untouched", async () => { + const { ws, model, events } = makeModel(3); + const before = model.get("df_data_dict"); + const beforeStats = before.all_stats; + const snapshot = JSON.parse(JSON.stringify(beforeStats)); + ws.deliver(update(3, [row("length", { a: 99 }), row("mean", { a: 2 })])); + await settle(); + const after = model.get("df_data_dict"); + expect(after).not.toBe(before); + expect(after.all_stats).not.toBe(beforeStats); + expect(beforeStats).toEqual(snapshot); + const dictEvents = events.filter((e) => e.event === "change:df_data_dict"); + expect(dictEvents).toHaveLength(1); + expect(dictEvents[0].value).toBe(after); + }); + + it("merges onto an all_stats that arrived as an undecoded envelope", async () => { + const { ws, model } = makeModel(3); + // A later initial_state hands the model the dict as the server sent it. + ws.deliver(frame(4, { format: "json", layout: "wide", data: schemaStats() })); + ws.deliver(update(4, [row("mean", { a: 2, b: 4.5, c: null })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(Array.isArray(stats)).toBe(true); + expect(stats.map((r: any) => r.index)).toEqual(["dtype", "length", "mean"]); + }); + + it("keeps the other df_data_dict keys", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { all_stats: schemaStats(), empty: [], main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const dict = model.get("df_data_dict"); + expect(dict.empty).toEqual([]); + expect(dict.main).toEqual([{ a: 1 }]); + expect(dict.all_stats).toHaveLength(3); + }); + + it("applies updates that arrive back to back, in order", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + ws.deliver(update(3, [row("mean", { b: 4.5 }), row("max", { a: 9 })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats.find((r: any) => r.index === "mean")).toEqual(row("mean", { a: 2, b: 4.5 })); + expect(stats.find((r: any) => r.index === "max")).toEqual(row("max", { a: 9 })); + }); +}); + +describe("stats_update and df_meta.stats", () => { + it("a final update marks the stats complete at the update's tier", async () => { + const { ws, model, events } = makeModel(3); + const metaBefore = model.get("df_meta"); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const meta = model.get("df_meta"); + expect(meta).not.toBe(metaBefore); + expect(meta.stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + expect(meta.total_rows).toBe(3); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(1); + }); + + it("a non-final update merges but leaves the status pending", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + expect(model.get("df_meta").stats.status).toBe("pending"); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(0); + }); +}); + +describe("stats_gen", () => { + it("drops a stats_update whose stats_gen is not the expected one", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(2, [row("mean", { a: 2 })])); + ws.deliver(update(3, [row("max", { a: 9 })])); + await settle(); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "max"]); + expect(events.filter((e) => e.event === "change:df_data_dict")).toHaveLength(1); + }); + + it("advances the expected gen on a broadcast initial_state with no reply_seq", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(4)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + }); + + it("discards a merge when an initial_state for a newer gen arrives while it decodes", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })])); + // Still decoding: the model has not merged anything yet. + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" }), row("mean", { a: 2 })]); + }); + + it("merges onto the new dict when a same-gen initial_state replaces it while the old one decodes", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const held = { format: "json", layout: "wide", data: schemaStats(), hold: true }; + const { ws, model } = makeModel(3, held); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the update is now waiting on the held decode + expect((decodeDFData as jest.Mock).mock.calls.some(([env]) => env === held)).toBe(true); + ws.deliver(frame(3, [row("dtype", { a: "int32" })])); + release(); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int32" }), + row("mean", { a: 2 }), + ]); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + +describe("stats_aborted", () => { + it("marks the stats failed when the run for the expected gen failed", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" }); + }); + + it("marks the stats not computed when the server says they cannot be requested", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("not_computed"); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts new file mode 100644 index 000000000..2ef586390 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -0,0 +1,6 @@ +// Stub: the implementation lands in the next commit. +export const STATS_UPDATE_CAP = "stats_update"; + +export function withStatsCapability(wsUrl: string): string { + return wsUrl; +} From 3118435aa40753e293f954cd5a00fbad02d905c2 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:28:27 -0400 Subject: [PATCH 04/18] test(client): failing tests for stats_update payloads and an empty dict (rows-first c2) Three more cases for the stats_update merge, found untested after the first test commit: a wide parquet_b64 payload as the server sends it (the shared summary_stats fixture, not a json envelope), a model that holds no df_data_dict yet, and a dict with no all_stats key. Co-Authored-By: Claude Sonnet 5.5 --- .../src/server/StatsChannel.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index 70a8f7c8d..ad675f03c 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -10,6 +10,11 @@ import { WebSocketModel } from "./WebSocketModel"; import { withStatsCapability } from "./StatsChannel"; import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; +// A wide summary-stats envelope (parquet_b64, layout "wide") as the server +// sends one; the decoder tests use the same fixture. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const wideFixture = require("../components/DFViewerParts/test-fixtures/summary_stats_parquet_b64.json"); + // The real decoder, except that an envelope carrying `hold` waits on `gate`, // so a test can deliver a frame while a decode is in flight. let gate: Promise = Promise.resolve(); @@ -169,6 +174,38 @@ describe("stats_update merge", () => { expect(dict.all_stats).toHaveLength(3); }); + it("merges a wide parquet_b64 payload as the server sends it", async () => { + const { ws, model } = makeModel(3, [row("orig_col_name", { a: "first" })]); + const decoded: any[] = await decodeDFData(wideFixture); + expect(decoded.length).toBeGreaterThan(1); + ws.deliver(update(3, [], { payload: wideFixture })); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats[0]).toEqual(row("orig_col_name", { a: "first" })); + for (const decodedRow of decoded) { + expect(stats).toContainEqual(decodedRow); + } + }); + + it("builds all_stats when the model holds no dict yet", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { df_meta: metaFor(pending(3)) }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ all_stats: [row("mean", { a: 2 })] }); + }); + + it("adds all_stats to a dict that has none", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ main: [{ a: 1 }], all_stats: [row("mean", { a: 2 })] }); + }); + it("applies updates that arrive back to back, in order", async () => { const { ws, model } = makeModel(3); ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); From 1fd9cff8159741f99a8dace40142394f9ce66728 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:32:11 -0400 Subject: [PATCH 05/18] test(client): failing test for stats_update ordering (rows-first c2) Two stats_update messages for one gen are applied in arrival order even when the first payload decodes more slowly: the final update merges last and the status completes only after both merges. Co-Authored-By: Claude Sonnet 5.5 --- .../src/server/StatsChannel.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index ad675f03c..a6b61e880 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -217,6 +217,30 @@ describe("stats_update merge", () => { }); }); +describe("stats_update order", () => { + it("merges updates in arrival order when an earlier payload decodes slower", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const { ws, model, events } = makeModel(3); + const held = { format: "json", layout: "wide", data: [row("mean", { a: 1 })], hold: true }; + ws.deliver(update(3, [], { final: false, payload: held })); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the second payload has decoded; the first is held + release(); + await settle(); + // The final update goes last: its value stands and the status + // completes only after both merges. + const mean = model.get("df_data_dict").all_stats.find((r: any) => r.index === "mean"); + expect(mean.a).toBe(2); + expect(events.map((e) => e.event)).toEqual(["change:df_data_dict", "change:df_data_dict", "change:df_meta"]); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + describe("stats_update and df_meta.stats", () => { it("a final update marks the stats complete at the update's tier", async () => { const { ws, model, events } = makeModel(3); From 5935432d6926ec10409c90449829742cccc50f9e Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 02:38:28 -0400 Subject: [PATCH 06/18] feat(client): merge stats_update into all_stats and advertise the capability (rows-first c2) StatsChannel is the client half of the stats wire. WebSocketModel hands it every text frame first. A stats_update for the gen the model's df_meta reports is decoded and key-merged into df_data_dict.all_stats (new row objects, a new dict), updates apply one at a time in arrival order, and a final update sets df_meta.stats to complete at the update's tier. A merge is dropped when the gen moves on while it decodes and redone when a frame replaces the dict. stats_aborted for the expected gen sets error or not_computed. Other message types are ignored as before. withStatsCapability adds caps=stats_update to a WebSocket URL; both wiring copies (BuckarooServerView and the standalone page) use it so the server treats them as capable clients. The guard tests that already pass before the fix (caps already present, no df_meta.stats, ignored aborts, unknown types, infinite_resp pairing) are added here, with a makeModel(null) helper so a model without stats can be built. Co-Authored-By: Claude Sonnet 5.5 --- packages/buckaroo-js-core/src/index.ts | 3 + .../src/server/BuckarooServerView.tsx | 4 +- .../src/server/StatsChannel.test.ts | 66 +++++- .../src/server/StatsChannel.ts | 208 +++++++++++++++++- .../src/server/WebSocketModel.ts | 7 + packages/js/standalone.tsx | 3 +- 6 files changed, 285 insertions(+), 6 deletions(-) diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index f26c4860d..11d10f022 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -19,6 +19,7 @@ import { BuckarooServerView, buckarooWsUrl } from './server/BuckarooServerView'; import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; +import { withStatsCapability } from './server/StatsChannel'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -62,6 +63,7 @@ export default { buckarooWsUrl, WebSocketModel, makeLatestDictDecoder, + withStatsCapability, }; // Named exports for direct imports @@ -92,6 +94,7 @@ export { buckarooWsUrl, WebSocketModel, makeLatestDictDecoder, + withStatsCapability, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx index 49fedbe3a..1687dddaa 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx @@ -2,6 +2,7 @@ import * as React from "react"; import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; import { BuckarooView, BuckarooServerMetadata, @@ -106,7 +107,8 @@ export function BuckarooServerView({ (async () => { try { - ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + ws = new WebSocket(withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; await new Promise((resolve, reject) => { diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index a6b61e880..9d2caa839 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -78,10 +78,11 @@ const update = (gen: number, rows: any[], extra: object = {}) => ({ ...extra, }); -function makeModel(gen: number | undefined = 3, allStats: any = schemaStats()) { +// `gen` null builds a model whose df_meta carries no stats, as an old server's does. +function makeModel(gen: number | null = 3, allStats: any = schemaStats()) { const ws = new FakeSocket(); const model = new WebSocketModel(ws as unknown as WebSocket, { - df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_meta: metaFor(gen === null ? undefined : pending(gen)), df_data_dict: { all_stats: allStats }, }); const events: { event: string; value: any }[] = []; @@ -104,6 +105,11 @@ describe("withStatsCapability", () => { expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update"); }); + it("leaves a URL that already advertises the capability alone", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update")).toBe("ws://h/ws/s?caps=stats_update"); + expect(withStatsCapability("ws://h/ws/s?caps=a,stats_update")).toBe("ws://h/ws/s?caps=a,stats_update"); + }); + it("keeps the fragment last", () => { expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update#frag"); }); @@ -274,6 +280,22 @@ describe("stats_gen", () => { expect(events.filter((e) => e.event === "change:df_data_dict")).toHaveLength(1); }); + it("drops every stats_update when the server reported no stats", async () => { + const { ws, model, events } = makeModel(null); + ws.deliver(update(0, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + expect(events).toHaveLength(0); + }); + + it("clears the expectation when an initial_state carries no df_meta.stats", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(undefined)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + }); + it("advances the expected gen on a broadcast initial_state with no reply_seq", async () => { const { ws, model } = makeModel(3); ws.deliver(frame(4)); @@ -336,4 +358,44 @@ describe("stats_aborted", () => { await settle(); expect(model.get("df_meta").stats.status).toBe("not_computed"); }); + + it("changes nothing for a stale reply, an unsupported scope or a session with no data", async () => { + const { ws, events } = makeModel(3); + for (const reason of ["stale", "unsupported_scope", "no_data"]) { + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 4, scope: "raw", reason }); + } + await settle(); + expect(events).toHaveLength(0); + }); + + it("ignores a reply for a gen the client has left", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 2, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("pending"); + }); +}); + +describe("other messages", () => { + it("ignores a type it does not know and keeps applying frames", async () => { + const { ws, model, events } = makeModel(3); + expect(() => ws.deliver({ type: "something_new", stats_gen: 3 })).not.toThrow(); + ws.deliver({ type: "error", message: "boom" }); + await settle(); + expect(events).toHaveLength(0); + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + }); + + it("still pairs an infinite_resp with the binary frame that follows it", () => { + const { ws, model } = makeModel(3); + const seen: [any, DataView[]][] = []; + model.on("msg:custom", (msg: any, buffers: DataView[]) => seen.push([msg, buffers])); + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 5 }, length: 5 }); + ws.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + expect(seen).toHaveLength(1); + expect(seen[0][0].type).toBe("infinite_resp"); + expect(seen[0][1][0].byteLength).toBe(8); + }); }); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts index 2ef586390..0ead005ed 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -1,6 +1,210 @@ -// Stub: the implementation lands in the next commit. +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A client that advertises `?caps=stats_update` gets a first `initial_state` + * whose `df_meta.stats` says the stats are pending, asks for them with + * `stats_request {stats_gen, scope}`, and receives either + * + * stats_update {stats_gen, scope, tier, final, payload, elapsed_ms} + * stats_aborted {stats_gen, current_gen?, scope, reason} + * + * `payload` is an inline wide DFEnvelope holding `all_stats`. `stats_gen` is the + * server's counter for the state the stats describe; it rides on every + * `initial_state` as `df_meta.stats.gen`, and a reply for any other gen is for + * a state the client has left. Sending the request is the scheduler's job, not + * this module's. + * + * Merge semantics are WebSocket-only: `WebSocketModel` hands every frame to + * `handle()` first, and Jupyter's widget sets `df_data_dict` whole. + */ +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFMeta, DFMetaStats } from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +/** The capability this client advertises, as one value of `?caps=` on the + * WebSocket URL: it merges `stats_update` messages. The server records it per + * connection when the socket opens, since it sends the first message before + * the client can say anything. */ export const STATS_UPDATE_CAP = "stats_update"; +const decodeQueryValue = (value: string): string => { + try { + return decodeURIComponent(value); + } catch { + return value; + } +}; + +/** `wsUrl` with `caps=stats_update` added: a new query on a bare URL, a new + * parameter after an existing query, or a comma-joined value when the host + * already passes `caps`. The fragment stays last and other parameters are + * left as the host wrote them. */ export function withStatsCapability(wsUrl: string): string { - return wsUrl; + const hashAt = wsUrl.indexOf("#"); + const fragment = hashAt === -1 ? "" : wsUrl.slice(hashAt); + const beforeFragment = hashAt === -1 ? wsUrl : wsUrl.slice(0, hashAt); + const queryAt = beforeFragment.indexOf("?"); + const path = queryAt === -1 ? beforeFragment : beforeFragment.slice(0, queryAt); + const params = queryAt === -1 ? [] : beforeFragment.slice(queryAt + 1).split("&").filter((p) => p !== ""); + + const capsAt = params.findIndex((p) => p === "caps" || p.startsWith("caps=")); + if (capsAt === -1) { + params.push(`caps=${STATS_UPDATE_CAP}`); + } else { + const caps = decodeQueryValue(params[capsAt].slice("caps=".length)) + .split(",") + .map((cap) => cap.trim()) + .filter((cap) => cap !== ""); + if (!caps.includes(STATS_UPDATE_CAP)) caps.push(STATS_UPDATE_CAP); + params[capsAt] = `caps=${caps.join(",")}`; + } + return `${path}?${params.join("&")}${fragment}`; +} + +export interface StatsUpdateMessage { + type: "stats_update"; + stats_gen: number; + scope?: string; + tier?: string; + final?: boolean; + payload?: DFDataOrPayload; + elapsed_ms?: number; +} + +export interface StatsAbortedMessage { + type: "stats_aborted"; + /** The request's gen. */ + stats_gen?: number; + /** The session's gen, omitted when the session has no data. */ + current_gen?: number; + scope?: string; + reason?: "stale" | "unsupported_scope" | "not_requestable" | "error" | "no_data"; +} + +/** + * Key-merge a stats payload into `all_stats`. Both are row-per-stat tables + * (`{index: , : , ...}`). For each stat row the payload + * names, every column it carries replaces that cell, columns it does not carry + * keep theirs, and a stat the table lacks is appended. + * + * A `null` in the payload never replaces a value already merged: the wide + * pivot fills a stat a column did not carry in this message with `null`. + * + * Returns new row objects and a new array. `base` may be the decoder's cached + * array, so nothing in it is touched. + */ +export function mergeStatRows(base: DFData, update: DFData): DFData { + const merged = base.slice(); + const at = new Map(); + merged.forEach((row, i) => at.set(row.index, i)); + + for (const updateRow of update) { + const i = at.get(updateRow.index); + if (i === undefined) { + at.set(updateRow.index, merged.length); + merged.push({ ...updateRow }); + continue; + } + const row = { ...merged[i] }; + for (const [column, value] of Object.entries(updateRow)) { + if (column === "index" || column === "level_0") continue; + if (value === null && row[column] != null) continue; + row[column] = value; + } + merged[i] = row; + } + return merged; +} + +const genOf = (meta: DFMeta | undefined): number | undefined => { + const gen = meta?.stats?.gen; + return typeof gen === "number" ? gen : undefined; +}; + +/** What the channel needs of a model. */ +type StatsModel = Pick; + +export class StatsChannel { + // Updates are applied one at a time: each reads the dict the previous one + // wrote, so a later update cannot overwrite an earlier one's merge. + private applying: Promise = Promise.resolve(); + + constructor(private model: StatsModel) {} + + /** + * The stats_gen of the state the client is showing, or `undefined` when the + * server reports none; a `stats_request` carries it. It is read off the + * model's `df_meta` each time, so it starts from the frame the model was + * built from and every applied `initial_state` (a broadcast frame with no + * `reply_seq` included) moves it. A `df_meta` with no `stats.gen` leaves + * nothing expected, so a server that stops reporting stats cannot have a + * late `stats_update` merged. Reading the model, not the message, keeps this + * correct under any scheme that drops stale `initial_state` frames. + */ + get expectedGen(): number | undefined { + return genOf(this.model.get("df_meta")); + } + + /** Consume a stats message. Returns false for every other message type, + * which the model handles (or ignores) as before. */ + handle(msg: { type?: string }): boolean { + if (msg.type === "stats_update") { + this.receiveUpdate(msg as StatsUpdateMessage); + return true; + } + if (msg.type === "stats_aborted") { + this.receiveAborted(msg as StatsAbortedMessage); + return true; + } + return false; + } + + private receiveUpdate(msg: StatsUpdateMessage): void { + if (msg.stats_gen !== this.expectedGen) return; + this.applying = this.applying + .then(() => this.applyUpdate(msg)) + .catch((e) => console.error("[StatsChannel] stats_update failed:", e)); + } + + private async applyUpdate(msg: StatsUpdateMessage): Promise { + const update = await decodeDFData(msg.payload); + for (;;) { + // A frame may have moved the gen on while something decoded. + if (msg.stats_gen !== this.expectedGen) return; + const dict: Record | null | undefined = this.model.get("df_data_dict"); + // The dict is decoded when the seed built it and raw when a later + // initial_state did. + const base = await decodeDFData(dict?.all_stats); + // A frame replaced the dict (or moved the gen) while it decoded: + // start over from the new one. + if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; + this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + if (msg.final) { + this.replaceStats((stats) => ({ status: "complete", tier: msg.tier ?? stats.tier, gen: stats.gen })); + } + return; + } + } + + private receiveAborted(msg: StatsAbortedMessage): void { + // Only a reply to a request for the state on screen says anything about + // it. A `stale` reply is answered by the initial_state that carries the + // new gen; taking `current_gen` without that frame would merge stats + // for a state the client has not seen into the one it shows. + if (msg.stats_gen !== this.expectedGen) return; + if (msg.reason === "error") { + this.replaceStats((stats) => ({ ...stats, status: "error", reason: "stats_failed" })); + } else if (msg.reason === "not_requestable") { + this.replaceStats((stats) => ({ ...stats, status: "not_computed" })); + } + } + + /** Replace `df_meta.stats` in a new `df_meta`, the reference that c0a's + * `inFlight` rule and the pinned rows' placeholders key on. */ + private replaceStats(change: (stats: DFMetaStats) => DFMetaStats): void { + const meta: DFMeta | undefined = this.model.get("df_meta"); + if (!meta?.stats) return; + this.model.set("df_meta", { ...meta, stats: change(meta.stats) }); + } } diff --git a/packages/buckaroo-js-core/src/server/WebSocketModel.ts b/packages/buckaroo-js-core/src/server/WebSocketModel.ts index 1e1fd4a2d..e7e5798d2 100644 --- a/packages/buckaroo-js-core/src/server/WebSocketModel.ts +++ b/packages/buckaroo-js-core/src/server/WebSocketModel.ts @@ -13,21 +13,28 @@ * Binary protocol (matching anywidget's msg + buffers pattern): * Server sends a JSON text frame (infinite_resp), then a binary frame (Parquet). * This class pairs them and emits "msg:custom" with (msg, [DataView]). + * + * stats_update and stats_aborted frames go to `stats` (see StatsChannel). */ +import { StatsChannel } from "./StatsChannel"; + export class WebSocketModel { private ws: WebSocket; private pendingMsg: any = null; private handlers: Map> = new Map(); private state: Record; private pendingChanges: Set = new Set(); + readonly stats: StatsChannel; constructor(ws: WebSocket, initialState: Record) { this.state = { ...initialState }; this.ws = ws; + this.stats = new StatsChannel(this); this.ws.onmessage = (event: MessageEvent) => { if (typeof event.data === "string") { const msg = JSON.parse(event.data); + if (this.stats.handle(msg)) return; if (msg.type === "infinite_resp") { // Expect a following binary frame — stash this JSON diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index 37f191c31..d7ccfe48f 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -287,7 +287,8 @@ async function main() { // Show connecting state rootEl.textContent = "Connecting..."; - const ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + const ws = new WebSocket(srt.withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; // Wait for connection From dd6ebe6a2b5f700779b67761d2c869178ed641a7 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:20:51 -0400 Subject: [PATCH 07/18] test(client): failing tests for the stats scheduler and its states (rows-first c4) Jest tests for the scheduler (StateOrchestrator rewritten around an IModel: a stats_request after the first infinite_resp, one request per reply until final, nothing after a state change, nothing for a search_string-only change), for the status bar's stats column and Compute summary stats control, for the widget and BuckarooView wiring of that control, and for pinned rows in the error state. Playwright tests on Storybook cover the four states without layout shift and the control's request, and the server spec runs the real standalone page against a session whose first frame says the stats are pending. StateOrchestrator.ts, StatusBar.tsx and BuckarooWidgetInfinite.tsx carry only the signatures the tests use, so the tests fail on assertions. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 89 +++ .../pw-tests/stats-scheduler-states.spec.ts | 96 +++ .../BuckarooInfiniteWidget.flash.test.tsx | 32 + .../src/components/BuckarooWidgetInfinite.tsx | 3 + .../DFViewerParts/gridUtils.test.ts | 12 + .../src/components/StatusBar.stats.test.tsx | 150 ++++ .../src/components/StatusBar.tsx | 9 +- .../src/server/BuckarooView.stats.test.tsx | 89 +++ .../src/server/StateOrchestrator.test.ts | 643 +++++++++++++----- .../src/server/StateOrchestrator.ts | 159 +---- .../stories/StatsSchedulerStates.stories.tsx | 170 +++++ scripts/test_playwright_storybook.sh | 1 + 12 files changed, 1165 insertions(+), 288 deletions(-) create mode 100644 packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts create mode 100644 packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx create mode 100644 packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx create mode 100644 packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index 3327d908a..bf98d7d19 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -501,6 +501,95 @@ test.describe('WebSocket data flow', () => { expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update'); }); + // Rows-first c4: the standalone page's scheduler. No session the server can + // build here defers its stats (that needs a xorq expression), so these + // tests put the pending state on a real session's first frame and answer + // the page's stats_request themselves. Rows, config and the stats payload + // all come from the real server. + async function loadBuckarooSession(request: any, sessionId: string) { + const resp = await request.post(`${BASE}/load`, { + data: { session: sessionId, path: csvPath, mode: 'buckaroo' }, + }); + expect(resp.ok()).toBe(true); + } + + test('a pending session: the page asks for stats after the first rows, merges the reply and shows them', async ({ page, request }) => { + const session = `ws-sched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const order: string[] = []; + const requests: any[] = []; + let realStats: unknown; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + server.onMessage((message) => { + if (typeof message !== 'string') { + ws.send(message); + return; + } + const msg = JSON.parse(message); + if (msg.type === 'infinite_resp') order.push('infinite_resp'); + if (msg.type === 'initial_state' && realStats === undefined) { + // The first frame as a deferring server sends it: no stats yet. + realStats = msg.df_data_dict.all_stats; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } }; + ws.send(JSON.stringify(msg)); + return; + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + order.push('stats_request'); + requests.push(msg); + ws.send(JSON.stringify({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: msg.scope, tier: 'full', + final: true, payload: realStats, elapsed_ms: 1, + })); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // The stats arrive: the status bar says so and the pinned dtype row has its values. + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); + + // One whole-run request, for the gen on the first frame, after rows had arrived. + expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw' }]); + expect(order.indexOf('infinite_resp')).toBeGreaterThanOrEqual(0); + expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); + }); + + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { + const session = `ws-nosched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const sentTypes: string[] = []; + page.on('websocket', (socket) => { + socket.on('framesent', (frame) => { + if (typeof frame.payload === 'string') sentTypes.push(JSON.parse(frame.payload).type); + }); + }); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.locator('.status-bar')).toBeVisible(); + // Longer than the scheduler would wait for rows that never come. + await page.waitForTimeout(3000); + + expect(sentTypes).toContain('infinite_request'); + expect(sentTypes).not.toContain('stats_request'); + await expect(page.getByTestId('stats-status')).toHaveCount(0); + }); + test('WebSocket receives data for scrolled rows', async ({ page, request }) => { // Create a larger dataset (100 rows) to force infinite scrolling const rows = []; diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts new file mode 100644 index 000000000..4470a9324 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -0,0 +1,96 @@ +/** + * Playwright test for the client states of the stats wire (rows-first c4). + * + * The StatsSchedulerStates story flips df_meta.stats.status. Each status has + * its own text in the status bar's stats column, the pinned area follows it as + * in c0a, and switching between them moves nothing: the status bar and the + * grid keep their boxes. In "not_computed" the status bar offers a Compute + * summary stats button, which sends stats_request {force: true}. + */ +import { test, expect, Page } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--primary&globals=&args="; + +const boxes = async (page: Page) => { + const round = (b: { x: number; y: number; width: number; height: number } | null) => + b && { x: Math.round(b.x), y: Math.round(b.y), width: Math.round(b.width), height: Math.round(b.height) }; + return { + statusBar: round(await page.locator(".status-bar").boundingBox()), + grid: round(await page.locator(".df-viewer").boundingBox()), + widget: round(await page.locator(".buckaroo-widget").boundingBox()), + }; +}; + +test("each stats status shows its own text in the status bar and nothing moves between them", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const statsCell = page.getByTestId("stats-status"); + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + + // pending: the loading text, and a placeholder for each pinned key. + await expect(statsCell).toHaveAttribute("data-stats-status", "pending"); + await expect(statsCell).toContainText("Computing summary stats"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + const reference = await boxes(page); + expect(reference.statusBar).not.toBeNull(); + expect(reference.grid).not.toBeNull(); + + // not_computed: the control, and no pinned rows. + await page.getByTestId("status-not_computed").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + expect(await boxes(page)).toEqual(reference); + + // error: the reason, and no pinned rows. + await page.getByTestId("status-error").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "error"); + await expect(statsCell).toContainText("Stats error: stats_failed"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + expect(await boxes(page)).toEqual(reference); + + // complete: the values are in the pinned area where the placeholders were. + await page.getByTestId("status-complete").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "complete"); + await expect(statsCell).toContainText("Summary stats ready"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + expect(await boxes(page)).toEqual(reference); + + expect(pageErrors).toEqual([]); +}); + +test("the Compute summary stats control sends stats_request with force", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + await page.getByTestId("status-not_computed").click(); + await page.getByRole("button", { name: "Compute summary stats" }).click(); + + const sent = page.getByTestId("sent-log"); + await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", force: true }, + ]); +}); + +test("no control is offered while the stats are pending, in error or complete", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + const control = page.getByRole("button", { name: "Compute summary stats" }); + for (const status of ["pending", "error", "complete"]) { + await page.getByTestId(`status-${status}`).click(); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", status); + await expect(control).toHaveCount(0); + } +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index 51a14cb41..e74a13b95 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -759,6 +759,21 @@ describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", expect(lastPinnedRows()).toEqual(stats); }); + it("error: omits the valueless keys, since no value is coming", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("error: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + it("no df_meta.stats (an older server) behaves as complete: valueless keys stay undefined rows", () => { const stats = [{ index: "dtype", a: "int64" }]; render(); @@ -815,6 +830,23 @@ describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", }); }); + describe("compute summary stats control (rows-first c4)", () => { + it("forwards on_compute_stats to the status bar", () => { + const onComputeStats = jest.fn(); + render( + , + ); + expect(mockStatusBarProps.onComputeStats).toBe(onComputeStats); + }); + + it("passes nothing on when the host gave no callback, so the status bar shows no control", () => { + render(); + expect(mockStatusBarProps.onComputeStats).toBeUndefined(); + }); + }); + describe("in-flight indicator", () => { const inFlightAttr = () => document.querySelector('[data-testid="status-bar-stub"]')!.getAttribute("data-inflight"); const dispatchSearch = () => diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index 07e3a7b50..ecb682153 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -148,6 +148,9 @@ export function BuckarooInfiniteWidget({ df_meta: DFMeta; df_data_dict: Record; df_display_args: Record; + /** Sends a forced stats_request. Server entry points pass it; the Jupyter + * widget does not, and then the status bar offers no control. */ + on_compute_stats?: () => void; operations: Operation[]; on_operations: (ops: Operation[]) => void; operation_results: OperationResult; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts index 944ceddbc..a32f4a7db 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts @@ -200,6 +200,18 @@ describe("testing utility functions in gridUtils ", () => { ]); }); + it("omits a required key with no value when the stats status is error, and keeps one that has a value", () => { + // An error is final for the state on screen, so no value is coming for + // the keys that are missing (rows-first c4). + const data: DFData = [{ index: "row1", value: 1 }]; + const pinnedConfig: PinnedRowConfig[] = [ + { primary_key_val: "row1", displayer_args: { displayer: "obj" } }, + { primary_key_val: "missing", displayer_args: { displayer: "obj" } } + ]; + expect(extractPinnedRows(data, pinnedConfig, "error")).toStrictEqual([{ index: "row1", value: 1 }]); + expect(extractPinnedRows([], pinnedConfig, "error")).toStrictEqual([]); + }); + it("includes an optional `?`-prefixed pinned row when the unprefixed key exists in data", () => { const data: DFData = [ { index: "histogram_bins", value: 1 }, diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx new file mode 100644 index 000000000..d4fe5c99e --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -0,0 +1,150 @@ +/** + * StatusBar — summary stats status (rows-first c4). + * + * A session that reports df_meta.stats gets one extra, fixed-width column in + * the status bar showing where the stats stand: loading ("pending"), not + * computed (with a control that asks for them), error (with the reason) or + * ready. A session that does not report df_meta.stats gets the status bar it + * always had. + * + * AG Grid is stubbed to capture the props the status bar hands it; the cell + * renderer is rendered on its own. + */ +import "@testing-library/jest-dom"; +import { render, screen, fireEvent } from "@testing-library/react"; + +// The props the status bar gave AG Grid on its last render. +const mockGrid: { props: any } = { props: null }; +jest.mock("ag-grid-react", () => ({ + AgGridReact: (props: any) => { + // React also calls this stub once with no props; keep the last real ones. + if (props) mockGrid.props = props; + return
; + }, +})); +jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); + +import { StatusBar, StatsStatusCell } from "./StatusBar"; +import { BuckarooOptions, BuckarooState, DFMeta, DFMetaStats } from "./WidgetTypes"; + +const baseMeta: DFMeta = { total_rows: 378, columns: 7, filtered_rows: 297, rows_shown: 297 }; +const options: BuckarooOptions = { + sampled: [], + cleaning_method: ["", "clean1"], + post_processing: ["", "post1"], + df_display: ["main", "summary"], + show_commands: ["0", "1"], +}; +const bState: BuckarooState = { + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, +}; + +const renderBar = (dfMeta: DFMeta, onComputeStats?: () => void) => + render( + {}} + buckarooOptions={options} + onComputeStats={onComputeStats} + />, + ); + +const fields = (): string[] => mockGrid.props.columnDefs.map((c: any) => c.field); + +describe("StatusBar stats column", () => { + beforeEach(() => { + mockGrid.props = null; + }); + + it("adds no column and no row field when df_meta has no stats (every session today)", () => { + renderBar(baseMeta); + expect(fields()).not.toContain("stats"); + expect(mockGrid.props.rowData[0]).not.toHaveProperty("stats"); + }); + + it("adds a fixed-width stats column after the summary-view selector when df_meta.stats is present", () => { + const stats: DFMetaStats = { status: "pending", tier: "schema", gen: 1 }; + renderBar({ ...baseMeta, stats }); + const names = fields(); + expect(names.indexOf("stats")).toBe(names.indexOf("df_display") + 1); + + const column = mockGrid.props.columnDefs[names.indexOf("stats")]; + expect(column.cellRenderer).toBe(StatsStatusCell); + // A fixed width, so the status changing never moves the other columns. + expect(typeof column.width).toBe("number"); + expect(column.flex).toBeUndefined(); + expect(mockGrid.props.rowData[0].stats).toBe(stats); + }); + + it("keeps the column for every status", () => { + for (const status of ["pending", "not_computed", "error", "complete"] as const) { + const { unmount } = renderBar({ ...baseMeta, stats: { status, gen: 1 } }); + expect(fields()).toContain("stats"); + unmount(); + } + }); + + it("hands the compute callback to the cell renderer through the grid context", () => { + const onComputeStats = jest.fn(); + renderBar({ ...baseMeta, stats: { status: "not_computed", gen: 1 } }, onComputeStats); + expect(mockGrid.props.context.onComputeStats).toBe(onComputeStats); + }); +}); + +describe("StatsStatusCell", () => { + const cell = (value: DFMetaStats | undefined, onComputeStats?: () => void) => + render(); + + it("pending: says the stats are being computed", () => { + cell({ status: "pending", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "pending"); + expect(root).toHaveTextContent("Computing summary stats"); + expect(root).toHaveAttribute("role", "status"); + }); + + it("not_computed: offers a Compute summary stats button that calls the handler", () => { + const onComputeStats = jest.fn(); + cell({ status: "not_computed", gen: 1 }, onComputeStats); + expect(screen.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); + + fireEvent.click(screen.getByRole("button", { name: "Compute summary stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("not_computed: with no handler there is no button, only the label", () => { + cell({ status: "not_computed", gen: 1 }); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); + }); + + it("error: shows the reason", () => { + cell({ status: "error", gen: 1, reason: "stats_failed" }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "error"); + expect(root).toHaveTextContent("Stats error: stats_failed"); + }); + + it("error: without a reason still says so", () => { + cell({ status: "error", gen: 1 }); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Stats error"); + }); + + it("complete: says the stats are ready", () => { + cell({ status: "complete", tier: "full", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "complete"); + expect(root).toHaveTextContent("Summary stats ready"); + }); + + it("renders nothing without stats", () => { + const { container } = cell(undefined); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.tsx b/packages/buckaroo-js-core/src/components/StatusBar.tsx index 1a08dedd8..a7ae9c856 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.tsx @@ -4,7 +4,7 @@ import * as _ from "lodash-es"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { ColDef, GridApi, GridOptions } from "ag-grid-community"; import { basicIntFormatter } from "./DFViewerParts/Displayer"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, DFMetaStats } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; import { BuckarooState, BKeys } from "./WidgetTypes"; import { CustomCellEditorProps } from 'ag-grid-react'; @@ -307,6 +307,11 @@ export const SearchEditor = memo(({ value, onValueChange, stopEditing }: Custom ); }); +/** Renders df_meta.stats in the status bar (rows-first c4). Stub. */ +export const StatsStatusCell = function (_params: { value?: DFMetaStats; context?: { onComputeStats?: () => void } }) { + return null; +}; + export function StatusBar({ dfMeta, buckarooState, @@ -334,6 +339,8 @@ export function StatusBar({ * Python's ComponentConfig TypedDict; cell renderers read them via * params.context.componentConfig. */ componentConfig?: Record; + /** Sends a forced stats_request; shown as a control while the stats are not computed. */ + onComputeStats?: () => void; }) { if (false) { console.log("heightOverride", heightOverride); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx new file mode 100644 index 000000000..9bb495202 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx @@ -0,0 +1,89 @@ +/** + * BuckarooView — the Compute summary stats control (rows-first c4). + * + * While df_meta.stats says the stats are not computed, the status bar offers a + * control that asks the server for them. BuckarooView hands the widget the + * callback, and the callback sends `stats_request {force: true}` through + * whatever IModel the host gave it. + */ +import { render, cleanup, act } from "@testing-library/react"; +import { BuckarooView } from "./BuckarooView"; +import type { IModel } from "./IModel"; + +const mockWidgetProps: any[] = []; +jest.mock("../components/BuckarooWidgetInfinite", () => ({ + BuckarooInfiniteWidget: (props: any) => { + mockWidgetProps.push(props); + return
; + }, + DFViewerInfiniteDS: () =>
, + getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), +})); + +function makeFakeModel(state: Record): { model: IModel; sent: any[] } { + const sent: any[] = []; + const model: IModel = { + send: (msg) => { sent.push(msg); }, + get: (k) => state[k], + set: (k, v) => { state[k] = v; }, + save_changes: () => {}, + on: () => {}, + off: () => {}, + }; + return { model, sent }; +} + +const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, +}; +const metaWith = (stats?: Record) => ({ + total_rows: 3, columns: 1, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const mountBuckaroo = async (state: Record) => { + const { model, sent } = makeFakeModel(state); + await act(async () => { + render(); + }); + return { model, sent, props: () => mockWidgetProps[mockWidgetProps.length - 1] }; +}; + +afterEach(() => { + mockWidgetProps.length = 0; + cleanup(); +}); + +describe("BuckarooView on_compute_stats (rows-first c4)", () => { + it("hands the widget a callback that sends a forced stats_request for the gen on screen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + + props().on_compute_stats(); + expect(sent).toEqual([{ type: "stats_request", stats_gen: 9, scope: "raw", force: true }]); + }); + + it("sends nothing when the model's df_meta carries no stats.gen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith(), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + props().on_compute_stats(); + expect(sent).toEqual([]); + }); + + it("sends no request on its own: asking is the scheduler's job, and a session that is not pending is left alone", async () => { + const { sent } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(sent).toEqual([]); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 94c2ccef1..37f0992c8 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -1,188 +1,525 @@ -import { StateOrchestrator, WsLike } from "./StateOrchestrator"; +/** + * StateOrchestrator — the client scheduler for the stats wire (rows-first c4). + * + * A session the server defers stats for sends a stats-free first frame with + * df_meta.stats.status "pending". The scheduler asks for the stats once the + * first rows have arrived (`stats_request`), asks again for each reply that + * leaves the stats pending, and stands down when the state changes. The merge + * of the replies is StatsChannel's; the scheduler only watches the model. + * + * The scheduler tests drive a fake model the way WebSocketModel drives a real + * one (set a key, emit its change event). The last block builds a real + * WebSocketModel from a fake socket. + */ +import { StateOrchestrator, StatsModel, requestStats, touchesDataflow } from "./StateOrchestrator"; +import { WebSocketModel } from "./WebSocketModel"; -class FakeWs implements WsLike { - sent: string[] = []; - send(message: string): void { - this.sent.push(message); +class FakeModel implements StatsModel { + sent: any[] = []; + private handlers = new Map>(); + + constructor(public state: Record) {} + + get(key: string) { + return this.state[key]; + } + set(key: string, value: any) { + this.state[key] = value; + this.emit(`change:${key}`, value); + } + send(msg: any) { + this.sent.push(msg); + } + on(event: string, handler: (...args: any[]) => void) { + if (!this.handlers.has(event)) this.handlers.set(event, new Set()); + this.handlers.get(event)!.add(handler); + } + off(event: string, handler: (...args: any[]) => void) { + this.handlers.get(event)?.delete(handler); + } + emit(event: string, ...args: any[]) { + for (const h of Array.from(this.handlers.get(event) ?? [])) h(...args); } - last(): Record { - return JSON.parse(this.sent[this.sent.length - 1]); + listenerCount() { + return Array.from(this.handlers.values()).reduce((n, set) => n + set.size, 0); } - byType(type: string): Record[] { - return this.sent - .map((s) => JSON.parse(s)) - .filter((m) => m.type === type); + /** A full frame: each key lands and fires its change event in turn, as + * WebSocketModel's initial_state branch does. */ + frame(msg: Record) { + for (const [key, value] of Object.entries(msg)) this.set(key, value); } } -describe("StateOrchestrator", () => { - let ws: FakeWs; - let orch: StateOrchestrator; +const meta = (stats?: Record) => ({ + total_rows: 3, columns: 2, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); +const complete = (gen: number) => ({ status: "complete", tier: "full", gen }); +const dict = (rows: any[] = []) => ({ all_stats: rows }); +const statRow = (stat: string) => ({ index: stat, level_0: stat, a: 1 }); +// Typed loosely: the server's buckaroo_state also carries keys (search_string) +// that BuckarooState does not declare. +const bState = (over: Record = {}): any => ({ + sampled: false, cleaning_method: false, quick_command_args: {}, post_processing: false, + df_display: "main", show_commands: false, ...over, +}); + +const request = (gen: number, extra: Record = {}) => ({ + type: "stats_request", stats_gen: gen, scope: "raw", ...extra, +}); + +const makeModel = (stats?: Record) => + new FakeModel({ df_meta: meta(stats), df_data_dict: dict(), buckaroo_state: bState() }); + +/** The first rows reached the client: WebSocketModel emits msg:custom once it + * has paired an infinite_resp with its parquet frame. */ +const rowsArrived = (model: FakeModel) => + model.emit("msg:custom", { type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }, []); + +const start = (model: FakeModel, opts: Record = {}) => { + const orchestrator = new StateOrchestrator({ model, ...opts }); + orchestrator.start(); + return orchestrator; +}; + +// With the defaults, a state change waits 2 x 250 ms before it asks again. +const DEBOUNCE = 500; +const FIRST_PAINT_TIMEOUT = 1500; + +// Runs due timers and every promise continuation they leave behind. +const tick = (ms = 0) => jest.advanceTimersByTimeAsync(ms); + +beforeEach(() => { + jest.useFakeTimers(); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +describe("when nothing is pending", () => { + it("requests nothing for a session whose df_meta has no stats (every session today)", async () => { + const model = makeModel(); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["complete", "not_computed", "error"])("requests nothing when the status is %s", async (status) => { + const model = makeModel({ status, tier: "schema", gen: 3 }); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); +}); + +describe("the first request", () => { + it("goes out after the first infinite_resp, not before", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(100); + expect(model.sent).toEqual([]); - beforeEach(() => { - jest.useFakeTimers(); - ws = new FakeWs(); - orch = new StateOrchestrator({ ws, minDebounceMs: 10, maxDebounceMs: 5000 }); + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); }); - afterEach(() => { - orch.dispose(); - jest.useRealTimers(); + it("goes out anyway when no rows come (an empty frame, the summary view)", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(FIRST_PAINT_TIMEOUT - 1); + expect(model.sent).toEqual([]); + + await tick(1); + expect(model.sent).toEqual([request(3)]); }); - it("starts with token 0", () => { - expect(orch.currentToken).toBe(0); + it("carries no force flag", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + await tick(); + expect(model.sent[0]).not.toHaveProperty("force"); }); - it("onStateChange bumps token and ships state_change", () => { - orch.onStateChange({ search_string: "x" }); - expect(orch.currentToken).toBe(1); - const msg = ws.last(); - expect(msg.type).toBe("state_change"); - expect(msg.state_token).toBe(1); - expect((msg.new_state as Record).search_string).toBe("x"); + it("is sent once, however many row responses follow", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + rowsArrived(model); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); +}); + +describe("one request per reply", () => { + const afterFirstRequest = async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; - it("schedules compute_stat_group after the debounce", () => { - orch.onStateChange({ search_string: "PIZZA" }); - // Only the immediate state_change has been sent so far. - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("asks again for each reply that leaves the stats pending, and stops at the final one", async () => { + const { model } = await afterFirstRequest(); + expect(model.sent).toEqual([request(3)]); - // Default baseline is 500ms × 2× = 1000ms; clamped to maxDebounceMs=5000. - jest.advanceTimersByTime(999); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + // No reply yet: nothing more goes out, however long the server takes. + await tick(10_000); + expect(model.sent).toEqual([request(3)]); - jest.advanceTimersByTime(1); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].scope).toBe("filt"); - expect(reqs[0].group).toBe("aggregate"); - expect(reqs[0].state_token).toBe(1); + // A partial reply is a new df_data_dict under the same df_meta, which is + // what StatsChannel does for a stats_update that is not final. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + + model.set("df_data_dict", dict([statRow("mean"), statRow("std")])); + await tick(); + expect(model.sent).toHaveLength(3); + + // The final reply also sets df_meta.stats to complete. + model.set("df_data_dict", dict([statRow("mean"), statRow("std"), statRow("max")])); + model.set("df_meta", meta(complete(3))); + await tick(10_000); + expect(model.sent).toHaveLength(3); }); - it("back-to-back state_changes cancel the previous debounce timer", () => { - orch.onStateChange({ search_string: "P" }); - jest.advanceTimersByTime(500); - orch.onStateChange({ search_string: "PI" }); - // The first timer would have fired at t=1000ms; the second - // resets it so at t=999ms from the SECOND call (= 1499 overall) - // no compute_stat_group has fired yet. - jest.advanceTimersByTime(998); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("reads a reply the same way whichever of the two events comes first", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta(complete(3))); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); - // The second timer fires. - jest.advanceTimersByTime(2); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(2); // second state_change's token + it.each([ + ["df_meta then df_data_dict", ["df_meta", "df_data_dict"]], + ["df_data_dict then df_meta", ["df_data_dict", "df_meta"]], + ])("does not take a full frame for the same state as a reply (%s)", async (_label, order) => { + // A search term that changes only the highlight comes back as a full + // initial_state for the same stats_gen, with a new df_meta and a new dict. + const { model } = await afterFirstRequest(); + const full: Record = { df_meta: meta(pending(3)), df_data_dict: dict([statRow("dtype")]) }; + model.frame(Object.fromEntries(order.map((k) => [k, full[k]]))); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); - it("rapid typing produces just one aggregate request, with the latest token", () => { - // Simulate 5 keystrokes at 100ms intervals — typical typing cadence. - for (let i = 0; i < 5; i++) { - orch.onStateChange({ search_string: "P".repeat(i + 1) }); - jest.advanceTimersByTime(100); - } - // Default debounce = 1000ms after the last keystroke. No aggregate yet. - expect(ws.byType("compute_stat_group")).toHaveLength(0); - - jest.advanceTimersByTime(1000); - const reqs = ws.byType("compute_stat_group"); - // Exactly one aggregate request, with the 5th (final) token. - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(5); - }); - - it("onStatGroupResult with matching token updates the baseline", () => { - orch.onStateChange({ search_string: "x" }); - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, - scope: "filt", - group: "aggregate", - elapsed_ms: 7500, - }); - expect(applied).toBe(true); - - // Next debounce is 2× 7500 = 15000ms, clamped to maxDebounceMs=5000. - expect(orch.computeDebounce("filt")).toBe(5000); - }); - - it("onStatGroupResult with stale token is silently dropped", () => { - orch.onStateChange({ search_string: "x" }); - orch.onStateChange({ search_string: "xy" }); - // Token is now 2. - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, // stale - scope: "filt", - group: "aggregate", - elapsed_ms: 9999, - }); - expect(applied).toBe(false); - // Baseline unchanged — debounce stays at the default. - expect(orch.computeDebounce("filt")).toBe(1000); + it("stops when the server reports an error", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); - it("computeDebounce respects minDebounceMs floor", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 500, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 10, // 2× 10 = 20, well under floor - }); - expect(o.computeDebounce("filt")).toBe(500); + it("stops when the server says the stats are not computed", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "not_computed", tier: "schema", gen: 3 })); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); }); +}); - it("computeDebounce respects maxDebounceMs ceiling", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 200, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 6000, // 2× = 12000, hits ceiling - }); - expect(o.computeDebounce("filt")).toBe(3000); +describe("a state change", () => { + const pendingAt = async (gen: number) => { + const model = makeModel(pending(gen)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; + + // The server answers a dataflow change with a frame for the next stats_gen. + const nextFrame = (model: FakeModel, gen: number) => { + model.frame({ df_meta: meta(pending(gen)), df_data_dict: dict() }); + rowsArrived(model); + }; + + it("stops the requests for the old state, then asks once for the new one", async () => { + const { model } = await pendingAt(3); + expect(model.sent).toEqual([request(3)]); + + // The user changes a dataflow field while the request is out. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(10); + // The reply to the old request lands. It is not followed by another. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10); + nextFrame(model, 4); + + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("reads a frame whose dict comes before its df_meta as the next state, not as a reply", async () => { + const { model } = await pendingAt(3); + // The frame for the next gen carries its dict before its df_meta. + model.frame({ df_data_dict: dict(), df_meta: meta(pending(4)) }); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it.each([ + ["post_processing", { post_processing: "log_scale" }], + ["cleaning_method", { cleaning_method: "aggressive" }], + ["quick_command_args", { quick_command_args: { search: ["x"] } }], + ])("a %s change cancels a request that is waiting out its delay", async (_field, change) => { + const { model } = await pendingAt(3); + nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(300); + // 600 ms in: the request that was due at 500 ms never went out. + expect(model.sent).toEqual([request(3)]); + + nextFrame(model, 5); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(5)]); }); - it("initialAggregateMs seeds the baseline before any observed compute", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - initialAggregateMs: { filt: 250 }, - minDebounceMs: 10, - maxDebounceMs: 5000, - multiplier: 2, + it.each([ + ["search_string (the #1015 path)", { search_string: "x" }], + ["df_display", { df_display: "summary" }], + ["show_commands", { show_commands: "1" }], + ["sampled", { sampled: "sample" }], + ])("a %s-only change is skipped", async (_label, change) => { + const { model } = await pendingAt(3); + nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(200); + // The request is still due at 500 ms, as if nothing had changed. + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it("a search_string-only change does not interrupt a chain of replies", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ search_string: "x" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("a frame that repeats the same dataflow state is not a state change", async () => { + // Every full frame carries buckaroo_state back as the client sent it. + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState()); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("asks again for the same state when the server never answers the change with a frame", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(FIRST_PAINT_TIMEOUT + DEBOUNCE); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("waits 2 x the last request's time, within the limits, before the next state's request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + expect(orchestrator.computeDebounce()).toBe(500); + + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); // the reply, 400 ms after the request + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + nextFrame(model, 4); + await tick(799); + expect(model.sent).toHaveLength(2); + await tick(1); + expect(model.sent).toHaveLength(3); + expect(model.sent[2]).toEqual(request(4)); + }); + + it.each([ + [10, 100], + [400, 800], + [5000, 2000], + ])("computeDebounce after a %i ms request is %i ms (floor 100, ceiling 2000)", async (elapsed, expected) => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + rowsArrived(model); + await tick(elapsed); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(orchestrator.computeDebounce()).toBe(expected); + }); +}); + +describe("requestStats", () => { + it("sends a stats_request for the gen the model shows", () => { + const model = makeModel(pending(7)); + expect(requestStats(model)).toBe(true); + expect(model.sent).toEqual([request(7)]); + }); + + it("force sends force: true, which the Compute summary stats control uses", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + expect(requestStats(model, { force: true })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true })]); + }); + + it("sends nothing when df_meta carries no stats.gen", () => { + const model = makeModel(); + expect(requestStats(model)).toBe(false); + expect(requestStats(model, { force: true })).toBe(false); + expect(model.sent).toEqual([]); + }); +}); + +describe("touchesDataflow", () => { + it("is true for each field the server reruns the dataflow for", () => { + expect(touchesDataflow(bState(), bState({ post_processing: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ cleaning_method: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ quick_command_args: { search: ["x"] } }))).toBe(true); + }); + + it("is false for the others, and for an equal quick_command_args that is a new object", () => { + expect(touchesDataflow(bState(), bState({ search_string: "x" }))).toBe(false); + expect(touchesDataflow(bState(), bState({ df_display: "summary" }))).toBe(false); + expect(touchesDataflow(bState({ quick_command_args: { search: ["x"] } }), bState({ quick_command_args: { search: ["x"] } }))).toBe(false); + }); + + it("is false when there is no earlier state to compare with", () => { + expect(touchesDataflow(undefined, bState({ post_processing: "x" }))).toBe(false); + }); +}); + +describe("start and stop", () => { + it("stop removes every listener and cancels the pending request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + expect(model.listenerCount()).toBeGreaterThan(0); + + rowsArrived(model); + orchestrator.stop(); + expect(model.listenerCount()).toBe(0); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("start is idempotent: a second call adds no listeners", () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + const listeners = model.listenerCount(); + orchestrator.start(); + expect(model.listenerCount()).toBe(listeners); + }); + + it("can start again after a stop", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + orchestrator.stop(); + orchestrator.start(); + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); + }); +}); + +// The scheduler is started by WebSocketModel, so a session reached through +// BuckarooServerView or the standalone page gets it with no wiring of its own. +describe("wired into WebSocketModel", () => { + class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } + deliverBinary() { + this.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + } + } + + const update = (gen: number, stat: string, final: boolean) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final, + payload: { format: "json", layout: "wide", data: [{ index: stat, level_0: stat, a: 1 }] }, + elapsed_ms: 5, + }); + + const makeSocketModel = (stats?: Record) => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: meta(stats), + df_data_dict: { all_stats: [{ index: "dtype", level_0: "dtype", a: "int64" }] }, + buckaroo_state: bState(), }); - expect(o.computeDebounce("filt")).toBe(500); // 2× 250 - // Unrelated scope still uses fallback. - expect(o.computeDebounce("clean")).toBe(1000); // 2× 500 (default) + return { ws, model }; + }; + const rowsFromServer = (ws: FakeSocket) => { + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }); + ws.deliverBinary(); + }; + + it("requests the stats after the first rows, once per reply, and stops at the final one", async () => { + const { ws, model } = makeSocketModel(pending(3)); + await tick(100); + expect(ws.sent).toEqual([]); + + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver(update(3, "mean", false)); + await tick(); + expect(ws.sent).toEqual([request(3), request(3)]); + + ws.deliver(update(3, "std", true)); + await tick(10_000); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_meta").stats.status).toBe("complete"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std"]); }); - it("dispose cancels all pending aggregate timers", () => { - orch.onStateChange({ search_string: "x" }); - orch.dispose(); - jest.advanceTimersByTime(10_000); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("moves on to the next gen when a state change frame arrives", async () => { + const { ws } = makeSocketModel(pending(3)); + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + rowsFromServer(ws); + await tick(DEBOUNCE); + expect(ws.sent).toEqual([request(3), request(4)]); }); - it("scopesForAggregate parameter overrides default ['filt']", () => { - orch.onStateChange({ search_string: "x" }, { scopesForAggregate: ["filt", "clean"] }); - jest.advanceTimersByTime(10_000); - const reqs = ws.byType("compute_stat_group"); - const scopes = reqs.map((r) => r.scope).sort(); - expect(scopes).toEqual(["clean", "filt"]); + it("sends nothing for a model whose df_meta has no stats", async () => { + const { ws } = makeSocketModel(); + rowsFromServer(ws); + await tick(10_000); + expect(ws.sent).toEqual([]); }); }); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts index e096ac606..d9f1e5446 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts @@ -1,153 +1,44 @@ /** - * Client-side scheduler for the JS-driven progressive-stats protocol. + * Client-side scheduler for the stats wire (rows-first c4). * - * On a state_change, ships the cheap (scalar) request immediately so - * df_meta + scalar pinned rows update fast. Schedules the expensive - * (aggregate) compute per scope behind an adaptive debounce — by - * default 2× the last observed aggregate compute time for that scope, - * clamped to ``[minDebounceMs, maxDebounceMs]``. - * - * Token-based cancellation: every state_change bumps an internal - * token; results carrying a stale token are dropped on arrival. - * - * See plans/js-driven-stat-debounce.md for the full protocol design. - * - * This module is transport-agnostic — pass any object with a - * ``send(s: string)`` method. Existing buckaroo WS connections fit. + * Stub: the API the tests drive, with no behaviour yet. */ +import { BuckarooState } from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +export type StatsModel = Pick; + +export const DATAFLOW_STATE_FIELDS = ["post_processing", "cleaning_method", "quick_command_args"] as const; -export type ScopeName = "raw" | "clean" | "filt"; -export type CostGroup = "scalar" | "aggregate"; +export function touchesDataflow(_prev: BuckarooState | undefined, _next: BuckarooState | undefined): boolean { + return false; +} + +export interface StatsRequestOptions { + force?: boolean; +} -export interface WsLike { - send(message: string): void; +export function requestStats(_model: Pick, _opts?: StatsRequestOptions): boolean { + return false; } export interface OrchestratorOptions { - ws: WsLike; - /** Lower bound on the per-scope debounce. Default 200 ms. */ + model: StatsModel; minDebounceMs?: number; - /** Upper bound on the per-scope debounce. Default 3000 ms. */ maxDebounceMs?: number; - /** Multiplier on the last observed aggregate compute time. Default 2. */ multiplier?: number; - /** Initial aggregate baseline per scope (used until we observe a real one). */ - initialAggregateMs?: Partial>; -} - -export interface StatGroupResult { - type: "stat_group_result"; - state_token: number; - scope: ScopeName; - group: CostGroup; - elapsed_ms: number; - stats?: unknown; + initialRequestMs?: number; + firstPaintTimeoutMs?: number; } export class StateOrchestrator { - private token = 0; - private aggregateTimers = new Map>(); - private lastAggregateMs = new Map(); - private readonly ws: WsLike; - private readonly minDebounceMs: number; - private readonly maxDebounceMs: number; - private readonly multiplier: number; - /** Fallback baseline before any real aggregate compute has been observed. */ - private readonly defaultBaselineMs = 500; + constructor(_opts: OrchestratorOptions) {} - constructor(opts: OrchestratorOptions) { - this.ws = opts.ws; - this.minDebounceMs = opts.minDebounceMs ?? 200; - this.maxDebounceMs = opts.maxDebounceMs ?? 3000; - this.multiplier = opts.multiplier ?? 2; - if (opts.initialAggregateMs) { - for (const [scope, ms] of Object.entries(opts.initialAggregateMs)) { - if (ms != null) this.lastAggregateMs.set(scope as ScopeName, ms); - } - } - } - - /** - * Current state-change token. Tests inspect this; production - * code doesn't usually need it. - */ - get currentToken(): number { - return this.token; - } - - /** - * Compute the debounce delay (ms) for a given scope based on - * the last observed aggregate compute time, clamped to - * ``[minDebounceMs, maxDebounceMs]``. - */ - computeDebounce(scope: ScopeName): number { - const last = this.lastAggregateMs.get(scope) ?? this.defaultBaselineMs; - const raw = last * this.multiplier; - return Math.max(this.minDebounceMs, Math.min(this.maxDebounceMs, raw)); - } + start(): void {} - /** - * Drive a single user-initiated state change. - * - * 1. Bump the state token. - * 2. Cancel any pending aggregate timers from the previous change. - * 3. Ship the ``state_change`` message (server will reply with - * the scalar stats fast). - * 4. For each scope expected to have aggregate work, schedule a - * debounced ``compute_stat_group`` request. Timer fires the - * request only if no further state_change has bumped the - * token meanwhile. - */ - onStateChange( - newState: Record, - opts?: { scopesForAggregate?: ScopeName[] }, - ): void { - const token = ++this.token; - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); - - this.ws.send(JSON.stringify({ - type: "state_change", - state_token: token, - new_state: newState, - })); - - const scopes = opts?.scopesForAggregate ?? ["filt"]; - for (const scope of scopes) { - const delay = this.computeDebounce(scope); - const tid = setTimeout(() => { - // If a newer state_change bumped the token while we - // were waiting, drop the request — sending it would - // produce a stat_group_aborted from the server anyway. - if (token !== this.token) return; - this.ws.send(JSON.stringify({ - type: "compute_stat_group", - state_token: token, - scope, - group: "aggregate", - })); - }, delay); - this.aggregateTimers.set(scope, tid); - } - } - - /** - * Process a ``stat_group_result`` from the server. Stale results - * (mismatched token) are silently ignored. Successful results - * update the per-scope aggregate baseline used by the next - * debounce. - * - * Returns true if the result was applied, false if it was stale. - */ - onStatGroupResult(msg: StatGroupResult): boolean { - if (msg.state_token !== this.token) return false; - this.lastAggregateMs.set(msg.scope, msg.elapsed_ms); - return true; - } + stop(): void {} - /** Cancel all pending aggregate timers. Call on widget unmount. */ - dispose(): void { - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); + computeDebounce(): number { + return 0; } } diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx new file mode 100644 index 000000000..04b3922b7 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -0,0 +1,170 @@ +/** + * Story for the client states of the stats wire (rows-first c4). + * + * df_meta.stats.status says where the summary stats stand, and the status bar + * shows it in a column of its own: + * + * - "pending" "Computing summary stats", pinned keys show placeholders + * - "not_computed" a "Compute summary stats" button, pinned keys omitted + * - "error" "Stats error: ", pinned keys omitted + * - "complete" "Summary stats ready", the values in place + * + * The buttons switch the status. The button in the status bar sends + * `stats_request {force: true}` through a fake model, which logs what it was + * asked to send. Used by stats-scheduler-states.spec.ts. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import { BuckarooInfiniteWidget } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { BuckarooOptions, BuckarooState, DFMeta, StatsStatus } from "../components/WidgetTypes"; +import { CommandConfigT } from "../components/CommandUtils"; +import { Operation } from "../components/OperationUtils"; +import { baseOperationResults } from "../components/DependentTabs"; +import { requestStats } from "../server/StateOrchestrator"; + +const STATUSES: StatsStatus[] = ["pending", "not_computed", "error", "complete"]; +const GEN = 7; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 2, b: "N/A" }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const buckarooOptions: BuckarooOptions = { + sampled: [], + cleaning_method: [], + post_processing: [], + df_display: ["main"], + show_commands: [], +}; +const commandConfig: CommandConfigT = { argspecs: {}, defaultArgs: {} }; + +const StatsSchedulerStatesInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + const [sent, setSent] = useState([]); + const [buckarooState, setBuckarooState] = useState({ + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, + }); + const [operations, setOperations] = useState([]); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { + status, + tier: status === "complete" ? "full" : "schema", + gen: GEN, + ...(status === "error" ? { reason: "stats_failed" } : {}), + }, + }) as DFMeta, + [status], + ); + + // The model the control sends through: it answers get("df_meta") from the + // story's state and logs what it is asked to send. + const model = useMemo( + () => ({ + get: (key: string) => (key === "df_meta" ? df_meta : undefined), + send: (msg: unknown) => setSent((log) => [...log, msg]), + }), + [df_meta], + ); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : ([] as DFData), + empty: [] as DFData, + }), + [status], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + df_meta.stats.status = {status} +
+
+ requestStats(model, { force: true })} + /> +
+
+        {JSON.stringify(sent)}
+      
+
+ ); +}; + +const meta = { + title: "Buckaroo/StatsSchedulerStates", + component: StatsSchedulerStatesInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; diff --git a/scripts/test_playwright_storybook.sh b/scripts/test_playwright_storybook.sh index 6d835ab2a..9445be842 100755 --- a/scripts/test_playwright_storybook.sh +++ b/scripts/test_playwright_storybook.sh @@ -96,6 +96,7 @@ STORYBOOK_TESTS=( "pw-tests/outside-params.spec.ts" "pw-tests/fitcontent-height.spec.ts" "pw-tests/stats-pending-pinned-rows.spec.ts" + "pw-tests/stats-scheduler-states.spec.ts" # "pw-tests/example.spec.ts" # Has pre-existing failures, excluded for now ) From d6afbf9835c2c9801591a4e9ebe15204bc7c3e6b Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:34:54 -0400 Subject: [PATCH 08/18] test(client): failing tests for the first state change, the stats status layout and the control wiring (rows-first c4) Cases found untested or mis-specified after the first push, written against the same stubs: a model that starts complete waits out the delay for its first state change, as does a change made after the earlier stats completed; a request's time is measured once; the Compute summary stats button calls its handler with no arguments; the standalone page offers that button for a session whose stats are not computed and asks for nothing until it is clicked. The layout spec now says what holds across statuses (nothing above or beside the grid moves, and the grid's height follows the pinned area), and the story loads the widget's stylesheet as the real page does. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 47 +++++++++++++ .../pw-tests/stats-scheduler-states.spec.ts | 27 ++++++-- .../src/components/StatusBar.stats.test.tsx | 2 + .../src/server/StateOrchestrator.test.ts | 69 ++++++++++++++++--- .../stories/StatsSchedulerStates.stories.tsx | 1 + 5 files changed, 131 insertions(+), 15 deletions(-) diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index bf98d7d19..af04c3f6e 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -569,6 +569,53 @@ test.describe('WebSocket data flow', () => { expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); }); + test('a session whose stats are not computed: the page asks for nothing until the Compute summary stats button is clicked', async ({ page, request }) => { + const session = `ws-notcomputed-${Date.now()}`; + await loadBuckarooSession(request, session); + + const requests: any[] = []; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + let firstFrame = true; + server.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'initial_state' && firstFrame) { + firstFrame = false; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'not_computed', tier: 'schema', gen: 4, reason: 'host' } }; + ws.send(JSON.stringify(msg)); + return; + } + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + requests.push(msg); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed'); + // The pinned rows are omitted, not left as placeholders. + await expect(page.locator('.ag-floating-top .ag-row')).toHaveCount(0); + + // Nothing is asked for on its own, however long the page waits. + await page.waitForTimeout(2500); + expect(requests).toEqual([]); + + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + await expect.poll(() => requests).toEqual([{ type: 'stats_request', stats_gen: 4, scope: 'raw', force: true }]); + }); + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { const session = `ws-nosched-${Date.now()}`; await loadBuckarooSession(request, session); diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts index 4470a9324..64788ad1d 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -2,10 +2,13 @@ * Playwright test for the client states of the stats wire (rows-first c4). * * The StatsSchedulerStates story flips df_meta.stats.status. Each status has - * its own text in the status bar's stats column, the pinned area follows it as - * in c0a, and switching between them moves nothing: the status bar and the - * grid keep their boxes. In "not_computed" the status bar offers a Compute - * summary stats button, which sends stats_request {force: true}. + * its own text in the status bar's stats column, and the pinned area follows it + * as in c0a. Switching between statuses moves nothing above or beside the grid: + * the status bar keeps its box, and the grid keeps its position and width. The + * grid's height changes only with the pinned area: placeholders hold the height + * of the values that replace them, and omitted rows take theirs away. In + * "not_computed" the status bar offers a Compute summary stats button, which + * sends stats_request {force: true}. */ import { test, expect, Page } from "@playwright/test"; import { waitForCells } from "./ag-pw-utils"; @@ -44,22 +47,32 @@ test("each stats status shows its own text in the status bar and nothing moves b const reference = await boxes(page); expect(reference.statusBar).not.toBeNull(); expect(reference.grid).not.toBeNull(); + // Nothing above or beside the grid moves, whatever the status. + const staysPut = async () => { + const now = await boxes(page); + expect(now.statusBar).toEqual(reference.statusBar); + expect(now.grid).toMatchObject({ x: reference.grid!.x, y: reference.grid!.y, width: reference.grid!.width }); + return now; + }; // not_computed: the control, and no pinned rows. await page.getByTestId("status-not_computed").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed"); await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); - expect(await boxes(page)).toEqual(reference); + const withoutPinned = await staysPut(); + // The omitted rows take their height with them. + expect(withoutPinned.grid!.height).toBeLessThan(reference.grid!.height); // error: the reason, and no pinned rows. await page.getByTestId("status-error").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "error"); await expect(statsCell).toContainText("Stats error: stats_failed"); await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); - expect(await boxes(page)).toEqual(reference); + expect((await staysPut()).grid).toEqual(withoutPinned.grid); - // complete: the values are in the pinned area where the placeholders were. + // complete: the values are in the pinned area where the placeholders were, so + // the grid is as tall as it was while pending. await page.getByTestId("status-complete").click(); await expect(statsCell).toHaveAttribute("data-stats-status", "complete"); await expect(statsCell).toContainText("Summary stats ready"); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx index d4fe5c99e..7db44f0e8 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -115,7 +115,9 @@ describe("StatsStatusCell", () => { expect(screen.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); fireEvent.click(screen.getByRole("button", { name: "Compute summary stats" })); + // Called with no arguments, not with the click event. expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); }); it("not_computed: with no handler there is no button, only the label", () => { diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 37f0992c8..2e7f92eda 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -134,7 +134,7 @@ describe("the first request", () => { await tick(FIRST_PAINT_TIMEOUT - 1); expect(model.sent).toEqual([]); - await tick(1); + await tick(2); expect(model.sent).toEqual([request(3)]); }); @@ -240,9 +240,12 @@ describe("a state change", () => { return { model, orchestrator }; }; - // The server answers a dataflow change with a frame for the next stats_gen. - const nextFrame = (model: FakeModel, gen: number) => { + // The server answers a dataflow change with a frame for the next stats_gen, + // then the grid's refetch brings rows. They are separate messages, so the + // scheduler has read the frame by the time the rows arrive. + const nextFrame = async (model: FakeModel, gen: number) => { model.frame({ df_meta: meta(pending(gen)), df_data_dict: dict() }); + await tick(); rowsArrived(model); }; @@ -256,7 +259,7 @@ describe("a state change", () => { // The reply to the old request lands. It is not followed by another. model.set("df_data_dict", dict([statRow("mean")])); await tick(10); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(DEBOUNCE - 1); expect(model.sent).toEqual([request(3)]); @@ -270,6 +273,7 @@ describe("a state change", () => { const { model } = await pendingAt(3); // The frame for the next gen carries its dict before its df_meta. model.frame({ df_data_dict: dict(), df_meta: meta(pending(4)) }); + await tick(); rowsArrived(model); await tick(10_000); expect(model.sent).toEqual([request(3), request(4)]); @@ -281,7 +285,7 @@ describe("a state change", () => { ["quick_command_args", { quick_command_args: { search: ["x"] } }], ])("a %s change cancels a request that is waiting out its delay", async (_field, change) => { const { model } = await pendingAt(3); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(300); model.set("buckaroo_state", bState(change)); @@ -289,7 +293,7 @@ describe("a state change", () => { // 600 ms in: the request that was due at 500 ms never went out. expect(model.sent).toEqual([request(3)]); - nextFrame(model, 5); + await nextFrame(model, 5); await tick(DEBOUNCE - 1); expect(model.sent).toEqual([request(3)]); await tick(1); @@ -303,7 +307,7 @@ describe("a state change", () => { ["sampled", { sampled: "sample" }], ])("a %s-only change is skipped", async (_label, change) => { const { model } = await pendingAt(3); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(300); model.set("buckaroo_state", bState(change)); @@ -329,6 +333,37 @@ describe("a state change", () => { expect(model.sent).toEqual([request(3), request(3)]); }); + it("waits out the delay for a state change made after the earlier stats completed", async () => { + const { model } = await pendingAt(3); + await tick(250); // a request that took as long as the default assumes, so the delay is DEBOUNCE + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); + + // The first state asked at once; this one is a change to it. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); + }); + + it("waits out the delay for the first state change of a model that started with its stats complete", async () => { + const model = makeModel(complete(3)); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + + model.set("buckaroo_state", bState({ quick_command_args: { search: ["a"] } })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([]); + await tick(1); + expect(model.sent).toEqual([request(4)]); + }); + it("asks again for the same state when the server never answers the change with a frame", async () => { const { model } = await pendingAt(3); model.set("buckaroo_state", bState({ post_processing: "log_scale" })); @@ -348,7 +383,7 @@ describe("a state change", () => { expect(orchestrator.computeDebounce()).toBe(800); model.set("buckaroo_state", bState({ post_processing: "log_scale" })); - nextFrame(model, 4); + await nextFrame(model, 4); await tick(799); expect(model.sent).toHaveLength(2); await tick(1); @@ -356,6 +391,23 @@ describe("a state change", () => { expect(model.sent[2]).toEqual(request(4)); }); + it("measures a request once: later events do not stretch the delay", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 20_000 }); + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + // A full frame for the finished state, long afterwards. + await tick(10_000); + model.frame({ df_meta: meta(complete(3)), df_data_dict: dict() }); + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + }); + it.each([ [10, 100], [400, 800], @@ -511,6 +563,7 @@ describe("wired into WebSocketModel", () => { expect(ws.sent).toEqual([request(3)]); ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); rowsFromServer(ws); await tick(DEBOUNCE); expect(ws.sent).toEqual([request(3), request(4)]); diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx index 04b3922b7..2644d4dc7 100644 --- a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -15,6 +15,7 @@ */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; +import "../style/dcf-npm.css"; import { BuckarooInfiniteWidget } from "../components/BuckarooWidgetInfinite"; import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; From 95dcceecbf164814f3a7e572d145618c2d6bae30 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 03:55:23 -0400 Subject: [PATCH 09/18] feat(client): scheduler requests stats after first paint, with pending, not computed and error states (rows-first c4) StateOrchestrator becomes the client scheduler for the stats wire. It takes an IModel, so it sends stats_request {stats_gen, scope: "raw"} through model.send and watches change:df_meta, change:df_data_dict, change:buckaroo_state and msg:custom. It asks once the first infinite_resp has arrived (or after 1.5 s without one), asks again for each reply that leaves df_meta.stats pending, and stops when the status changes. A change to post_processing, cleaning_method or quick_command_args waits out a delay of twice the last request's time (200 to 3000 ms) before asking for the next gen; a search_string-only change, or any other field, leaves the schedule alone. Nothing is requested unless df_meta.stats says pending. WebSocketModel starts the scheduler, so BuckarooServerView and the standalone page get it with no wiring of their own. The status bar gets a fixed-width stats column, for sessions that report df_meta.stats, showing loading, a Compute summary stats button that sends stats_request {force: true}, the error reason, or ready. Valueless pinned rows are omitted in the error state, as in not_computed. requestStats and StateOrchestrator are exported for hosts with their own IModel. Co-Authored-By: Claude Sonnet 5.5 --- .../src/components/BuckarooWidgetInfinite.tsx | 2 + .../src/components/DFViewerParts/gridUtils.ts | 8 +- .../src/components/StatusBar.tsx | 64 ++++- packages/buckaroo-js-core/src/index.ts | 5 + .../src/server/BuckarooView.tsx | 8 + .../src/server/StateOrchestrator.ts | 239 +++++++++++++++++- .../src/server/WebSocketModel.ts | 9 +- .../buckaroo-js-core/src/style/dcf-npm.css | 18 ++ packages/js/standalone.tsx | 7 + 9 files changed, 340 insertions(+), 20 deletions(-) diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index ecb682153..e83cb3250 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -134,6 +134,7 @@ export function BuckarooInfiniteWidget({ df_data_dict, df_display_args, df_meta, + on_compute_stats, operations, on_operations, operation_results, @@ -348,6 +349,7 @@ export function BuckarooInfiniteWidget({ themeConfig={cDisp.df_viewer_config?.component_config?.theme} inFlight={inFlight} componentConfig={effectiveDisplayArgs['main']?.df_viewer_config?.component_config as Record | undefined} + onComputeStats={on_compute_stats} /> void } }) { - return null; +/** + * Where the summary stats stand, as the server reports it in df_meta.stats: + * loading while they are pending, a control to ask for them while they are not + * computed, the reason when they failed. The cell always renders one line in a + * fixed-width column, so changing status moves nothing. + */ +export const StatsStatusCell = function (params: { value?: DFMetaStats; context?: { onComputeStats?: () => void } }) { + const stats = params.value; + if (stats === undefined) return null; + const onComputeStats = params.context?.onComputeStats; + const cell = (content: React.ReactNode, extra: React.HTMLAttributes = {}) => ( + + {content} + + ); + switch (stats.status) { + case "pending": + return cell( + <> +
diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 11d10f022..4a56e1810 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -20,6 +20,7 @@ import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { withStatsCapability } from './server/StatsChannel'; +import { StateOrchestrator, requestStats } from './server/StateOrchestrator'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -64,6 +65,8 @@ export default { WebSocketModel, makeLatestDictDecoder, withStatsCapability, + StateOrchestrator, + requestStats, }; // Named exports for direct imports @@ -95,6 +98,8 @@ export { WebSocketModel, makeLatestDictDecoder, withStatsCapability, + StateOrchestrator, + requestStats, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 9c86f5bee..d69eab528 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -10,6 +10,7 @@ import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; +import { requestStats } from "./StateOrchestrator"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -284,6 +285,12 @@ export function BuckarooView({ model.save_changes(); }, [model]); + // The status bar's Compute summary stats button, shown while the server + // reports the stats as not computed. + const onComputeStats = React.useCallback(() => { + requestStats(model, { force: true }); + }, [model]); + // gridUtils honors component_config.layoutType. Stamp it per entry so the // prop wins when provided. autoHeight=undefined → server value left intact. const effectiveDisplayArgs = React.useMemo( @@ -322,6 +329,7 @@ export function BuckarooView({ on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} /> ) : ( ; +/** The fields of buckaroo_state the server reruns the dataflow for, and so + * bumps stats_gen on. Mirrors _DATAFLOW_FIELDS in + * buckaroo/server/websocket_handler.py. */ export const DATAFLOW_STATE_FIELDS = ["post_processing", "cleaning_method", "quick_command_args"] as const; -export function touchesDataflow(_prev: BuckarooState | undefined, _next: BuckarooState | undefined): boolean { - return false; +/** Whether `next` differs from `prev` in a dataflow field. False when there is + * no earlier state to compare with. */ +export function touchesDataflow(prev: BuckarooState | undefined, next: BuckarooState | undefined): boolean { + if (prev === undefined || next === undefined) return false; + return DATAFLOW_STATE_FIELDS.some((field) => JSON.stringify(prev[field]) !== JSON.stringify(next[field])); } export interface StatsRequestOptions { + /** Ask for stats the server did not plan to compute. The "Compute summary + * stats" control sends this. */ force?: boolean; } -export function requestStats(_model: Pick, _opts?: StatsRequestOptions): boolean { - return false; +/** + * Send a `stats_request` for the stats_gen of the state the model shows. + * Returns false, and sends nothing, when its df_meta carries no stats.gen. + */ +export function requestStats(model: Pick, opts: StatsRequestOptions = {}): boolean { + const gen = (model.get("df_meta") as DFMeta | undefined)?.stats?.gen; + if (typeof gen !== "number") return false; + model.send({ type: "stats_request", stats_gen: gen, scope: "raw", ...(opts.force ? { force: true } : {}) }); + return true; } export interface OrchestratorOptions { model: StatsModel; + /** Lower bound on the delay before asking for a new state's stats. Default 200 ms. */ minDebounceMs?: number; + /** Upper bound on that delay. Default 3000 ms. */ maxDebounceMs?: number; + /** Multiplier on the last observed request time. Default 2. */ multiplier?: number; + /** Request time assumed until one has been observed. Default 250 ms. */ initialRequestMs?: number; + /** How long to wait for the first rows before asking anyway (an empty + * frame, the summary view, and a grid that never fetches send none). + * Default 1500 ms. */ firstPaintTimeoutMs?: number; } +type Timer = ReturnType; + export class StateOrchestrator { - constructor(_opts: OrchestratorOptions) {} + private readonly model: StatsModel; + private readonly minDebounceMs: number; + private readonly maxDebounceMs: number; + private readonly multiplier: number; + private readonly initialRequestMs: number; + private readonly firstPaintTimeoutMs: number; - start(): void {} + private started = false; + // The stats_gen being driven; undefined while nothing is pending. + private gen: number | undefined; + // A request is out and its reply has not been seen. + private inFlight = false; + // Delay before the next request: 0 for the first state and for each reply + // in a chain, the debounce after a state change. + private delayMs = 0; + // The state the model held at start has been read, so a pending state after + // it is a state change, whatever the first one was. + private began = false; + private sentAt = 0; + private lastRequestMs: number | undefined; + private requestTimer: Timer | undefined; + private paintTimer: Timer | undefined; + private syncQueued = false; + private seenMeta: unknown; + private seenDict: unknown; + private seenState: BuckarooState | undefined; - stop(): void {} + constructor(opts: OrchestratorOptions) { + this.model = opts.model; + this.minDebounceMs = opts.minDebounceMs ?? 200; + this.maxDebounceMs = opts.maxDebounceMs ?? 3000; + this.multiplier = opts.multiplier ?? 2; + this.initialRequestMs = opts.initialRequestMs ?? 250; + this.firstPaintTimeoutMs = opts.firstPaintTimeoutMs ?? 1500; + } + /** Start watching the model, and adopt the state it already holds. */ + start(): void { + if (this.started) return; + this.started = true; + this.seenMeta = this.model.get("df_meta"); + this.seenDict = this.model.get("df_data_dict"); + this.seenState = this.model.get("buckaroo_state"); + this.model.on("change:df_meta", this.onModelChange); + this.model.on("change:df_data_dict", this.onModelChange); + this.model.on("change:buckaroo_state", this.onState); + this.model.on("msg:custom", this.onMessage); + this.sync(); + this.began = true; + } + + /** Stop watching and cancel anything scheduled. Call on unmount. */ + stop(): void { + if (!this.started) return; + this.started = false; + this.model.off("change:df_meta", this.onModelChange); + this.model.off("change:df_data_dict", this.onModelChange); + this.model.off("change:buckaroo_state", this.onState); + this.model.off("msg:custom", this.onMessage); + this.standDown(); + this.began = false; + } + + /** + * The delay (ms) before asking for a new state's stats: the last observed + * request time times the multiplier, clamped to + * `[minDebounceMs, maxDebounceMs]`. A request that took longer means the + * server was busy, so the next one waits longer. + */ computeDebounce(): number { - return 0; + const raw = (this.lastRequestMs ?? this.initialRequestMs) * this.multiplier; + return Math.max(this.minDebounceMs, Math.min(this.maxDebounceMs, raw)); + } + + // A frame fires one change event per key, in the order the server wrote + // them (df_data_dict before df_meta), so the model is read once they have + // all landed, in a microtask, not at the first event. + private readonly onModelChange = (): void => { + if (this.syncQueued) return; + this.syncQueued = true; + void Promise.resolve().then(() => { + this.syncQueued = false; + if (this.started) this.sync(); + }); + }; + + private readonly onState = (next?: BuckarooState): void => { + const prev = this.seenState; + this.seenState = next ?? this.model.get("buckaroo_state"); + if (this.gen === undefined || !touchesDataflow(prev, this.seenState)) return; + this.begin(this.gen); + }; + + private readonly onMessage = (msg?: { type?: string }): void => { + if (msg?.type === "infinite_resp") this.markPainted(); + }; + + private sync(): void { + const meta = this.model.get("df_meta") as DFMeta | undefined; + const dict = this.model.get("df_data_dict"); + const metaChanged = meta !== this.seenMeta; + const dictChanged = dict !== this.seenDict; + this.seenMeta = meta; + this.seenDict = dict; + + const stats = meta?.stats; + if (stats?.status !== "pending" || typeof stats.gen !== "number") { + // Complete, not computed, an error, or a server that reports no stats. + this.standDown(); + } else if (stats.gen !== this.gen) { + this.begin(stats.gen); + } else if (this.inFlight && dictChanged && !metaChanged) { + // A new df_data_dict under the same df_meta is a stats_update that + // is not final (a full frame for this state replaces both). One + // request per reply: ask again. + this.inFlight = false; + this.noteRequestTime(); + this.arm(); + } + } + + // Wait for rows, then ask, for `gen`. The state the model starts with asks as + // soon as rows are up; every later one is a state change and waits out the + // debounce. + private begin(gen: number): void { + this.clearTimers(); + this.gen = gen; + this.inFlight = false; + this.delayMs = this.began ? this.computeDebounce() : 0; + this.paintTimer = setTimeout(() => { + this.paintTimer = undefined; + this.markPainted(); + }, this.firstPaintTimeoutMs); + } + + private standDown(): void { + if (this.inFlight) this.noteRequestTime(); + this.clearTimers(); + this.gen = undefined; + this.inFlight = false; + } + + // The first rows are up (or the wait for them timed out): ask. + private markPainted(): void { + if (this.paintTimer !== undefined) { + clearTimeout(this.paintTimer); + this.paintTimer = undefined; + } + this.arm(); + } + + private arm(): void { + if (this.inFlight || this.gen === undefined || this.requestTimer !== undefined) return; + this.requestTimer = setTimeout(() => { + this.requestTimer = undefined; + this.fire(); + }, this.delayMs); + } + + private fire(): void { + if (requestStats(this.model)) { + this.inFlight = true; + this.sentAt = Date.now(); + this.delayMs = 0; + } + } + + private noteRequestTime(): void { + this.lastRequestMs = Date.now() - this.sentAt; + } + + private clearTimers(): void { + if (this.requestTimer !== undefined) clearTimeout(this.requestTimer); + if (this.paintTimer !== undefined) clearTimeout(this.paintTimer); + this.requestTimer = undefined; + this.paintTimer = undefined; } } diff --git a/packages/buckaroo-js-core/src/server/WebSocketModel.ts b/packages/buckaroo-js-core/src/server/WebSocketModel.ts index e7e5798d2..0d249081d 100644 --- a/packages/buckaroo-js-core/src/server/WebSocketModel.ts +++ b/packages/buckaroo-js-core/src/server/WebSocketModel.ts @@ -14,9 +14,11 @@ * Server sends a JSON text frame (infinite_resp), then a binary frame (Parquet). * This class pairs them and emits "msg:custom" with (msg, [DataView]). * - * stats_update and stats_aborted frames go to `stats` (see StatsChannel). + * stats_update and stats_aborted frames go to `stats` (see StatsChannel), and + * `scheduler` asks for the stats a session defers (see StateOrchestrator). */ import { StatsChannel } from "./StatsChannel"; +import { StateOrchestrator } from "./StateOrchestrator"; export class WebSocketModel { private ws: WebSocket; @@ -25,11 +27,13 @@ export class WebSocketModel { private state: Record; private pendingChanges: Set = new Set(); readonly stats: StatsChannel; + readonly scheduler: StateOrchestrator; constructor(ws: WebSocket, initialState: Record) { this.state = { ...initialState }; this.ws = ws; this.stats = new StatsChannel(this); + this.scheduler = new StateOrchestrator({ model: this }); this.ws.onmessage = (event: MessageEvent) => { if (typeof event.data === "string") { @@ -67,6 +71,9 @@ export class WebSocketModel { } } }; + + // Idle unless df_meta.stats says pending, which no default session does. + this.scheduler.start(); } send(msg: any): void { diff --git a/packages/buckaroo-js-core/src/style/dcf-npm.css b/packages/buckaroo-js-core/src/style/dcf-npm.css index d64c928f5..f7f689283 100644 --- a/packages/buckaroo-js-core/src/style/dcf-npm.css +++ b/packages/buckaroo-js-core/src/style/dcf-npm.css @@ -435,6 +435,24 @@ div.dependent-tabs ul.tabs li.active { @media (prefers-reduced-motion: reduce) { .bk-status-inflight-dot { animation: none; opacity: 1; } } +/* Summary stats status in the status bar (rows-first c4). One line in a + * fixed-width column, whatever the status, so a status change moves nothing. */ +.bk-stats-status { + display: inline-flex; + align-items: center; + gap: 6px; + max-width: 100%; + font-size: 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.bk-stats-status button { + font-size: 11px; + line-height: 16px; + padding: 0 6px; + cursor: pointer; +} .status-bar .ag-center-cols-viewport { min-height: unset !important; } diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index d7ccfe48f..a9487912f 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -252,6 +252,12 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.save_changes(); }, [model]); + // The status bar's Compute summary stats button, shown while the server + // reports the stats as not computed. + const onComputeStats = React.useCallback(() => { + srt.requestStats(model, { force: true }); + }, [model]); + if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
Waiting for data... @@ -272,6 +278,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} />
); From d7eda0021359ff2d2b1a7d26226bb81dac30db49 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 05:27:04 -0400 Subject: [PATCH 10/18] test(client): failing tests for incremental stats requests with a columns hint (rows-first c4b) The scheduler and the Compute summary stats control should send stats_request with incremental: true and, when the grid has reported them, the columns it shows as the columns hint. A run is one request per partial reply, with the stats pending until the final reply, and a gen change mid-run stops it and starts one for the new gen. Jest: the scheduler's request shape, hint and chain against a fake model and against a real WebSocketModel (partial replies, a final that carries the complete all_stats, a whole-run final from a server that ignores the field, a gen change mid-run), requestStats, BuckarooView's callbacks, and the grid reporting its viewport columns. Playwright: the server spec (a harness answers stats_request over a real session, including a wide table whose hint follows a horizontal scroll) and the Storybook control. The existing request assertions gain incremental: true. The two unused on_visible_columns props are stubs so the tests compile. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 181 +++++++++++++++++- .../pw-tests/stats-scheduler-states.spec.ts | 29 ++- .../BuckarooInfiniteWidget.flash.test.tsx | 36 +++- .../src/components/BuckarooWidgetInfinite.tsx | 2 + .../DFViewerParts/DFViewerInfinite.test.tsx | 83 +++++++- .../DFViewerParts/DFViewerInfinite.tsx | 2 + .../src/server/BuckarooView.stats.test.tsx | 22 ++- .../src/server/StateOrchestrator.test.ts | 156 ++++++++++++++- .../stories/StatsSchedulerStates.stories.tsx | 51 ++++- 9 files changed, 536 insertions(+), 26 deletions(-) diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index af04c3f6e..dca23e6dd 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; import { loadSession, waitForGrid, getRowCount, getCellText } from './server-helpers'; import { execSync } from 'child_process'; import * as fs from 'fs'; @@ -506,9 +506,9 @@ test.describe('WebSocket data flow', () => { // tests put the pending state on a real session's first frame and answer // the page's stats_request themselves. Rows, config and the stats payload // all come from the real server. - async function loadBuckarooSession(request: any, sessionId: string) { + async function loadBuckarooSession(request: any, sessionId: string, path: string = csvPath) { const resp = await request.post(`${BASE}/load`, { - data: { session: sessionId, path: csvPath, mode: 'buckaroo' }, + data: { session: sessionId, path, mode: 'buckaroo' }, }); expect(resp.ok()).toBe(true); } @@ -563,8 +563,11 @@ test.describe('WebSocket data flow', () => { await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); - // One whole-run request, for the gen on the first frame, after rows had arrived. - expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw' }]); + // One request, for the gen on the first frame, after rows had arrived: an + // incremental step carrying the columns the grid shows (rows-first c4b). This + // harness answers it with a final reply carrying every stat, as a server + // that does not know the field does, and the page takes that as the end. + expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw', incremental: true, columns: ['a', 'b', 'c'] }]); expect(order.indexOf('infinite_resp')).toBeGreaterThanOrEqual(0); expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); }); @@ -613,7 +616,173 @@ test.describe('WebSocket data flow', () => { expect(requests).toEqual([]); await page.getByRole('button', { name: 'Compute summary stats' }).click(); - await expect.poll(() => requests).toEqual([{ type: 'stats_request', stats_gen: 4, scope: 'raw', force: true }]); + await expect.poll(() => requests).toEqual([ + { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, columns: ['a', 'b', 'c'] }, + ]); + }); + + // Rows-first c4b: a server on the unit path answers an incremental request + // with a partial stats_update and expects another request. The harness + // answers from the test, with the session's real all_stats as the final + // payload, so no xorq is needed. + async function routeStatsRequests( + page: Page, + session: string, + onRequest: (msg: any, send: (m: object) => void, h: { realStats: unknown; firstFrame: any }) => void, + ) { + const h = { requests: [] as any[], realStats: undefined as unknown, firstFrame: undefined as any }; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + server.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'initial_state' && h.realStats === undefined) { + h.realStats = msg.df_data_dict.all_stats; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } }; + h.firstFrame = msg; + ws.send(JSON.stringify(msg)); + return; + } + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + h.requests.push(msg); + onRequest(msg, (m) => ws.send(JSON.stringify(m)), h); + return; + } + } + server.send(message); + }); + }); + return h; + } + const partialUpdate = (gen: number, remaining: number) => ({ + type: 'stats_update', stats_gen: gen, scope: 'raw', tier: 'full', final: false, remaining, + payload: { format: 'json', layout: 'wide', data: [{ index: 'dtype', level_0: 'dtype', a: 'object', b: 'int64', c: 'float64' }] }, + elapsed_ms: 1, + }); + + test('partial updates: the page asks again for each one, keeps the stats pending, and completes on the final', async ({ page, request }) => { + const session = `ws-incremental-${Date.now()}`; + await loadBuckarooSession(request, session); + + let releaseFinal!: () => void; + const finalGate = new Promise((resolve) => { releaseFinal = resolve; }); + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length <= 2) { + send(partialUpdate(msg.stats_gen, 3 - harness.requests.length)); + return; + } + void finalGate.then(() => send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + })); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // Two partial replies, two more requests, and the third waits for its reply. + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(3); + const asked = { type: 'stats_request', stats_gen: 1, scope: 'raw', incremental: true, columns: ['a', 'b', 'c'] }; + expect(h.requests).toEqual([asked, asked, asked]); + // The partial rows are in, and the stats are still pending: no final yet. + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).toHaveText('int64', { timeout: 10_000 }); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); + + releaseFinal(); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); + }); + + test('the columns hint follows the grid viewport: a wide table sends the columns on screen, and the next request the ones scrolled to', async ({ page, request }) => { + const wideCsv = path.join(os.tmpdir(), `buckaroo_e2e_wide_${Date.now()}.csv`); + const names = Array.from({ length: 60 }, (_, i) => `column_${i}`); + fs.writeFileSync(wideCsv, [names.join(','), names.map((_, i) => String(i)).join(','), names.map((_, i) => String(i * 2)).join(',')].join('\n') + '\n'); + try { + const session = `ws-incremental-wide-${Date.now()}`; + await loadBuckarooSession(request, session, wideCsv); + + // Each partial reply waits for the test, so the grid can scroll between requests. + const replies: Array<() => void> = []; + const h = await routeStatsRequests(page, session, (msg, send) => { + replies.push(() => send(partialUpdate(msg.stats_gen, 5))); + }); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // The data grid's header cells (the status bar is a grid too). + const headerIds = () => page.locator('.df-viewer .ag-header-viewport .ag-header-cell[col-id]').evaluateAll( + (cells) => cells.map((c) => c.getAttribute('col-id') as string)); + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(1); + const first: string[] = h.requests[0].columns; + expect(first).toEqual(expect.any(Array)); + // Only the columns on screen, from the left edge: not all sixty. + expect(first.length).toBeGreaterThan(2); + expect(first.length).toBeLessThan(40); + expect(first[0]).toBe('a'); + // The grid may lay out more columns after the first rows, so the hint is + // the columns it showed at the time: the leading ones. + expect((await headerIds()).filter((id) => id !== 'index').slice(0, first.length)).toEqual(first); + + // Scroll to the right edge, then let the first reply through. The scroll + // event is sent by hand each time: the grid may not have been listening + // for the first one, and setting the same offset again sends none. + await expect.poll(async () => { + await page.locator('.df-viewer .ag-body-horizontal-scroll-viewport').evaluate((el) => { + el.scrollLeft = el.scrollWidth; + el.dispatchEvent(new Event('scroll')); + }); + return (await headerIds()).includes('a'); + }, { timeout: 10_000 }).toBe(false); + const scrolledTo = (await headerIds()).filter((id) => id !== 'index'); + replies[0](); + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(2); + const second: string[] = h.requests[1].columns; + expect(second).toEqual(expect.any(Array)); + expect(second).not.toContain('a'); + expect(second).toContain(scrolledTo[scrolledTo.length - 1]); + } finally { + cleanupFile(wideCsv); + } + }); + + test('a gen change in the middle of a run: the old gen is dropped and the page asks again for the new one', async ({ page, request }) => { + const session = `ws-incremental-gen-${Date.now()}`; + await loadBuckarooSession(request, session); + + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length === 1) { + send(partialUpdate(msg.stats_gen, 2)); + } else if (harness.requests.length === 2) { + // The server moved to gen 2 (a state change) with the run for gen 1 + // in flight; its late reply for gen 1 follows the frame. + const frame = { ...harness.firstFrame, df_meta: { ...harness.firstFrame.df_meta, stats: { status: 'pending', tier: 'schema', gen: 2 } } }; + send(frame); + send(partialUpdate(1, 1)); + } else { + send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + }); + } + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 15_000 }); + // Two requests for gen 1, then one for gen 2, and none for gen 1 after the change. + expect(h.requests.map((r) => r.stats_gen)).toEqual([1, 1, 2]); + expect(h.requests[2]).toMatchObject({ incremental: true, columns: ['a', 'b', 'c'] }); }); test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts index 64788ad1d..322ca5d3e 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -8,7 +8,9 @@ * grid's height changes only with the pinned area: placeholders hold the height * of the values that replace them, and omitted rows take theirs away. In * "not_computed" the status bar offers a Compute summary stats button, which - * sends stats_request {force: true}. + * sends stats_request {force: true, incremental: true} with the columns the grid + * shows (rows-first c4b). While the stats are pending, a stat row that has + * arrived for some columns leaves the cells of the others empty. */ import { test, expect, Page } from "@playwright/test"; import { waitForCells } from "./ag-pw-utils"; @@ -83,19 +85,40 @@ test("each stats status shows its own text in the status bar and nothing moves b expect(pageErrors).toEqual([]); }); -test("the Compute summary stats control sends stats_request with force", async ({ page }) => { +test("the Compute summary stats control sends an incremental stats_request with force and the visible columns", async ({ page }) => { await page.goto(STORY_URL); await waitForCells(page); await page.getByTestId("status-not_computed").click(); await page.getByRole("button", { name: "Compute summary stats" }).click(); + // The story's grid shows columns a and b (the index column is not a stats column). const sent = page.getByTestId("sent-log"); await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([ - { type: "stats_request", stats_gen: 7, scope: "raw", force: true }, + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, columns: ["a", "b"] }, ]); }); +test("a stat row that has arrived for one column leaves the other columns' cells empty while the stats are pending", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + await page.getByTestId("status-partial").click(); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByTestId("stats-status")).toContainText("Computing summary stats"); + + const meanCell = (col: string) => page.locator(`.ag-floating-top .ag-row[row-id="main-mean"] [col-id="${col}"]`); + await expect(meanCell("a")).toHaveText("2", { timeout: 10_000 }); + // Column b has no mean yet: the cell is empty. + await expect(meanCell("b")).toHaveText(""); + // dtype came with the schema, for both columns. + await expect(page.locator('.ag-floating-top .ag-row[row-id="main-dtype"] [col-id="b"]')).toHaveText("object"); + + // The stats arrive for column b too: the cell fills in. + await page.getByTestId("status-complete").click(); + await expect(meanCell("b")).toHaveText("N/A", { timeout: 10_000 }); +}); + test("no control is offered while the stats are pending, in error or complete", async ({ page }) => { await page.goto(STORY_URL); await waitForCells(page); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index e74a13b95..b56524bb9 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -7,7 +7,7 @@ * Tests assert CURRENT behavior on main (Option A in docs/rerender-test-plan.md). * Tests tagged "[captures current flash]" are tracking pain, not validating it. */ -import { render, act } from "@testing-library/react"; +import { render, act, waitFor } from "@testing-library/react"; import { BuckarooInfiniteWidget } from "./BuckarooWidgetInfinite"; import { KeyAwareSmartRowCache } from "./DFViewerParts/SmartRowCache"; import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridSpy"; @@ -893,3 +893,37 @@ describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", }); }); }); + +// Rows-first c4b: the widget hands the grid the callback that reports which +// columns it shows, which the stats scheduler sends as its hint. +describe("BuckarooInfiniteWidget on_visible_columns (rows-first c4b)", () => { + it("passes the callback to the grid, which reports the data columns of its viewport through it", async () => { + const onVisible = jest.fn(); + const { container } = render( + , + ); + + // The grid is mocked, so give it the header cell AG Grid would render. + container.querySelector(".theme-hanger")!.insertAdjacentHTML( + "beforeend", + '
', + ); + const gridProps = getSpyCalls().lastProps; + expect(typeof gridProps.onVirtualColumnsChanged).toBe("function"); + gridProps.onVirtualColumnsChanged({}); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith(["a"])); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index e83cb3250..a7f237be5 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -152,6 +152,8 @@ export function BuckarooInfiniteWidget({ /** Sends a forced stats_request. Server entry points pass it; the Jupyter * widget does not, and then the status bar offers no control. */ on_compute_stats?: () => void; + /** Called with the data columns the grid shows, now and whenever they change. */ + on_visible_columns?: (columns: string[]) => void; operations: Operation[]; on_operations: (ops: Operation[]) => void; operation_results: OperationResult; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx index 6e77c3cb8..c69836244 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx @@ -1,9 +1,13 @@ -import { render } from "@testing-library/react"; +import { render, waitFor } from "@testing-library/react"; import { DFViewerInfinite } from "./DFViewerInfinite"; import { DFViewerConfig } from "./DFWhole"; const setGridOptionMock = jest.fn(); let latestAgGridProps: any = null; +// The ids of the columns the mocked grid has in its viewport, rendered as the +// header cells AG Grid renders for them (jest.mock factories may read +// variables whose names start with "mock"). +let mockVirtualColumnIds: string[] = []; jest.mock("ag-grid-react", () => { const React = require("react"); @@ -24,7 +28,15 @@ jest.mock("ag-grid-react", () => { }); }, [props]); - return
; + return ( +
+
+ {mockVirtualColumnIds.map((id) => ( +
+ ))} +
+
+ ); }), }; }); @@ -47,6 +59,7 @@ describe("DFViewerInfinite", () => { beforeEach(() => { setGridOptionMock.mockClear(); latestAgGridProps = null; + mockVirtualColumnIds = []; }); it("renders error_info and custom class name", () => { @@ -131,3 +144,69 @@ describe("DFViewerInfinite", () => { expect(latestAgGridProps.datasource.rowCount).toBe(50); }); }); + +// Rows-first c4b: the stats scheduler asks the server for the units covering +// the columns the user is looking at first, so the grid reports them. +describe("DFViewerInfinite on_visible_columns (rows-first c4b)", () => { + const wideConfig: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + { col_name: "c", header_name: "c", displayer_args: { displayer: "obj" } }, + { col_name: "d", header_name: "d", displayer_args: { displayer: "obj" } }, + ], + }; + const viewer = (onVisible?: (columns: string[]) => void) => ( + + ); + // The grid renders its header cells a frame after the event that moves them. + const aFrame = () => new Promise((resolve) => requestAnimationFrame(resolve)); + + beforeEach(() => { + mockVirtualColumnIds = []; + }); + + it("reports the data columns in the grid's viewport once the grid is ready, not the index column", async () => { + mockVirtualColumnIds = ["index", "a", "b"]; + const onVisible = jest.fn(); + render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith(["a", "b"])); + await aFrame(); + expect(onVisible).toHaveBeenCalledTimes(1); + }); + + it("reports again when the viewport's columns change, and not when they stay the same", async () => { + mockVirtualColumnIds = ["index", "a", "b"]; + const onVisible = jest.fn(); + const { rerender } = render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledTimes(1)); + expect(typeof latestAgGridProps.onVirtualColumnsChanged).toBe("function"); + + // The same columns again, as a resize can produce. + latestAgGridProps.onVirtualColumnsChanged({}); + await aFrame(); + expect(onVisible).toHaveBeenCalledTimes(1); + + // The user scrolls right: the grid swaps its header cells and says so. + mockVirtualColumnIds = ["index", "c", "d"]; + rerender(viewer(onVisible)); + latestAgGridProps.onVirtualColumnsChanged({}); + await waitFor(() => expect(onVisible).toHaveBeenCalledTimes(2)); + expect(onVisible).toHaveBeenLastCalledWith(["c", "d"]); + }); + + it("reports an empty list when no data column is in the viewport (a grid that is not laid out yet)", async () => { + mockVirtualColumnIds = ["index"]; + const onVisible = jest.fn(); + render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith([])); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx index 4f4465ccb..ccd14add6 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx @@ -183,6 +183,8 @@ export function DFViewerInfinite({ // placeholder row; when "not_computed" it is omitted. Undefined behaves // as "complete". stats_status?: StatsStatus; + // Called with the data columns in the grid's viewport, now and whenever they change. + on_visible_columns?: (columns: string[]) => void; }) { /* The idea is to do some pre-setup here for diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx index 9bb495202..31c187ebd 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx @@ -4,7 +4,8 @@ * While df_meta.stats says the stats are not computed, the status bar offers a * control that asks the server for them. BuckarooView hands the widget the * callback, and the callback sends `stats_request {force: true}` through - * whatever IModel the host gave it. + * whatever IModel the host gave it. The grid reports the columns it shows, and + * BuckarooView keeps them on the model, where the request reads its hint. */ import { render, cleanup, act } from "@testing-library/react"; import { BuckarooView } from "./BuckarooView"; @@ -64,7 +65,24 @@ describe("BuckarooView on_compute_stats (rows-first c4)", () => { expect(typeof props().on_compute_stats).toBe("function"); props().on_compute_stats(); - expect(sent).toEqual([{ type: "stats_request", stats_gen: 9, scope: "raw", force: true }]); + // A time-boxed step, like the scheduler's (rows-first c4b). + expect(sent).toEqual([{ type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true }]); + }); + + it("records the columns the grid shows on the model, and the control's request carries them as its hint", async () => { + const { model, sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_visible_columns).toBe("function"); + + props().on_visible_columns(["a", "b"]); + expect(model.get("visible_columns")).toEqual(["a", "b"]); + props().on_compute_stats(); + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, columns: ["a", "b"] }, + ]); }); it("sends nothing when the model's df_meta carries no stats.gen", async () => { diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 2e7f92eda..4d1494ccf 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -65,8 +65,10 @@ const bState = (over: Record = {}): any => ({ df_display: "main", show_commands: false, ...over, }); +// Every request is a time-boxed step (rows-first c4b); the earlier phases sent +// the whole run, so this helper had no `incremental`. const request = (gen: number, extra: Record = {}) => ({ - type: "stats_request", stats_gen: gen, scope: "raw", ...extra, + type: "stats_request", stats_gen: gen, scope: "raw", incremental: true, ...extra, }); const makeModel = (stats?: Record) => @@ -231,6 +233,66 @@ describe("one request per reply", () => { }); }); +describe("incremental requests (rows-first c4b)", () => { + const sendFirst = async (model: FakeModel) => { + start(model); + rowsArrived(model); + await tick(); + }; + + it("asks for a time-boxed step, with no columns hint while the grid has not reported its columns", async () => { + const model = makeModel(pending(3)); + await sendFirst(model); + expect(model.sent).toEqual([{ type: "stats_request", stats_gen: 3, scope: "raw", incremental: true }]); + expect(model.sent[0]).not.toHaveProperty("columns"); + expect(model.sent[0]).not.toHaveProperty("force"); + }); + + it("sends the columns the grid shows as the hint", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["b", "c"]; + await sendFirst(model); + expect(model.sent).toEqual([request(3, { columns: ["b", "c"] })]); + }); + + it("sends no hint for an empty list of visible columns", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = []; + await sendFirst(model); + expect(model.sent).toEqual([request(3)]); + }); + + it("reads the hint again for each request of a run, so a column scrolled into view goes out next", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + + model.set("visible_columns", ["e", "f"]); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + model.set("df_data_dict", dict([statRow("mean"), statRow("std")])); + await tick(); + expect(model.sent).toEqual([ + request(3, { columns: ["a", "b"] }), + request(3, { columns: ["e", "f"] }), + request(3, { columns: ["e", "f"] }), + ]); + }); + + it("sends the same shape for the first request of the next gen after a state change", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + model.frame({ df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([request(3, { columns: ["a", "b"] }), request(4, { columns: ["a", "b"] })]); + }); +}); + describe("a state change", () => { const pendingAt = async (gen: number) => { const model = makeModel(pending(gen)); @@ -436,6 +498,13 @@ describe("requestStats", () => { expect(model.sent).toEqual([request(7, { force: true })]); }); + it("carries the columns the grid shows, for a forced request too", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + model.state.visible_columns = ["a", "b"]; + expect(requestStats(model, { force: true })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true, columns: ["a", "b"] })]); + }); + it("sends nothing when df_meta carries no stats.gen", () => { const model = makeModel(); expect(requestStats(model)).toBe(false); @@ -512,12 +581,13 @@ describe("wired into WebSocketModel", () => { } } - const update = (gen: number, stat: string, final: boolean) => ({ + const update = (gen: number, stat: string, final: boolean, remaining?: number) => ({ type: "stats_update", stats_gen: gen, scope: "raw", tier: "full", final, + ...(remaining === undefined ? {} : { remaining }), payload: { format: "json", layout: "wide", data: [{ index: stat, level_0: stat, a: 1 }] }, elapsed_ms: 5, }); @@ -556,6 +626,88 @@ describe("wired into WebSocketModel", () => { expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std"]); }); + it("walks partial replies to the final one: one request per reply, the stats pending until the last", async () => { + const { ws, model } = makeSocketModel(pending(3)); + model.set("visible_columns", ["a"]); + const pendingMeta = model.get("df_meta"); + rowsFromServer(ws); + await tick(); + const asked = request(3, { columns: ["a"] }); + expect(ws.sent).toEqual([asked]); + + for (const [i, stat] of ["mean", "std", "max"].entries()) { + ws.deliver(update(3, stat, false, 3 - i)); + await tick(); + expect(ws.sent).toEqual(Array(i + 2).fill(asked)); + // The merge adds the rows, and the stats stay pending: the same + // df_meta, so the placeholders and the loading text stay. + expect(model.get("df_meta")).toBe(pendingMeta); + expect(model.get("df_meta").stats.status).toBe("pending"); + } + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std", "max"]); + + // The final reply carries the complete all_stats. + ws.deliver({ + ...update(3, "min", true, 0), + payload: { + format: "json", + layout: "wide", + data: ["dtype", "mean", "std", "max", "min"].map((stat) => ({ index: stat, level_0: stat, a: stat === "dtype" ? "int64" : 2 })), + }, + }); + await tick(10_000); + expect(ws.sent).toHaveLength(4); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std", "max", "min"]); + expect(model.get("df_data_dict").all_stats[1].a).toBe(2); + }); + + it("ends the run at a whole-run final from a server that ignores the incremental field", async () => { + const { ws, model } = makeSocketModel(pending(3)); + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + // An older server runs every unit and answers final, with no `remaining`. + ws.deliver(update(3, "mean", true)); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + expect(model.get("df_meta").stats.status).toBe("complete"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean"]); + }); + + it("stops a run when the gen changes, drops its late replies, and starts one for the new gen", async () => { + const { ws, model } = makeSocketModel(pending(3)); + model.set("visible_columns", ["a"]); + rowsFromServer(ws); + await tick(); + ws.deliver(update(3, "mean", false, 2)); + await tick(); + expect(ws.sent).toEqual([request(3, { columns: ["a"] }), request(3, { columns: ["a"] })]); + + // A state change moved the server to gen 4 with the run for gen 3 in flight. + ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); + ws.deliver(update(3, "std", false, 1)); + await tick(100); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_data_dict").all_stats).toEqual([]); + + rowsFromServer(ws); + await tick(DEBOUNCE); + expect(ws.sent[2]).toEqual(request(4, { columns: ["a"] })); + + // The new run chains like the old one did. + ws.deliver(update(4, "mean", false, 1)); + await tick(); + expect(ws.sent).toHaveLength(4); + expect(ws.sent[3]).toEqual(request(4, { columns: ["a"] })); + ws.deliver(update(4, "std", true, 0)); + await tick(10_000); + expect(ws.sent).toHaveLength(4); + expect(model.get("df_meta").stats.status).toBe("complete"); + }); + it("moves on to the next gen when a state change frame arrives", async () => { const { ws } = makeSocketModel(pending(3)); rowsFromServer(ws); diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx index 2644d4dc7..4acc863e4 100644 --- a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -9,9 +9,12 @@ * - "error" "Stats error: ", pinned keys omitted * - "complete" "Summary stats ready", the values in place * - * The buttons switch the status. The button in the status bar sends - * `stats_request {force: true}` through a fake model, which logs what it was - * asked to send. Used by stats-scheduler-states.spec.ts. + * The buttons switch the status. The "partial" button is "pending" with the + * stats of one column in, as a partial stats_update leaves them. The button in + * the status bar sends `stats_request {force: true, incremental: true}` through + * a fake model, which logs what it was asked to send and answers + * `visible_columns` with the columns the grid reports. Used by + * stats-scheduler-states.spec.ts. */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; @@ -40,6 +43,12 @@ const completeStats: DFData = [ { index: "mean", a: 2, b: "N/A" }, ]; +// dtype comes with the schema; mean has arrived for column a only. +const partialStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 2 }, +]; + const viewerConfig: DFViewerConfig = { column_config: [ { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, @@ -67,6 +76,8 @@ const commandConfig: CommandConfigT = { argspecs: {}, defaultArgs: {} }; const StatsSchedulerStatesInner: React.FC = () => { const [status, setStatus] = useState("pending"); + const [partial, setPartial] = useState(false); + const [visibleColumns, setVisibleColumns] = useState([]); const [sent, setSent] = useState([]); const [buckarooState, setBuckarooState] = useState({ sampled: false, @@ -95,14 +106,15 @@ const StatsSchedulerStatesInner: React.FC = () => { [status], ); - // The model the control sends through: it answers get("df_meta") from the - // story's state and logs what it is asked to send. + // The model the control sends through: it answers get("df_meta") and + // get("visible_columns") from the story's state and logs what it is asked to + // send. const model = useMemo( () => ({ - get: (key: string) => (key === "df_meta" ? df_meta : undefined), + get: (key: string) => (key === "df_meta" ? df_meta : key === "visible_columns" ? visibleColumns : undefined), send: (msg: unknown) => setSent((log) => [...log, msg]), }), - [df_meta], + [df_meta, visibleColumns], ); const src = useMemo(() => { @@ -120,20 +132,38 @@ const StatsSchedulerStatesInner: React.FC = () => { const df_data_dict = useMemo( () => ({ main: [] as DFData, - all_stats: status === "complete" ? completeStats : ([] as DFData), + all_stats: status === "complete" ? completeStats : partial ? partialStats : ([] as DFData), empty: [] as DFData, }), - [status], + [status, partial], ); return (
{STATUSES.map((s) => ( - ))} + df_meta.stats.status = {status}
@@ -150,6 +180,7 @@ const StatsSchedulerStatesInner: React.FC = () => { buckaroo_options={buckarooOptions} src={src} on_compute_stats={() => requestStats(model, { force: true })} + on_visible_columns={setVisibleColumns} />

From 5d1e0467b1d15f5a6c395d3c6a12a1e1b2ffea51 Mon Sep 17 00:00:00 2001
From: Paddy Mullen 
Date: Sun, 4 Oct 2026 05:48:03 -0400
Subject: [PATCH 11/18] test(client): let the grid's column report land before
 the wide-table test releases a reply (rows-first c4b)

The grid reports the columns it shows a frame after it renders them, and
the wide-table test released the reply as soon as the header cells had
changed, so on a loaded machine the next request could go out with the
earlier hint. Wait two animation frames first. No assertion changes.

Co-Authored-By: Claude Sonnet 5.5 
---
 packages/buckaroo-js-core/pw-tests/server.spec.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts
index dca23e6dd..16e534a01 100644
--- a/packages/buckaroo-js-core/pw-tests/server.spec.ts
+++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts
@@ -744,6 +744,9 @@ test.describe('WebSocket data flow', () => {
         return (await headerIds()).includes('a');
       }, { timeout: 10_000 }).toBe(false);
       const scrolledTo = (await headerIds()).filter((id) => id !== 'index');
+      // The grid reports its columns a frame after it renders them: let that pass
+      // before the next request can be sent.
+      await page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))));
       replies[0]();
       await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(2);
       const second: string[] = h.requests[1].columns;

From e5649605ee0d35561054bed0a45e05994de6227a Mon Sep 17 00:00:00 2001
From: Paddy Mullen 
Date: Sun, 4 Oct 2026 05:48:13 -0400
Subject: [PATCH 12/18] feat(client): scheduler and control send incremental
 stats requests with a columns hint (rows-first c4b)

requestStats, which the scheduler and the Compute summary stats control both
use, now sends stats_request with incremental: true and, when the grid has
reported them, the columns it shows as the columns hint, read from the
model's visible_columns key when the request goes out. A server on the unit
path answers with a partial stats_update per request, the scheduler asks
again for each, StatsChannel merges them without leaving "pending", and the
final reply with the complete all_stats ends the run. A server that ignores
the fields answers with a whole-run final, which ends it the same way.

DFViewerInfinite takes on_visible_columns and reports the data columns that
have a header cell in the grid's viewport on grid ready and on
onVirtualColumnsChanged. BuckarooInfiniteWidget forwards it, and BuckarooView
and the standalone page store the list on the model with setVisibleColumns.
The columns are read from the rendered header cells because the grid's column
API is not registered, and registering it would switch on the column-state
calls that are inert today.

Co-Authored-By: Claude Sonnet 5.5 
---
 .../src/components/BuckarooWidgetInfinite.tsx |  2 +
 .../DFViewerParts/DFViewerInfinite.tsx        | 40 ++++++++++++++++
 packages/buckaroo-js-core/src/index.ts        |  4 +-
 .../src/server/BuckarooView.tsx               |  9 +++-
 .../src/server/StateOrchestrator.ts           | 46 ++++++++++++++++---
 packages/js/standalone.tsx                    |  7 +++
 6 files changed, 99 insertions(+), 9 deletions(-)

diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx
index a7f237be5..508183914 100644
--- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx
+++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx
@@ -135,6 +135,7 @@ export function BuckarooInfiniteWidget({
         df_display_args,
         df_meta,
         on_compute_stats,
+        on_visible_columns,
         operations,
         on_operations,
         operation_results,
@@ -365,6 +366,7 @@ export function BuckarooInfiniteWidget({
                         view_name={buckaroo_state.df_display}
                         data_key={cDisp.data_key}
                         stats_status={getStatsStatus(df_meta)}
+                        on_visible_columns={on_visible_columns}
                     />
                 
{buckaroo_state.show_commands ? ( diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx index ccd14add6..8ad80e038 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx @@ -4,6 +4,7 @@ import { useEffect, useRef, } from "react"; +import type { RefObject } from "react"; import * as _ from "lodash-es"; import { DFData, DFDataRow, DFViewerConfig, SDFT } from "./DFWhole"; @@ -156,6 +157,7 @@ export function DFViewerInfinite({ view_name, data_key, stats_status, + on_visible_columns, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -193,6 +195,8 @@ export function DFViewerInfinite({ //console.log("137renderStartTime"); return Date.now(); } , []); + // The element that holds the grid, where the columns in its viewport are read. + const hangerRef = useRef(null); const totalRows=5; const compConfig = df_viewer_config?.component_config; @@ -231,6 +235,7 @@ export function DFViewerInfinite({
{error_info ?
{error_info}
: null}
) @@ -264,6 +271,8 @@ export function DFViewerInfiniteInner({ view_name, data_key, stats_status, + on_visible_columns, + grid_container, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -281,6 +290,8 @@ export function DFViewerInfiniteInner({ view_name?: string; data_key?: string; stats_status?: StatsStatus; + on_visible_columns?: (columns: string[]) => void; + grid_container?: RefObject; }) { /* const lastProps = useRef(null); @@ -432,6 +443,33 @@ export function DFViewerInfiniteInner({ [data_wrapper, gridOptions, hs]); // Use grid API to set pinned rows imperatively, avoiding a full React prop update that can flash const gridRef = useRef | null>(null); + // Report the data columns in the grid's viewport (the index column is not + // one) when the grid is ready and whenever they change. They are read + // from the header cells the grid has rendered, since the grid's own + // column API (getAllDisplayedVirtualColumns) is not registered here and + // registering it would also switch on the column-state calls above. The + // cells follow the event by a frame, so the read waits for one. An + // unchanged list is not reported again, so a resize or a vertical + // scroll costs nothing. + const dataFields = useMemo( + () => new Set(df_viewer_config.column_config.map(getFieldVal)), + [df_viewer_config.column_config], + ); + const lastVisibleRef = useRef(undefined); + const reportVisibleColumns = useCallback(() => { + if (on_visible_columns === undefined) return; + window.requestAnimationFrame(() => { + const container = grid_container?.current; + if (!container) return; + const ids = Array.from(container.querySelectorAll(".ag-header-viewport .ag-header-cell[col-id]")) + .map((cell) => cell.getAttribute("col-id") ?? "") + .filter((id) => dataFields.has(id)); + const sig = JSON.stringify(ids); + if (sig === lastVisibleRef.current) return; + lastVisibleRef.current = sig; + on_visible_columns(ids); + }); + }, [on_visible_columns, dataFields, grid_container]); // Keep latest pinned rows in a ref so onGridReady can apply them once API is ready const topRowsRef = useRef(null); // Build a content signature based on visible fields and pinned values, @@ -615,7 +653,9 @@ export function DFViewerInfiniteInner({ // Ensure pinned rows are applied once API is ready params.api.setGridOption('pinnedTopRowData', topRowsRef.current || []); } catch (_e) {} + reportVisibleColumns(); }} + onVirtualColumnsChanged={reportVisibleColumns} context={{ outside_df_params, ...extra_context }} > ); diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 4a56e1810..85923ae9d 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -20,7 +20,7 @@ import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { withStatsCapability } from './server/StatsChannel'; -import { StateOrchestrator, requestStats } from './server/StateOrchestrator'; +import { StateOrchestrator, requestStats, setVisibleColumns } from './server/StateOrchestrator'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -67,6 +67,7 @@ export default { withStatsCapability, StateOrchestrator, requestStats, + setVisibleColumns, }; // Named exports for direct imports @@ -100,6 +101,7 @@ export { withStatsCapability, StateOrchestrator, requestStats, + setVisibleColumns, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index d69eab528..88788cac1 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -10,7 +10,7 @@ import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; -import { requestStats } from "./StateOrchestrator"; +import { requestStats, setVisibleColumns } from "./StateOrchestrator"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -291,6 +291,12 @@ export function BuckarooView({ requestStats(model, { force: true }); }, [model]); + // The columns the grid shows, kept on the model for stats requests to carry + // as their hint. + const onVisibleColumns = React.useCallback((columns: string[]) => { + setVisibleColumns(model, columns); + }, [model]); + // gridUtils honors component_config.layoutType. Stamp it per entry so the // prop wins when provided. autoHeight=undefined → server value left intact. const effectiveDisplayArgs = React.useMemo( @@ -330,6 +336,7 @@ export function BuckarooView({ buckaroo_options={buckarooOptions} src={src} on_compute_stats={onComputeStats} + on_visible_columns={onVisibleColumns} /> ) : ( JSON.stringify(prev[field]) !== JSON.stringify(next[field])); } +/** The model key that holds the grid's column names currently on screen. */ +export const VISIBLE_COLUMNS_KEY = "visible_columns"; + +/** Record the columns the grid shows, for the next `stats_request` to carry as + * its `columns` hint. An empty list means none are known. */ +export function setVisibleColumns(model: Pick, columns: string[]): void { + model.set(VISIBLE_COLUMNS_KEY, columns); +} + export interface StatsRequestOptions { /** Ask for stats the server did not plan to compute. The "Compute summary * stats" control sends this. */ @@ -54,13 +77,22 @@ export interface StatsRequestOptions { } /** - * Send a `stats_request` for the stats_gen of the state the model shows. + * Send a time-boxed `stats_request` for the stats_gen of the state the model + * shows, with the columns the grid shows as the hint when they are known. * Returns false, and sends nothing, when its df_meta carries no stats.gen. */ export function requestStats(model: Pick, opts: StatsRequestOptions = {}): boolean { const gen = (model.get("df_meta") as DFMeta | undefined)?.stats?.gen; if (typeof gen !== "number") return false; - model.send({ type: "stats_request", stats_gen: gen, scope: "raw", ...(opts.force ? { force: true } : {}) }); + const columns = model.get(VISIBLE_COLUMNS_KEY) as string[] | undefined; + model.send({ + type: "stats_request", + stats_gen: gen, + scope: "raw", + incremental: true, + ...(Array.isArray(columns) && columns.length > 0 ? { columns } : {}), + ...(opts.force ? { force: true } : {}), + }); return true; } diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index a9487912f..bb59ad2ea 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -258,6 +258,12 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { srt.requestStats(model, { force: true }); }, [model]); + // The columns the grid shows, kept on the model for stats requests to carry + // as their hint. + const onVisibleColumns = React.useCallback((columns: string[]) => { + srt.setVisibleColumns(model, columns); + }, [model]); + if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
Waiting for data... @@ -279,6 +285,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { buckaroo_options={buckarooOptions} src={src} on_compute_stats={onComputeStats} + on_visible_columns={onVisibleColumns} />
); From 01609fa3dcbcd83ea40f0affba145fb37c1e7152 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 06:35:03 -0400 Subject: [PATCH 13/18] test(client): failing tests for not_computed states, the compute control and demand units (rows-first c5) Tests for plan 3 phase 5, with the stubs they compile against (empty StatsEmptyState, forceStats and the df_meta.stats helpers return nothing): - a not_computed session renders typed columns with no blank pinned rows, the summary view shows an empty state with the compute control, and for reason "ceiling" a message and no control - auto_request false leaves the scheduler idle except for demand_columns, which it asks for as one scoped request - the control sends stats_request {force, tier} with the tier taken from requestable (scalar before full), and a per-column form names the columns; the scheduler continues a forced run - a final stats_update with status not_computed and reason ceiling ends in the ceiling message - the client advertises ?caps=stats_update,stats_ondemand from both wiring copies Existing assertions of the capability URL and of the control's request change because the request shape changes by design. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 89 ++++- .../pw-tests/stats-scheduler-states.spec.ts | 109 ++++- .../BuckarooInfiniteWidget.flash.test.tsx | 113 +++++- .../src/components/BuckarooWidgetInfinite.tsx | 2 +- .../src/components/StatsEmptyState.test.tsx | 148 +++++++ .../src/components/StatsEmptyState.tsx | 26 ++ .../src/components/StatusBar.stats.test.tsx | 75 +++- .../src/components/StatusBar.tsx | 4 +- .../src/components/WidgetTypes.test.ts | 116 ++++++ .../src/components/WidgetTypes.tsx | 31 ++ .../server/BuckarooServerView.caps.test.tsx | 11 +- .../src/server/BuckarooView.stats.test.tsx | 53 ++- .../src/server/StateOrchestrator.test.ts | 377 +++++++++++++++++- .../src/server/StateOrchestrator.ts | 12 + .../src/server/StatsChannel.test.ts | 101 ++++- .../src/server/StatsChannel.ts | 6 +- .../stories/StatsSchedulerStates.stories.tsx | 175 +++++++- 17 files changed, 1407 insertions(+), 41 deletions(-) create mode 100644 packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx create mode 100644 packages/buckaroo-js-core/src/components/StatsEmptyState.tsx create mode 100644 packages/buckaroo-js-core/src/components/WidgetTypes.test.ts diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index 16e534a01..524a4a760 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -485,12 +485,14 @@ test.describe('WebSocket data flow', () => { expect(await getCellText(page, COL.name, 0)).toBe('Alice'); }); - test('standalone page advertises stats_update on its WebSocket URL', async ({ page, request }) => { + test('standalone page advertises stats_update and stats_ondemand on its WebSocket URL', async ({ page, request }) => { const session = `ws-caps-${Date.now()}`; await loadSession(request, session, csvPath); // The server reads capabilities from ?caps= at open, before the client // sends anything, so the page has to put them on the URL it connects to. + // stats_ondemand (rows-first c5) says the page can show a session whose + // stats are not computed and send tiered requests. const socketUrls: string[] = []; page.on('websocket', (socket) => socketUrls.push(socket.url())); await page.goto(`${BASE}/s/${session}`); @@ -498,7 +500,7 @@ test.describe('WebSocket data flow', () => { const wsUrls = socketUrls.filter((u) => u.includes(`/ws/${session}`)); expect(wsUrls).toHaveLength(1); - expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update'); + expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update,stats_ondemand'); }); // Rows-first c4: the standalone page's scheduler. No session the server can @@ -616,8 +618,10 @@ test.describe('WebSocket data flow', () => { expect(requests).toEqual([]); await page.getByRole('button', { name: 'Compute summary stats' }).click(); + // A server that names no requestable tier can be asked for full; the + // whole-table request names no columns (rows-first c5). await expect.poll(() => requests).toEqual([ - { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, columns: ['a', 'b', 'c'] }, + { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'full' }, ]); }); @@ -629,6 +633,7 @@ test.describe('WebSocket data flow', () => { page: Page, session: string, onRequest: (msg: any, send: (m: object) => void, h: { realStats: unknown; firstFrame: any }) => void, + firstStats: object = { status: 'pending', tier: 'schema', gen: 1 }, ) { const h = { requests: [] as any[], realStats: undefined as unknown, firstFrame: undefined as any }; await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { @@ -639,7 +644,7 @@ test.describe('WebSocket data flow', () => { if (msg.type === 'initial_state' && h.realStats === undefined) { h.realStats = msg.df_data_dict.all_stats; msg.df_data_dict.all_stats = []; - msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } }; + msg.df_meta = { ...msg.df_meta, stats: firstStats }; h.firstFrame = msg; ws.send(JSON.stringify(msg)); return; @@ -788,6 +793,82 @@ test.describe('WebSocket data flow', () => { expect(h.requests[2]).toMatchObject({ incremental: true, columns: ['a', 'b', 'c'] }); }); + // Rows-first c5: a session the server's policy left without stats. The first + // frame carries the policy fields a capable client is sent, and the harness + // answers the page's requests as a server that serves tiers would. + const policyStats = (over: object = {}) => ({ + status: 'not_computed', tier: 'schema', gen: 4, reason: 'size', tier_target: 'schema', + estimate: { rows: 12_400_000, cols: 3 }, auto_request: false, requestable: ['scalar', 'full'], ...over, + }); + + test('a policy session: the page asks for nothing, the summary view says why, and the control asks for the basic tier', async ({ page, request }) => { + const session = `ws-policy-${Date.now()}`; + await loadBuckarooSession(request, session); + const h = await routeStatsRequests(page, session, () => {}, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed'); + // Nothing is asked for on its own: auto_request is false and no column needs stats. + await page.waitForTimeout(2500); + expect(h.requests).toEqual([]); + + // The summary view has no stats to list: it says why, with the control. + await page.locator('.status-bar select').first().selectOption('summary'); + const empty = page.getByTestId('stats-empty-state'); + await expect(empty).toBeVisible(); + await expect(empty).toContainText('12.4M rows x 3 columns'); + await empty.getByRole('button', { name: 'Compute basic stats' }).click(); + await expect.poll(() => h.requests).toEqual([ + { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }, + ]); + }); + + test('a policy session: a request over the ceiling is answered with the ceiling message, and the page offers nothing more', async ({ page, request }) => { + const session = `ws-ceiling-${Date.now()}`; + await loadBuckarooSession(request, session); + const h = await routeStatsRequests(page, session, (msg, send) => { + send({ type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, status: 'not_computed', reason: 'ceiling' }); + }, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + + await expect(page.getByTestId('stats-status')).toContainText('Summary stats unavailable'); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-reason', 'ceiling'); + await expect(page.getByRole('button', { name: 'Compute summary stats' })).toHaveCount(0); + await page.locator('.status-bar select').first().selectOption('summary'); + await expect(page.getByTestId('stats-empty-state')).toContainText('over the size limit'); + await expect(page.getByTestId('stats-empty-state').getByRole('button')).toHaveCount(0); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(1); + }); + + test('a policy session: a forced run is asked for again after each partial reply and ends with the final one', async ({ page, request }) => { + const session = `ws-forced-${Date.now()}`; + await loadBuckarooSession(request, session); + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length < 3) { + send({ ...partialUpdate(msg.stats_gen, 3 - harness.requests.length), tier: 'scalar' }); + } else { + send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'scalar', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + }); + } + }, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + const forced = { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }; + expect(h.requests).toEqual([forced, forced, forced]); + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); + }); + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { const session = `ws-nosched-${Date.now()}`; await loadBuckarooSession(request, session); diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts index 322ca5d3e..443c99f76 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -11,6 +11,12 @@ * sends stats_request {force: true, incremental: true} with the columns the grid * shows (rows-first c4b). While the stats are pending, a stat row that has * arrived for some columns leaves the cells of the others empty. + * + * The NotComputed story (rows-first c5) is a session the server's policy left + * without stats. Its main view renders typed columns over the schema-tier + * stats with no blank pinned row, its summary view shows why the stats are + * missing with the control that asks for them (basic before full, with a + * per-column form), and over the ceiling it says so and offers nothing. */ import { test, expect, Page } from "@playwright/test"; import { waitForCells } from "./ag-pw-utils"; @@ -85,17 +91,19 @@ test("each stats status shows its own text in the status bar and nothing moves b expect(pageErrors).toEqual([]); }); -test("the Compute summary stats control sends an incremental stats_request with force and the visible columns", async ({ page }) => { +test("the Compute summary stats control sends an incremental stats_request with force and a tier, and no columns hint", async ({ page }) => { await page.goto(STORY_URL); await waitForCells(page); await page.getByTestId("status-not_computed").click(); await page.getByRole("button", { name: "Compute summary stats" }).click(); - // The story's grid shows columns a and b (the index column is not a stats column). + // The story's server names no requestable tier, so the one it can be asked for + // is full. The grid shows columns a and b, and the whole-table request does not + // send them: on a forced request `columns` names what it is for (rows-first c5). const sent = page.getByTestId("sent-log"); await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([ - { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, columns: ["a", "b"] }, + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "full" }, ]); }); @@ -130,3 +138,98 @@ test("no control is offered while the stats are pending, in error or complete", await expect(control).toHaveCount(0); } }); + +const NOT_COMPUTED_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--not-computed&globals=&args="; + +const sentLog = async (page: Page): Promise => JSON.parse((await page.getByTestId("sent-log").textContent()) ?? "[]"); + +test("a not computed session renders typed columns with no blank pinned rows", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + + // The columns are typed from the schema: integers without decimals, floats + // with three, strings as they are. + const firstRow = (col: string) => page.locator(`.ag-center-cols-container .ag-row[row-index="0"] [col-id="${col}"]`); + await expect(firstRow("a")).toHaveText("1"); + await expect(firstRow("b")).toHaveText("x"); + await expect(firstRow("c")).toHaveText("1.500"); + + // dtype came with the schema. histogram has no value and none is coming, so + // there is one pinned row and no blank one. + const pinnedRows = page.locator(".df-viewer .ag-floating-top .ag-row"); + // A pinned row renders once per column container, with the same id each time. + await expect + .poll(async () => Array.from(new Set(await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id")))))) + .toEqual(["main-dtype"]); + const dtypeCell = (col: string) => page.locator(`.df-viewer .ag-floating-top .ag-row[row-id="main-dtype"] [col-id="${col}"]`); + await expect(dtypeCell("a")).toHaveText("int64"); + await expect(dtypeCell("c")).toHaveText("float64"); + + // The status bar says the stats are not computed and offers the control. + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); + expect(pageErrors).toEqual([]); +}); + +test("the summary view shows the empty state, and its control asks for the basic tier, for one column or all", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("view-summary").click(); + + const empty = page.getByTestId("stats-empty-state"); + await expect(empty).toBeVisible(); + await expect(empty).toContainText("not computed"); + await expect(empty).toContainText("12.4M rows x 3 columns"); + // The grid is not shown in its place. + await expect(page.locator(".df-viewer")).toHaveCount(0); + + // The whole table: the smallest tier the server allows, force, no columns. + await empty.getByRole("button", { name: "Compute basic stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + + // One column: the same request, naming the column by the grid's own name. + await empty.getByRole("combobox", { name: "Columns to compute" }).selectOption("b"); + await empty.getByRole("button", { name: "Compute basic stats" }).click(); + await expect.poll(async () => (await sentLog(page)).length).toBe(2); + expect((await sentLog(page))[1]).toEqual({ + type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar", columns: ["b"], + }); + + // Back to the main view: its grid is there again. + await page.getByTestId("view-main").click(); + await expect(page.getByTestId("stats-empty-state")).toHaveCount(0); + await waitForCells(page); +}); + +test("over the ceiling the summary view says so and offers no control, and the status bar offers none either", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("policy-ceiling").click(); + await page.getByTestId("view-summary").click(); + + const empty = page.getByTestId("stats-empty-state"); + await expect(empty).toContainText("over the size limit"); + await expect(page.getByRole("button", { name: /compute|continue/i })).toHaveCount(0); + await expect(page.getByRole("combobox", { name: "Columns to compute" })).toHaveCount(0); + await expect(page.getByTestId("stats-status")).toContainText("Summary stats unavailable"); + expect(await sentLog(page)).toEqual([]); +}); + +test("a paused run offers Continue, which asks for the tier that is left", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("policy-cost").click(); + await page.getByTestId("view-summary").click(); + + await expect(page.getByTestId("stats-empty-state")).toContainText("paused"); + await page.getByTestId("stats-empty-state").getByRole("button", { name: "Continue computing stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index b56524bb9..b949f63ef 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -7,7 +7,7 @@ * Tests assert CURRENT behavior on main (Option A in docs/rerender-test-plan.md). * Tests tagged "[captures current flash]" are tracking pain, not validating it. */ -import { render, act, waitFor } from "@testing-library/react"; +import { render, act, waitFor, screen, fireEvent } from "@testing-library/react"; import { BuckarooInfiniteWidget } from "./BuckarooWidgetInfinite"; import { KeyAwareSmartRowCache } from "./DFViewerParts/SmartRowCache"; import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridSpy"; @@ -927,3 +927,114 @@ describe("BuckarooInfiniteWidget on_visible_columns (rows-first c4b)", () => { await waitFor(() => expect(onVisible).toHaveBeenCalledWith(["a"])); }); }); + +// Rows-first c5: a session whose stats are not computed has nothing for the +// summary view to list. It shows why, with the control that asks for the stats, +// in place of the grid; the main view keeps its grid. +describe("BuckarooInfiniteWidget summary view, stats not computed (rows-first c5)", () => { + const notComputed = (over: Record = {}): DFMeta => + ({ + ...baseDfMeta, + stats: { + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 2 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, + }, + }) as DFMeta; + + const widgetProps = (over: Record = {}) => ({ + df_data_dict: { summary_stats: [] as any[] }, + df_display_args: baseDisplayArgs, + df_meta: notComputed(), + operations: [], + on_operations: jest.fn(), + operation_results: {} as any, + command_config: { argspecs: {}, defaultArgs: {} }, + buckaroo_state: { ...initialState, df_display: "summary" }, + on_buckaroo_state: jest.fn(), + buckaroo_options: baseOptions, + src: mkSrc(), + ...over, + }); + + it("shows the empty state with the compute control in place of the grid", () => { + const onComputeStats = jest.fn(); + render(); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("not computed"); + expect(getSpyCalls().mountCount).toBe(0); + + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("offers a per-column form over the columns the view has, naming the column by the grid's own name", () => { + const onComputeStats = jest.fn(); + render(); + const picker = screen.getByRole("combobox", { name: "Columns to compute" }); + fireEvent.change(picker, { target: { value: "a" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith({ columns: ["a"] }); + }); + + it("ceiling: the message and no control", () => { + render( + , + ); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("over the size limit"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(0); + }); + + it("without a callback from the host it is the message alone", () => { + render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("the main view keeps its grid and shows no empty state", () => { + render( + , + ); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it.each(["pending", "error", "complete"])("a %s session keeps the summary view's grid", (status) => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it("a session that reports no df_meta.stats keeps the summary view's grid, as before", () => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it("gives the grid back when the stats arrive", () => { + const props = widgetProps(); + const { rerender } = render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + + rerender( + , + ); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index 508183914..c6f382ca8 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -152,7 +152,7 @@ export function BuckarooInfiniteWidget({ df_display_args: Record; /** Sends a forced stats_request. Server entry points pass it; the Jupyter * widget does not, and then the status bar offers no control. */ - on_compute_stats?: () => void; + on_compute_stats?: (opts?: { columns?: string[] }) => void; /** Called with the data columns the grid shows, now and whenever they change. */ on_visible_columns?: (columns: string[]) => void; operations: Operation[]; diff --git a/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx new file mode 100644 index 000000000..ecd92b05b --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx @@ -0,0 +1,148 @@ +/** + * StatsEmptyState — the summary view of a session whose stats are not computed + * (rows-first c5). + * + * The summary view lists stats, and there are none to list, so it shows why + * and offers the control that asks for them: one button for the smallest tier + * the server allows (basic before full), with a per-column form. When the + * server's ceiling refuses the stats the view says so and offers nothing. + */ +import "@testing-library/jest-dom"; +import { render, screen, fireEvent, within } from "@testing-library/react"; + +jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); + +import { StatsEmptyState, StatsColumnOption, formatStatsEstimate } from "./StatsEmptyState"; +import { DFMetaStats } from "./WidgetTypes"; + +const policy = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, +}); + +const columns: StatsColumnOption[] = [ + { field: "a", label: "name" }, + { field: "b", label: "age" }, +]; + +const renderState = (stats: DFMetaStats, onComputeStats?: jest.Mock, cols: StatsColumnOption[] = columns) => + render(); + +describe("StatsEmptyState", () => { + it("size: says the stats were not computed because the table is large, and how large", () => { + renderState(policy(), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveAttribute("data-stats-reason", "size"); + expect(root).toHaveTextContent("not computed"); + expect(root).toHaveTextContent("12.4M rows x 44 columns"); + }); + + it("offers one button for the smallest tier the server allows, and it calls the handler with no arguments", () => { + const onComputeStats = jest.fn(); + renderState(policy(), onComputeStats); + expect(screen.getAllByRole("button")).toHaveLength(1); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("names full when it is the only tier left to ask for", () => { + renderState(policy({ requestable: ["full"] }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Compute basic stats" })).not.toBeInTheDocument(); + }); + + it("names full once the basic tier has been reached", () => { + renderState(policy({ tier: "scalar" }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + }); + + it("has a per-column form: the button asks for the column picked, and all columns by default", () => { + const onComputeStats = jest.fn(); + renderState(policy(), onComputeStats); + const picker = screen.getByRole("combobox", { name: "Columns to compute" }); + expect(within(picker).getAllByRole("option").map((o) => o.textContent)).toEqual(["All columns", "name", "age"]); + + fireEvent.change(picker, { target: { value: "b" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith({ columns: ["b"] }); + + fireEvent.change(picker, { target: { value: "" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith(); + }); + + it("has no per-column picker when it knows no columns", () => { + renderState(policy(), jest.fn(), []); + expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Compute basic stats" })).toBeInTheDocument(); + }); + + it("ceiling: says the table is over the size limit, and offers no control even with a handler", () => { + renderState(policy({ reason: "ceiling", requestable: [] }), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveAttribute("data-stats-reason", "ceiling"); + expect(root).toHaveTextContent("over the size limit"); + expect(root).toHaveTextContent("12.4M rows x 44 columns"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); + }); + + it("ceiling: offers no control whatever requestable lists", () => { + renderState(policy({ reason: "ceiling", requestable: ["scalar", "full"] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("host: says the stats were turned off, and offers the control", () => { + renderState(policy({ reason: "host" }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("turned off"); + expect(screen.getByRole("button", { name: "Compute basic stats" })).toBeInTheDocument(); + }); + + it("cost: says the run was paused, and the button reads Continue", () => { + const onComputeStats = jest.fn(); + renderState(policy({ reason: "cost" }), onComputeStats); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("paused"); + fireEvent.click(screen.getByRole("button", { name: "Continue computing stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("with no handler it is the message alone", () => { + renderState(policy()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("not computed"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("leaves the size out when the server sent no estimate", () => { + renderState(policy({ estimate: undefined }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).not.toHaveTextContent("rows"); + }); + + it("an older server's not_computed (no policy fields) gets the message and a full-stats button", () => { + renderState({ status: "not_computed", tier: "schema", gen: 1, reason: "host" }, jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + }); +}); + +describe("formatStatsEstimate", () => { + it.each([ + [{ rows: 12_400_000, cols: 44 }, "12.4M rows x 44 columns"], + [{ rows: 78_000_000 }, "78M rows"], + [{ rows: 1_000_000 }, "1M rows"], + [{ rows: 999_999 }, "999,999 rows"], + [{ rows: 1 }, "1 row"], + [{ rows: 3, cols: 1 }, "3 rows x 1 column"], + [{ cols: 44 }, "44 columns"], + [{}, ""], + [undefined, ""], + ])("%j reads %j", (estimate, expected) => { + expect(formatStatsEstimate(estimate)).toBe(expected); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx b/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx new file mode 100644 index 000000000..7c1d5e84c --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx @@ -0,0 +1,26 @@ +import type { DFMetaStats } from "./WidgetTypes"; +import type { ThemeConfig } from "./DFViewerParts/gridUtils"; + +/** A column the empty state can compute stats for on its own. */ +export interface StatsColumnOption { + /** The grid's own column name (a, b, c), which a request names. */ + field: string; + /** What the column is called on screen. */ + label: string; +} + +export interface StatsEmptyStateProps { + stats: DFMetaStats; + columns?: StatsColumnOption[]; + onComputeStats?: (opts?: { columns?: string[] }) => void; + themeConfig?: ThemeConfig; +} + +// Stub: the real functions come with the fix. +export function formatStatsEstimate(_estimate: DFMetaStats["estimate"]): string { + return ""; +} + +export function StatsEmptyState(_props: StatsEmptyStateProps) { + return null; +} diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx index 7db44f0e8..852d8b862 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -44,7 +44,7 @@ const bState: BuckarooState = { show_commands: false, }; -const renderBar = (dfMeta: DFMeta, onComputeStats?: () => void) => +const renderBar = (dfMeta: DFMeta, onComputeStats?: (opts?: { columns?: string[] }) => void) => render( { }); describe("StatsStatusCell", () => { - const cell = (value: DFMetaStats | undefined, onComputeStats?: () => void) => + const cell = (value: DFMetaStats | undefined, onComputeStats?: (opts?: { columns?: string[] }) => void) => render(); it("pending: says the stats are being computed", () => { @@ -150,3 +150,74 @@ describe("StatsStatusCell", () => { expect(container).toBeEmptyDOMElement(); }); }); + +// The server's policy fields say why the stats were not computed and what may +// still be asked for (rows-first c5). +describe("StatsStatusCell, not computed by policy (rows-first c5)", () => { + const policy = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, + }); + const cell = (value: DFMetaStats | undefined, onComputeStats?: (opts?: { columns?: string[] }) => void) => + render(); + + it("size: offers the control, and says how large the table is in its title", () => { + const onComputeStats = jest.fn(); + cell(policy(), onComputeStats); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "not_computed"); + expect(root).toHaveAttribute("data-stats-reason", "size"); + + const button = screen.getByRole("button", { name: "Compute summary stats" }); + expect(button).toHaveAttribute("title", expect.stringContaining("12.4M rows")); + fireEvent.click(button); + // The whole table, with no arguments: the wiring picks the tier from df_meta. + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("host: offers the control", () => { + cell(policy({ reason: "host" }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute summary stats" })).toBeInTheDocument(); + }); + + it("cost: the control reads Continue, since the run was paused", () => { + const onComputeStats = jest.fn(); + cell(policy({ reason: "cost" }), onComputeStats); + expect(screen.queryByRole("button", { name: "Compute summary stats" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Continue computing stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("ceiling: a message and no control, even with a handler and a requestable list", () => { + cell(policy({ reason: "ceiling" }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-reason", "ceiling"); + expect(root).toHaveTextContent("Summary stats unavailable"); + expect(root).toHaveAttribute("title", expect.stringContaining("size limit")); + }); + + it("nothing requestable: the label only", () => { + cell(policy({ requestable: [] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); + }); + + it("an older server's not_computed (no policy fields) still offers the control", () => { + cell({ status: "not_computed", tier: "schema", gen: 1, reason: "host" }, jest.fn()); + expect(screen.getByRole("button", { name: "Compute summary stats" })).toBeInTheDocument(); + }); + + it("keeps the label for the other statuses whatever the reason says", () => { + cell({ status: "complete", tier: "full", gen: 1, reason: "ceiling" } as DFMetaStats, jest.fn()); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats ready"); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.tsx b/packages/buckaroo-js-core/src/components/StatusBar.tsx index e7c9be79e..50894140a 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.tsx @@ -313,7 +313,7 @@ export const SearchEditor = memo(({ value, onValueChange, stopEditing }: Custom * computed, the reason when they failed. The cell always renders one line in a * fixed-width column, so changing status moves nothing. */ -export const StatsStatusCell = function (params: { value?: DFMetaStats; context?: { onComputeStats?: () => void } }) { +export const StatsStatusCell = function (params: { value?: DFMetaStats; context?: { onComputeStats?: (opts?: { columns?: string[] }) => void } }) { const stats = params.value; if (stats === undefined) return null; const onComputeStats = params.context?.onComputeStats; @@ -380,7 +380,7 @@ export function StatusBar({ componentConfig?: Record; /** Sends a forced stats_request. The stats column shows it as a button while * df_meta.stats.status is "not_computed"; without it there is no button. */ - onComputeStats?: () => void; + onComputeStats?: (opts?: { columns?: string[] }) => void; }) { if (false) { console.log("heightOverride", heightOverride); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts new file mode 100644 index 000000000..979c4a4c0 --- /dev/null +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts @@ -0,0 +1,116 @@ +/** + * df_meta.stats helpers (rows-first c5). + * + * The server reports its stats policy to a client that advertises + * stats_ondemand, and leaves a field out when it holds the documented default: + * `auto_request` absent means true, `requestable` absent means ["full"]. These + * helpers read the fields the way the control and the scheduler need them. + */ +import { + DFMetaStats, + canRequestStats, + demandTier, + nextRequestTier, + statsAutoRequest, + statsRequestable, +} from "./WidgetTypes"; + +const stats = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + ...over, +}); + +describe("defaults for fields the server leaves out", () => { + it("requestable is ['full'] when absent, and the server's list when present", () => { + expect(statsRequestable(stats())).toEqual(["full"]); + expect(statsRequestable(undefined)).toEqual(["full"]); + expect(statsRequestable(stats({ requestable: ["scalar", "full"] }))).toEqual(["scalar", "full"]); + expect(statsRequestable(stats({ requestable: [] }))).toEqual([]); + }); + + it("auto_request is true when absent, and false only when the server says so", () => { + expect(statsAutoRequest(stats())).toBe(true); + expect(statsAutoRequest(undefined)).toBe(true); + expect(statsAutoRequest(stats({ auto_request: true }))).toBe(true); + expect(statsAutoRequest(stats({ auto_request: false }))).toBe(false); + }); +}); + +describe("nextRequestTier", () => { + it.each([ + // The smallest tier the policy allows goes first: scalar before full. + [["scalar", "full"], "schema", "scalar"], + [["full", "scalar"], "schema", "scalar"], + [["full"], "schema", "full"], + [["scalar"], "schema", "scalar"], + // After scalar has been reached the next one up is offered. + [["scalar", "full"], "scalar", "full"], + [["scalar"], "scalar", undefined], + // Nothing above the target: a ceiling left nothing to ask for. + [[], "schema", undefined], + ])("requestable %j with %s reached asks for %s", (requestable, tier, expected) => { + expect(nextRequestTier(stats({ requestable, tier }))).toBe(expected); + }); + + it("asks for full when the server reports no requestable (its default)", () => { + expect(nextRequestTier(stats())).toBe("full"); + }); + + it("treats a session with no tier as having reached schema", () => { + expect(nextRequestTier({ status: "not_computed", gen: 1, requestable: ["scalar"] })).toBe("scalar"); + }); + + it("ignores names that are not tiers", () => { + expect(nextRequestTier(stats({ requestable: ["columns", "scalar"] }))).toBe("scalar"); + expect(nextRequestTier(stats({ requestable: ["columns"] }))).toBeUndefined(); + }); + + it("is undefined without stats", () => { + expect(nextRequestTier(undefined)).toBeUndefined(); + }); +}); + +describe("canRequestStats", () => { + it("is true for a not_computed session with a tier left to ask for", () => { + expect(canRequestStats(stats())).toBe(true); + expect(canRequestStats(stats({ requestable: ["scalar", "full"], reason: "size" }))).toBe(true); + expect(canRequestStats(stats({ reason: "cost" }))).toBe(true); + expect(canRequestStats(stats({ reason: "host" }))).toBe(true); + }); + + it("is false at the ceiling, whatever requestable lists", () => { + expect(canRequestStats(stats({ reason: "ceiling" }))).toBe(false); + expect(canRequestStats(stats({ reason: "ceiling", requestable: ["scalar", "full"] }))).toBe(false); + }); + + it("is false when requestable is empty", () => { + expect(canRequestStats(stats({ requestable: [] }))).toBe(false); + }); + + it.each(["pending", "complete", "error"] as const)("is false when the status is %s", (status) => { + expect(canRequestStats(stats({ status }))).toBe(false); + }); + + it("is false without stats", () => { + expect(canRequestStats(undefined)).toBe(false); + }); +}); + +describe("demandTier", () => { + it("is the smallest tier that carries min and max the policy allows", () => { + expect(demandTier(stats({ requestable: ["scalar", "full"] }))).toBe("scalar"); + expect(demandTier(stats({ requestable: ["scalar"] }))).toBe("scalar"); + expect(demandTier(stats({ requestable: ["full"] }))).toBe("full"); + }); + + it("counts the target tier as allowed", () => { + expect(demandTier(stats({ tier_target: "scalar", requestable: [] }))).toBe("scalar"); + }); + + it("is undefined when the policy allows nothing above schema", () => { + expect(demandTier(stats({ tier_target: "schema", requestable: [] }))).toBeUndefined(); + expect(demandTier(undefined)).toBeUndefined(); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index d15baac8e..d63e56e4f 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -4,11 +4,35 @@ // what servers that predate the field send. export type StatsStatus = "complete" | "pending" | "not_computed" | "error"; +// The tiers a session can reach, lowest first. "schema" is dtypes and identity +// keys, "scalar" adds the cheap per-column stats (min, max, mean, std, null +// count), "full" is every stat. +export type StatsTier = "schema" | "scalar" | "full"; +export const STATS_TIERS: readonly StatsTier[] = ["schema", "scalar", "full"]; + export interface DFMetaStats { status: StatsStatus; + // The tier reached so far. tier?: string; + // Why the stats are not computed: "size", "host", "cost" or "ceiling". reason?: string; gen?: number; + // The fields below are sent to a client that advertises stats_ondemand, + // and only where they differ from the default noted on each. + // The tier the session is headed for. + tier_target?: string; + // The numbers the server's decision rested on. + estimate?: { rows?: number; cols?: number; bytes?: number }; + // Whether the client should ask for the stats on its own. Absent means true. + auto_request?: boolean; + // Tiers above the target an explicit request may still be granted. Absent + // means ["full"]. + requestable?: string[]; + // Columns whose styling needs stats now (a color_map reads min and max). + demand_columns?: string[]; + // Stat keys left out of the run, and keys computed from a sample. + omitted_keys?: string[]; + approx_keys?: string[]; } export interface DFMeta { @@ -24,6 +48,13 @@ export interface DFMeta { export const getStatsStatus = (meta: DFMeta | undefined): StatsStatus => meta?.stats?.status ?? "complete"; +// Stubs: the real helpers come with the fix. +export const statsRequestable = (_stats: DFMetaStats | undefined): string[] => []; +export const statsAutoRequest = (_stats: DFMetaStats | undefined): boolean => false; +export const nextRequestTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; +export const canRequestStats = (_stats: DFMetaStats | undefined): boolean => false; +export const demandTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; + export interface BuckarooOptions { sampled: string[]; cleaning_method: string[]; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx index 21e8b8421..2b12a393b 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx @@ -4,7 +4,8 @@ * The server records a client's capabilities from `?caps=` on the WebSocket * URL, because it sends the first message before the client says anything. A * client that merges `stats_update` must put it there, whatever URL the host - * passes in. + * passes in. So must one that can show a session whose stats are not computed + * and send tiered requests, which is `stats_ondemand` (rows-first c5). */ import { render, cleanup, waitFor } from "@testing-library/react"; import { BuckarooServerView } from "./BuckarooServerView"; @@ -68,17 +69,17 @@ afterEach(() => { cleanup(); }); -describe("BuckarooServerView advertises stats_update", () => { - it("opens the socket with ?caps=stats_update", async () => { +describe("BuckarooServerView advertises stats_update and stats_ondemand", () => { + it("opens the socket with ?caps=stats_update,stats_ondemand", async () => { render(); await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); expect(FakeWebSocket.instances).toHaveLength(1); - expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?caps=stats_update"); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?caps=stats_update,stats_ondemand"); }); it("keeps the query string the host passed", async () => { render(); await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); - expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?token=abc&caps=stats_update"); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?token=abc&caps=stats_update,stats_ondemand"); }); }); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx index 31c187ebd..4c4d95661 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx @@ -3,9 +3,10 @@ * * While df_meta.stats says the stats are not computed, the status bar offers a * control that asks the server for them. BuckarooView hands the widget the - * callback, and the callback sends `stats_request {force: true}` through - * whatever IModel the host gave it. The grid reports the columns it shows, and - * BuckarooView keeps them on the model, where the request reads its hint. + * callback, and the callback sends `stats_request {force: true, tier}` through + * whatever IModel the host gave it, with the tier taken from df_meta.stats + * (rows-first c5). The grid reports the columns it shows, and BuckarooView + * keeps them on the model, where the scheduler's requests read their hint. */ import { render, cleanup, act } from "@testing-library/react"; import { BuckarooView } from "./BuckarooView"; @@ -65,11 +66,48 @@ describe("BuckarooView on_compute_stats (rows-first c4)", () => { expect(typeof props().on_compute_stats).toBe("function"); props().on_compute_stats(); - // A time-boxed step, like the scheduler's (rows-first c4b). - expect(sent).toEqual([{ type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true }]); + // A time-boxed step, like the scheduler's (rows-first c4b), for the one + // tier a server that names none can be asked for (rows-first c5). + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); + }); + + it("takes the tier from df_meta.stats.requestable: scalar before full", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, requestable: ["scalar", "full"] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats(); + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + }); + + it("hands the per-column form's columns on to the request", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, requestable: ["scalar", "full"] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats({ columns: ["b"] }); + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "scalar", columns: ["b"] }, + ]); + }); + + it("sends nothing when the server left no tier to ask for", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, reason: "ceiling", requestable: [] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats(); + expect(sent).toEqual([]); }); - it("records the columns the grid shows on the model, and the control's request carries them as its hint", async () => { + it("records the columns the grid shows on the model, and the whole-table control does not send them", async () => { const { model, sent, props } = await mountBuckaroo({ df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), df_data_dict: {}, @@ -80,8 +118,9 @@ describe("BuckarooView on_compute_stats (rows-first c4)", () => { props().on_visible_columns(["a", "b"]); expect(model.get("visible_columns")).toEqual(["a", "b"]); props().on_compute_stats(); + // On a forced request `columns` names what it is for, so the hint stays off. expect(sent).toEqual([ - { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, columns: ["a", "b"] }, + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "full" }, ]); }); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 4d1494ccf..aa2c947cc 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -11,7 +11,7 @@ * one (set a key, emit its change event). The last block builds a real * WebSocketModel from a fake socket. */ -import { StateOrchestrator, StatsModel, requestStats, touchesDataflow } from "./StateOrchestrator"; +import { StateOrchestrator, StatsModel, forceStats, requestStats, touchesDataflow } from "./StateOrchestrator"; import { WebSocketModel } from "./WebSocketModel"; class FakeModel implements StatsModel { @@ -485,6 +485,297 @@ describe("a state change", () => { }); }); +// A session whose stats the server did not plan to compute says so in +// df_meta.stats (rows-first c5): the tier it is headed for, whether to ask for +// it on its own (`auto_request`, absent means true) and the columns whose +// styling needs stats now (`demand_columns`). With auto_request false the +// scheduler asks for those columns and nothing else. +const policy = (over: Record = {}) => ({ + status: "not_computed", + tier: "schema", + gen: 3, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, +}); + +describe("auto_request false: only the demand columns are asked for (rows-first c5)", () => { + // The request for demand columns: scoped to them, at the tier that carries + // min and max, and not forced (it is the scheduler's, not the user's). + const demand = (gen: number, columns: string[], tier = "scalar") => request(gen, { columns, tier }); + const sendFirst = async (model: FakeModel) => { + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return orchestrator; + }; + + it.each(["pending", "not_computed"])("asks for nothing when the status is %s and no column needs stats", async (status) => { + const model = makeModel(policy({ status })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["pending", "not_computed"])("asks for nothing when demand_columns is empty (status %s)", async (status) => { + const model = makeModel(policy({ status, demand_columns: [] })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["pending", "not_computed"])("sends one scoped request for the demand columns (status %s)", async (status) => { + const model = makeModel(policy({ status, demand_columns: ["a", "c"] })); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + // The visible columns are not what it asks for, and nothing is forced. + expect(model.sent).toEqual([demand(3, ["a", "c"])]); + expect(model.sent[0]).not.toHaveProperty("force"); + }); + + it("goes out after the first rows, not before, and once however many row responses follow", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + start(model); + await tick(100); + expect(model.sent).toEqual([]); + rowsArrived(model); + rowsArrived(model); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("asks for the next step of the demand for each reply that is not final, and stops at the final one", async () => { + const model = makeModel(policy({ demand_columns: ["a", "c"] })); + await sendFirst(model); + expect(model.sent).toHaveLength(1); + + // A partial reply is a new df_data_dict under the same df_meta. + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([demand(3, ["a", "c"]), demand(3, ["a", "c"])]); + + // The final reply says the session is still not computed (it ran for some + // columns only), in a new df_meta. + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + model.set("df_meta", meta(policy({ demand_columns: ["a", "c"] }))); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("does not ask again for the same demand after a refusal or a reply that carried no stats", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + // stats_aborted not_requestable and a ceiling reply both replace df_meta only. + model.set("df_meta", meta(policy({ demand_columns: ["a"], reason: "ceiling" }))); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("asks for the next gen's demand after a state change", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + model.set("df_meta", meta(policy({ demand_columns: ["a"] }))); + await tick(); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + model.frame({ df_meta: meta(policy({ gen: 4, demand_columns: ["b"] })), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([demand(3, ["a"]), demand(4, ["b"])]); + }); + + it("asks for a changed list of demand columns on the same gen", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + model.set("df_meta", meta(policy({ demand_columns: ["a"] }))); + await tick(); + + model.set("df_meta", meta(policy({ demand_columns: ["a", "d"] }))); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([demand(3, ["a"]), demand(3, ["a", "d"])]); + }); + + it("asks at the tier the policy allows: full when scalar is not requestable", async () => { + const model = makeModel(policy({ demand_columns: ["a"], requestable: ["full"] })); + await sendFirst(model); + expect(model.sent).toEqual([demand(3, ["a"], "full")]); + }); + + it("asks for nothing when the policy allows no tier above schema (a ceiling)", async () => { + const model = makeModel(policy({ demand_columns: ["a"], requestable: [], reason: "ceiling" })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("a pending session with auto_request false is not run whole", async () => { + const model = makeModel(policy({ status: "pending", demand_columns: ["a"] })); + await sendFirst(model); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("leaves a session that does auto-request alone: demand_columns do not change its whole run", async () => { + for (const over of [{ auto_request: true }, { auto_request: undefined }]) { + const model = makeModel({ status: "pending", tier: "schema", gen: 3, demand_columns: ["a"], ...over }); + model.state.visible_columns = ["b"]; + await sendFirst(model); + expect(model.sent).toEqual([request(3, { columns: ["b"] })]); + } + }); + + it("does not ask when the policy says not computed and auto_request is true (nothing is served yet)", async () => { + const model = makeModel(policy({ auto_request: true, demand_columns: ["a"] })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); +}); + +describe("forceStats: the control's request (rows-first c5)", () => { + const notComputed = (over: Record = {}) => policy({ gen: 5, ...over }); + + it("asks for the smallest tier the policy allows: scalar before full", () => { + const model = makeModel(notComputed()); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar" })]); + }); + + it("asks for full when the server leaves requestable out (its default)", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 5, reason: "host" }); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "full" })]); + }); + + it("asks for the tier above the one reached once scalar is in", () => { + const model = makeModel(notComputed({ tier: "scalar" })); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "full" })]); + }); + + it("asks for scalar alone when only scalar is requestable", () => { + const model = makeModel(notComputed({ requestable: ["scalar"] })); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar" })]); + }); + + it("sends nothing when no tier is left to ask for", () => { + const model = makeModel(notComputed({ requestable: [], reason: "ceiling" })); + expect(forceStats(model)).toBe(false); + expect(model.sent).toEqual([]); + }); + + it("sends nothing when df_meta carries no stats.gen", () => { + const model = makeModel(); + expect(forceStats(model)).toBe(false); + expect(model.sent).toEqual([]); + }); + + it("names no columns for the whole table, whatever the grid shows", () => { + const model = makeModel(notComputed()); + model.state.visible_columns = ["a", "b"]; + forceStats(model); + expect(model.sent[0]).not.toHaveProperty("columns"); + }); + + it("has a per-column form: the columns it is given, with the same force and tier", () => { + const model = makeModel(notComputed()); + model.state.visible_columns = ["a", "b"]; + expect(forceStats(model, { columns: ["c"] })).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar", columns: ["c"] })]); + }); +}); + +describe("a forced run is continued by the scheduler (rows-first c5)", () => { + const notComputed = (over: Record = {}) => policy({ gen: 5, ...over }); + const forced = (extra: Record = {}) => request(5, { force: true, tier: "scalar", ...extra }); + const startForced = async (opts?: { columns?: string[] }, stats = notComputed()) => { + const model = makeModel(stats); + start(model); + forceStats(model, opts); + await tick(); + return model; + }; + + it("asks again, with the same force, tier and columns, for each reply that is not final", async () => { + const model = await startForced({ columns: ["c"] }); + expect(model.sent).toEqual([forced({ columns: ["c"] })]); + + // No reply yet: nothing more goes out, however long the server takes. + await tick(10_000); + expect(model.sent).toHaveLength(1); + + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([forced({ columns: ["c"] }), forced({ columns: ["c"] })]); + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + await tick(); + expect(model.sent).toHaveLength(3); + }); + + it("sends the whole-table form again as the whole-table form", async () => { + const model = await startForced(); + model.state.visible_columns = ["a"]; + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([forced(), forced()]); + }); + + it("stops at a final reply, which replaces df_meta", async () => { + const model = await startForced(); + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toHaveLength(2); + + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + model.set("df_meta", meta({ status: "complete", tier: "scalar", gen: 5 })); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("stops at a ceiling reply or a refusal, which change df_meta and nothing else", async () => { + const model = await startForced(); + model.set("df_meta", meta(notComputed({ reason: "ceiling" }))); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + + it("drops the run when the gen moves on, and does not continue a late reply", async () => { + const model = await startForced(); + model.frame({ df_meta: meta(notComputed({ gen: 6 })), df_data_dict: dict() }); + await tick(); + // A reply for the old run arrives after the new frame. + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + + it("is not started for a run recorded against a gen that is not on screen", async () => { + const model = makeModel(notComputed()); + start(model); + model.set("stats_forced", { gen: 9, tier: "scalar" }); + await tick(); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("does nothing for a model nobody started a scheduler on", async () => { + const model = makeModel(notComputed()); + forceStats(model); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); +}); + describe("requestStats", () => { it("sends a stats_request for the gen the model shows", () => { const model = makeModel(pending(7)); @@ -498,11 +789,37 @@ describe("requestStats", () => { expect(model.sent).toEqual([request(7, { force: true })]); }); - it("carries the columns the grid shows, for a forced request too", () => { + // Plan 3 s3.4: the control sends {stats_gen, scope, tier, force}, with a + // `columns` form per column. On a forced request `columns` names the + // columns the request is for, so the grid's visible columns (an ordering + // hint on the scheduler's requests) are not added to it (rows-first c5). + it("adds no columns hint to a forced request, however many columns the grid shows", () => { const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); model.state.visible_columns = ["a", "b"]; expect(requestStats(model, { force: true })).toBe(true); - expect(model.sent).toEqual([request(7, { force: true, columns: ["a", "b"] })]); + expect(model.sent).toEqual([request(7, { force: true })]); + expect(model.sent[0]).not.toHaveProperty("columns"); + }); + + it("carries the tier it is asked for", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + expect(requestStats(model, { force: true, tier: "scalar" })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true, tier: "scalar" })]); + }); + + it("names the columns it is given, in place of the visible-columns hint", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + model.state.visible_columns = ["a", "b"]; + expect(requestStats(model, { force: true, tier: "scalar", columns: ["c"] })).toBe(true); + expect(requestStats(model, { columns: ["d"] })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true, tier: "scalar", columns: ["c"] }), request(7, { columns: ["d"] })]); + }); + + it("sends no tier and no columns the caller did not name, and no hint for an empty list", () => { + const model = makeModel(pending(7)); + expect(requestStats(model, { columns: [] })).toBe(true); + expect(model.sent).toEqual([request(7)]); + expect(model.sent[0]).not.toHaveProperty("tier"); }); it("sends nothing when df_meta carries no stats.gen", () => { @@ -727,4 +1044,58 @@ describe("wired into WebSocketModel", () => { await tick(10_000); expect(ws.sent).toEqual([]); }); + + // Rows-first c5: a not computed session, the control, and the replies. + describe("a session whose stats are not computed", () => { + const forcedRequest = (extra: Record = {}) => request(3, { force: true, tier: "scalar", ...extra }); + + it("sends nothing on its own, and the control's forced run goes tier by tier to its final reply", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + rowsFromServer(ws); + await tick(10_000); + expect(ws.sent).toEqual([]); + + expect(forceStats(model)).toBe(true); + expect(ws.sent).toEqual([forcedRequest()]); + + ws.deliver({ ...update(3, "min", false, 2), tier: "scalar" }); + await tick(); + expect(ws.sent).toEqual([forcedRequest(), forcedRequest()]); + // Still not computed until the final reply says otherwise. + expect(model.get("df_meta").stats.status).toBe("not_computed"); + + ws.deliver({ ...update(3, "max", true, 0), tier: "scalar" }); + await tick(10_000); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "scalar", gen: 3 }); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "min", "max"]); + }); + + it("a request over the ceiling ends in the ceiling message, and nothing more is sent", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "full", final: true, status: "not_computed", reason: "ceiling" }); + await tick(10_000); + expect(ws.sent).toEqual([forcedRequest()]); + expect(model.get("df_meta").stats).toMatchObject({ status: "not_computed", reason: "ceiling", gen: 3 }); + }); + + it("asks for the demand columns after the first rows and nothing else", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3, demand_columns: ["a"] })); + model.set("visible_columns", ["a", "b"]); + await tick(100); + expect(ws.sent).toEqual([]); + + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3, { columns: ["a"], tier: "scalar" })]); + + // The reply carries stats for those columns and leaves the session not computed. + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + expect(model.get("df_meta").stats.status).toBe("not_computed"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "min"]); + }); + }); }); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts index 5eeba3d59..d43b07bd4 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts @@ -74,6 +74,18 @@ export interface StatsRequestOptions { /** Ask for stats the server did not plan to compute. The "Compute summary * stats" control sends this. */ force?: boolean; + /** The tier asked for. */ + tier?: string; + /** The columns the request is for. */ + columns?: string[]; +} + +/** The model key that records a forced run, for the scheduler to continue. */ +export const FORCED_RUN_KEY = "stats_forced"; + +// Stub: the real function comes with the fix. +export function forceStats(_model: Pick, _opts: { columns?: string[] } = {}): boolean { + return false; } /** diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index 9d2caa839..ee8df46e8 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -92,26 +92,41 @@ function makeModel(gen: number | null = 3, allStats: any = schemaStats()) { return { ws, model, events }; } +// The capabilities a client advertises on the WebSocket URL. stats_update +// means it merges stats_update messages; stats_ondemand (rows-first c5) means it +// can show a session whose stats are not computed and send tiered requests, so +// the server may apply its stats policy to it. describe("withStatsCapability", () => { - it("adds ?caps=stats_update to a bare URL", () => { - expect(withStatsCapability("ws://localhost:8700/ws/sales")).toBe("ws://localhost:8700/ws/sales?caps=stats_update"); + it("adds ?caps=stats_update,stats_ondemand to a bare URL", () => { + expect(withStatsCapability("ws://localhost:8700/ws/sales")).toBe( + "ws://localhost:8700/ws/sales?caps=stats_update,stats_ondemand", + ); }); it("appends to an existing query string", () => { - expect(withStatsCapability("ws://h/ws/s?token=abc")).toBe("ws://h/ws/s?token=abc&caps=stats_update"); + expect(withStatsCapability("ws://h/ws/s?token=abc")).toBe("ws://h/ws/s?token=abc&caps=stats_update,stats_ondemand"); }); it("extends an existing caps value", () => { - expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update"); + expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update,stats_ondemand"); }); - it("leaves a URL that already advertises the capability alone", () => { - expect(withStatsCapability("ws://h/ws/s?caps=stats_update")).toBe("ws://h/ws/s?caps=stats_update"); - expect(withStatsCapability("ws://h/ws/s?caps=a,stats_update")).toBe("ws://h/ws/s?caps=a,stats_update"); + it("leaves a URL that already advertises both capabilities alone", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update,stats_ondemand")).toBe( + "ws://h/ws/s?caps=stats_update,stats_ondemand", + ); + expect(withStatsCapability("ws://h/ws/s?caps=a,stats_ondemand,stats_update")).toBe( + "ws://h/ws/s?caps=a,stats_ondemand,stats_update", + ); + }); + + it("adds only the capability a host's caps value lacks", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update")).toBe("ws://h/ws/s?caps=stats_update,stats_ondemand"); + expect(withStatsCapability("ws://h/ws/s?caps=stats_ondemand")).toBe("ws://h/ws/s?caps=stats_ondemand,stats_update"); }); it("keeps the fragment last", () => { - expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update#frag"); + expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update,stats_ondemand#frag"); }); }); @@ -270,6 +285,76 @@ describe("stats_update and df_meta.stats", () => { }); }); +// A reply that did not run, or ran for some columns only, names the status the +// session is left in (rows-first c5). The server answers a request over its +// ceiling with `stats_update {final: true, status: "not_computed", reason: +// "ceiling"}` and no payload. +describe("a final stats_update with a status", () => { + const policyStats = { + status: "not_computed", + tier: "schema", + gen: 3, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + }; + const makePolicyModel = () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(policyStats), + df_data_dict: { all_stats: schemaStats() }, + }); + const events: string[] = []; + for (const key of ["df_data_dict", "df_meta"]) model.on(`change:${key}`, () => events.push(key)); + return { ws, model, events }; + }; + + it("not_computed with reason ceiling ends in the ceiling message: the reason changes, the policy fields stay", async () => { + const { ws, model, events } = makePolicyModel(); + // The reply names the tier that was asked for; the session stays at the + // tier it had reached. + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "full", final: true, status: "not_computed", reason: "ceiling" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, reason: "ceiling" }); + // No payload: the stats the client has are left as they were. + expect(events).toEqual(["df_meta"]); + expect(model.get("df_data_dict").all_stats).toEqual(schemaStats()); + }); + + it("takes the reply's status and reason whatever they are", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", final: true, status: "not_computed", reason: "cost" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, reason: "cost" }); + }); + + it("merges a payload it carries, and leaves the session not computed (a reply for some columns only)", async () => { + const { ws, model, events } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 }), row("max", { a: 9 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "min", "max"]); + expect(model.get("df_meta").stats).toEqual(policyStats); + expect(events).toEqual(["df_data_dict", "df_meta"]); + }); + + it("a final update with no status still completes the session and drops the policy fields", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + }); + + it("is ignored for a gen the client has left", async () => { + const { ws, model, events } = makePolicyModel(); + ws.deliver({ type: "stats_update", stats_gen: 2, scope: "raw", final: true, status: "not_computed", reason: "ceiling" }); + await settle(); + expect(model.get("df_meta").stats.reason).toBe("size"); + expect(events).toEqual([]); + }); +}); + describe("stats_gen", () => { it("drops a stats_update whose stats_gen is not the expected one", async () => { const { ws, model, events } = makeModel(3); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts index 0ead005ed..9567fa526 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -19,7 +19,7 @@ */ import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; -import { DFMeta, DFMetaStats } from "../components/WidgetTypes"; +import { DFMeta, DFMetaStats, StatsStatus } from "../components/WidgetTypes"; import { IModel } from "./IModel"; /** The capability this client advertises, as one value of `?caps=` on the @@ -68,6 +68,10 @@ export interface StatsUpdateMessage { scope?: string; tier?: string; final?: boolean; + // A final reply that did not run, or ran for some columns only, says where + // the stats stand. Absent means "complete". + status?: StatsStatus; + reason?: string; payload?: DFDataOrPayload; elapsed_ms?: number; } diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx index 4acc863e4..20bee07f4 100644 --- a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -11,10 +11,15 @@ * * The buttons switch the status. The "partial" button is "pending" with the * stats of one column in, as a partial stats_update leaves them. The button in - * the status bar sends `stats_request {force: true, incremental: true}` through - * a fake model, which logs what it was asked to send and answers + * the status bar sends `stats_request {force: true, incremental: true, tier}` + * through a fake model, which logs what it was asked to send and answers * `visible_columns` with the columns the grid reports. Used by * stats-scheduler-states.spec.ts. + * + * The NotComputed story (rows-first c5) is a session the server's policy left + * without stats: typed columns over schema-tier stats, a main and a summary + * view, and the policy in df_meta.stats. The buttons switch the view and the + * reason the stats were not computed; the controls send through `forceStats`. */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; @@ -27,7 +32,7 @@ import { BuckarooOptions, BuckarooState, DFMeta, StatsStatus } from "../componen import { CommandConfigT } from "../components/CommandUtils"; import { Operation } from "../components/OperationUtils"; import { baseOperationResults } from "../components/DependentTabs"; -import { requestStats } from "../server/StateOrchestrator"; +import { forceStats } from "../server/StateOrchestrator"; const STATUSES: StatsStatus[] = ["pending", "not_computed", "error", "complete"]; const GEN = 7; @@ -112,6 +117,7 @@ const StatsSchedulerStatesInner: React.FC = () => { const model = useMemo( () => ({ get: (key: string) => (key === "df_meta" ? df_meta : key === "visible_columns" ? visibleColumns : undefined), + set: () => {}, send: (msg: unknown) => setSent((log) => [...log, msg]), }), [df_meta, visibleColumns], @@ -179,7 +185,7 @@ const StatsSchedulerStatesInner: React.FC = () => { on_buckaroo_state={setBuckarooState} buckaroo_options={buckarooOptions} src={src} - on_compute_stats={() => requestStats(model, { force: true })} + on_compute_stats={(opts) => forceStats(model, opts)} on_visible_columns={setVisibleColumns} />
@@ -200,3 +206,164 @@ export default meta; type Story = StoryObj; export const Primary: Story = {}; + + +// ---- NotComputed: a session whose stats the server's policy did not compute ---- + +type Reason = "size" | "ceiling" | "cost"; +const REASONS: Record = { + size: { reason: "size", requestable: ["scalar", "full"] }, + ceiling: { reason: "ceiling", requestable: [] }, + cost: { reason: "cost", requestable: ["full"] }, +}; + +const typedData: DFData = [ + { index: 0, a: 1, b: "x", c: 1.5 }, + { index: 1, a: 2, b: "y", c: 2.25 }, + { index: 2, a: 3, b: "z", c: 3 }, +]; + +// What the schema tier sends: dtype and identity keys, nothing computed. +const schemaStats: DFData = [ + { index: "dtype", a: "int64", b: "object", c: "float64" }, + { index: "length", a: 3, b: 3, c: 3 }, +]; + +const typedColumns: DFViewerConfig["column_config"] = [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "float", min_fraction_digits: 0, max_fraction_digits: 0 } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "string", max_length: 35 } }, + { col_name: "c", header_name: "c", displayer_args: { displayer: "float", min_fraction_digits: 3, max_fraction_digits: 3 } }, +]; +const indexColumn: DFViewerConfig["left_col_configs"] = [ + { col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }, +]; +// dtype is in the schema stats; histogram is not, and no value is coming. +const typedPinned: DFViewerConfig["pinned_rows"] = [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "histogram", displayer_args: { displayer: "histogram" } }, +]; + +const notComputedDisplayArgs: Record = { + main: { + data_key: "main", + df_viewer_config: { column_config: typedColumns, left_col_configs: indexColumn, pinned_rows: typedPinned }, + summary_stats_key: "all_stats", + }, + summary: { + data_key: "empty", + df_viewer_config: { + column_config: typedColumns, + left_col_configs: indexColumn, + pinned_rows: [{ primary_key_val: "dtype", displayer_args: { displayer: "obj" } }], + }, + summary_stats_key: "all_stats", + }, +}; + +const notComputedOptions: BuckarooOptions = { ...buckarooOptions, df_display: ["main", "summary"] }; + +const NotComputedInner: React.FC = () => { + const [reason, setReason] = useState("size"); + const [sent, setSent] = useState([]); + const [buckarooState, setBuckarooState] = useState({ + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, + }); + const [operations, setOperations] = useState([]); + + const df_meta = useMemo( + () => + ({ + total_rows: typedData.length, + columns: 3, + filtered_rows: typedData.length, + rows_shown: typedData.length, + stats: { + status: "not_computed", + tier: "schema", + gen: GEN, + reason: REASONS[reason].reason, + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 3 }, + auto_request: false, + requestable: REASONS[reason].requestable, + }, + }) as DFMeta, + [reason], + ); + + // The model the controls send through: it answers get("df_meta") from the + // story's state and logs what it is asked to send. + const model = useMemo( + () => ({ + get: (key: string) => (key === "df_meta" ? df_meta : undefined), + set: () => {}, + send: (msg: unknown) => setSent((log) => [...log, msg]), + }), + [df_meta], + ); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: typedData.slice(pa.start, Math.min(pa.end, typedData.length)), + length: typedData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_data_dict = useMemo( + () => ({ main: [] as DFData, all_stats: schemaStats, empty: [] as DFData }), + [], + ); + + const setView = (view: string) => setBuckarooState((state) => ({ ...state, df_display: view })); + + return ( +
+
+ {["main", "summary"].map((view) => ( + + ))} + {(Object.keys(REASONS) as Reason[]).map((r) => ( + + ))} + reason = {reason} +
+
+ forceStats(model, opts)} + /> +
+
+        {JSON.stringify(sent)}
+      
+
+ ); +}; + +export const NotComputed: Story = { + render: () => , +}; From f2c4c0a43c67b8e2ed8d7d7ea80a3fc62d4a454e Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 06:53:43 -0400 Subject: [PATCH 14/18] test(client): failing tests for the pending control, the ceiling reading and finished demand runs (rows-first c5) Found while running the first test commit's behaviour end to end against the p33 server: - the control marks the stats pending on the client, since the server sends no frame to a capable client, so the loading text shows at once and a second click cannot start a second chain of requests; a refusal or a run for some columns returns the session to not_computed, and a full frame for the same gen ends the run - a session the server sized at the ceiling itself (reason "size", requestable empty) reads as over the limit: the message and no control - a demand run that has ended is not asked for again when a later frame arrives, and a full frame for the same state does not end one in flight Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/pw-tests/server.spec.ts | 32 +++++-- .../pw-tests/stats-scheduler-states.spec.ts | 26 +++++- .../src/components/StatsEmptyState.test.tsx | 13 +++ .../src/components/StatusBar.stats.test.tsx | 8 +- .../src/components/WidgetTypes.test.ts | 39 +++++++++ .../src/components/WidgetTypes.tsx | 1 + .../src/server/StateOrchestrator.test.ts | 87 ++++++++++++++++++- .../stories/StatsSchedulerStates.stories.tsx | 27 ++++-- 8 files changed, 216 insertions(+), 17 deletions(-) diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index 524a4a760..b4eafb064 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -822,6 +822,11 @@ test.describe('WebSocket data flow', () => { await expect.poll(() => h.requests).toEqual([ { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }, ]); + // The page marks the stats pending itself, since the server sends no frame to + // a capable client: the empty state and the control give way to the loading text. + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await expect(page.getByTestId('stats-empty-state')).toHaveCount(0); + await expect(page.getByRole('button', { name: /compute|continue/i })).toHaveCount(0); }); test('a policy session: a request over the ceiling is answered with the ceiling message, and the page offers nothing more', async ({ page, request }) => { @@ -845,28 +850,39 @@ test.describe('WebSocket data flow', () => { expect(h.requests).toHaveLength(1); }); - test('a policy session: a forced run is asked for again after each partial reply and ends with the final one', async ({ page, request }) => { + test('a policy session: a forced run is asked for again after each partial reply, stays pending, and ends with the final one', async ({ page, request }) => { const session = `ws-forced-${Date.now()}`; await loadBuckarooSession(request, session); + let releaseFinal!: () => void; + const finalGate = new Promise((resolve) => { releaseFinal = resolve; }); const h = await routeStatsRequests(page, session, (msg, send, harness) => { if (harness.requests.length < 3) { send({ ...partialUpdate(msg.stats_gen, 3 - harness.requests.length), tier: 'scalar' }); - } else { - send({ - type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'scalar', final: true, remaining: 0, - payload: harness.realStats, elapsed_ms: 1, - }); + return; } + void finalGate.then(() => send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'scalar', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + })); }, policyStats()); await page.goto(`${BASE}/s/${session}`); await waitForGrid(page); await page.getByRole('button', { name: 'Compute summary stats' }).click(); - await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + // Two partial replies, two more requests, and the third waits for its reply. + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(3); const forced = { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }; expect(h.requests).toEqual([forced, forced, forced]); - await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); + // The partial rows are in and the stats are pending, with no control. + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).toHaveText('int64', { timeout: 10_000 }); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await expect(page.getByRole('button', { name: 'Compute summary stats' })).toHaveCount(0); + + releaseFinal(); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); }); test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts index 443c99f76..bf35699a3 100644 --- a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -192,8 +192,19 @@ test("the summary view shows the empty state, and its control asks for the basic await expect.poll(() => sentLog(page)).toEqual([ { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar" }, ]); + // The stats are marked pending, so the empty state and the control give way to + // the loading text and the grid, and a second click is not possible. + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByTestId("stats-status")).toContainText("Computing summary stats"); + await expect(page.getByTestId("stats-empty-state")).toHaveCount(0); + await expect(page.getByRole("button", { name: /compute|continue/i })).toHaveCount(0); + await expect(page.locator(".df-viewer")).toHaveCount(1); - // One column: the same request, naming the column by the grid's own name. + // The session as the server described it again: the empty state and its + // per-column form. One column: the same request, naming the column by the + // grid's own name. + await page.getByTestId("policy-size").click(); + await expect(empty).toBeVisible(); await empty.getByRole("combobox", { name: "Columns to compute" }).selectOption("b"); await empty.getByRole("button", { name: "Compute basic stats" }).click(); await expect.poll(async () => (await sentLog(page)).length).toBe(2); @@ -207,6 +218,18 @@ test("the summary view shows the empty state, and its control asks for the basic await waitForCells(page); }); +test("the status bar's control asks for the tier too, and the control is gone while the stats are pending", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + + await page.getByRole("button", { name: "Compute summary stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toHaveCount(0); +}); + test("over the ceiling the summary view says so and offers no control, and the status bar offers none either", async ({ page }) => { await page.goto(NOT_COMPUTED_URL); await waitForCells(page); @@ -232,4 +255,5 @@ test("a paused run offers Continue, which asks for the tier that is left", async await expect.poll(() => sentLog(page)).toEqual([ { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "full" }, ]); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); }); diff --git a/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx index ecd92b05b..c7113736e 100644 --- a/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx +++ b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx @@ -95,6 +95,19 @@ describe("StatsEmptyState", () => { expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); }); + it("a size the server chose with nothing left above it reads as over the limit, with no control", () => { + renderState(policy({ requestable: [] }), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveTextContent("over the size limit"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("host with nothing requestable keeps its own message, with no control", () => { + renderState(policy({ reason: "host", requestable: [] }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("turned off"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + it("ceiling: offers no control whatever requestable lists", () => { renderState(policy({ reason: "ceiling", requestable: ["scalar", "full"] }), jest.fn()); expect(screen.queryByRole("button")).not.toBeInTheDocument(); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx index 852d8b862..eeb448e24 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -205,9 +205,15 @@ describe("StatsStatusCell, not computed by policy (rows-first c5)", () => { expect(root).toHaveAttribute("title", expect.stringContaining("size limit")); }); - it("nothing requestable: the label only", () => { + it("nothing requestable for a size the server chose: it is at the ceiling, so the message and no control", () => { cell(policy({ requestable: [] }), jest.fn()); expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats unavailable"); + }); + + it("nothing requestable for a host's choice: the label only", () => { + cell(policy({ reason: "host", requestable: [] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); }); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts index 979c4a4c0..2493c3a08 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts @@ -12,6 +12,7 @@ import { demandTier, nextRequestTier, statsAutoRequest, + statsOverCeiling, statsRequestable, } from "./WidgetTypes"; @@ -114,3 +115,41 @@ describe("demandTier", () => { expect(demandTier(undefined)).toBeUndefined(); }); }); + +// The server's ceiling keeps the stats from being computed. It says so with +// reason "ceiling" when the ceiling cut a request down; when the server sized the +// session at the ceiling itself, the reason is "size" and nothing is left to ask +// for. +describe("statsOverCeiling", () => { + it("is true for reason ceiling, whatever requestable lists", () => { + expect(statsOverCeiling(stats({ reason: "ceiling" }))).toBe(true); + expect(statsOverCeiling(stats({ reason: "ceiling", requestable: ["scalar", "full"] }))).toBe(true); + }); + + it("is true for a session the server sized with nothing left above it to ask for", () => { + expect(statsOverCeiling(stats({ reason: "size", requestable: [] }))).toBe(true); + expect(statsOverCeiling(stats({ reason: "size", tier: "scalar", requestable: ["scalar"] }))).toBe(true); + }); + + it("is false while a tier is left to ask for", () => { + expect(statsOverCeiling(stats({ reason: "size", requestable: ["scalar", "full"] }))).toBe(false); + expect(statsOverCeiling(stats({ reason: "size", tier: "scalar", requestable: ["scalar", "full"] }))).toBe(false); + }); + + it("is false when the server named no requestable (its default is full)", () => { + expect(statsOverCeiling(stats({ reason: "size" }))).toBe(false); + }); + + it("is false for the reasons that are not about size", () => { + expect(statsOverCeiling(stats({ reason: "host", requestable: [] }))).toBe(false); + expect(statsOverCeiling(stats({ reason: "cost", requestable: [] }))).toBe(false); + }); + + it.each(["pending", "complete", "error"] as const)("is false when the status is %s", (status) => { + expect(statsOverCeiling(stats({ status, reason: "ceiling" }))).toBe(false); + }); + + it("is false without stats", () => { + expect(statsOverCeiling(undefined)).toBe(false); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index d63e56e4f..e01074a29 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -54,6 +54,7 @@ export const statsAutoRequest = (_stats: DFMetaStats | undefined): boolean => fa export const nextRequestTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; export const canRequestStats = (_stats: DFMetaStats | undefined): boolean => false; export const demandTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; +export const statsOverCeiling = (_stats: DFMetaStats | undefined): boolean => false; export interface BuckarooOptions { sampled: string[]; diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index aa2c947cc..bed3d241a 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -574,6 +574,26 @@ describe("auto_request false: only the demand columns are asked for (rows-first model.set("df_meta", meta(policy({ demand_columns: ["a"], reason: "ceiling" }))); await tick(10_000); expect(model.sent).toEqual([demand(3, ["a"])]); + + // A full frame for the same state (a search highlight) does not restart it. + model.frame({ df_meta: meta(policy({ demand_columns: ["a"], reason: "ceiling" })), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("a full frame for the same state does not end it: the reply to the request in flight is still answered", async () => { + const model = makeModel(policy({ demand_columns: ["a", "c"] })); + await sendFirst(model); + // A search highlight comes back as a full frame with the same stats. + model.frame({ df_meta: meta(policy({ demand_columns: ["a", "c"] })), df_data_dict: dict([statRow("dtype")]) }); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a", "c"])]); + + model.set("df_data_dict", dict([statRow("dtype"), statRow("min")])); + await tick(); + expect(model.sent).toEqual([demand(3, ["a", "c"]), demand(3, ["a", "c"])]); }); it("asks for the next gen's demand after a state change", async () => { @@ -668,8 +688,24 @@ describe("forceStats: the control's request (rows-first c5)", () => { it("sends nothing when no tier is left to ask for", () => { const model = makeModel(notComputed({ requestable: [], reason: "ceiling" })); + const before = model.get("df_meta"); expect(forceStats(model)).toBe(false); expect(model.sent).toEqual([]); + expect(model.get("df_meta")).toBe(before); + }); + + // The server cannot push a frame to a capable client, so the control marks + // the stats pending itself: the loading text and the placeholder rows show + // at once, and the control is gone, so a second click cannot start a second + // chain of requests. + it("marks the stats pending in a new df_meta, keeping the rest of it, so the control gives way to the loading state", () => { + const model = makeModel(notComputed()); + const before = model.get("df_meta"); + expect(forceStats(model)).toBe(true); + const after = model.get("df_meta"); + expect(after).not.toBe(before); + expect(after.stats).toEqual({ ...notComputed(), status: "pending" }); + expect(after.total_rows).toBe(before.total_rows); }); it("sends nothing when df_meta carries no stats.gen", () => { @@ -767,6 +803,25 @@ describe("a forced run is continued by the scheduler (rows-first c5)", () => { expect(model.sent).toEqual([]); }); + it("does not continue a run recorded before the scheduler started", async () => { + const model = makeModel(notComputed()); + model.state.stats_forced = { gen: 5, tier: "scalar" }; + start(model); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("a full frame for the same gen ends the run: the model shows what the server says, and a late reply is not continued", async () => { + const model = await startForced(); + // The server's own view of the session is still not computed. + model.frame({ df_meta: meta(notComputed()), df_data_dict: dict([statRow("dtype")]) }); + await tick(10_000); + model.set("df_data_dict", dict([statRow("dtype"), statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + it("does nothing for a model nobody started a scheduler on", async () => { const model = makeModel(notComputed()); forceStats(model); @@ -1058,11 +1113,14 @@ describe("wired into WebSocketModel", () => { expect(forceStats(model)).toBe(true); expect(ws.sent).toEqual([forcedRequest()]); + // The control marked the stats pending, and they stay pending until the + // final reply says otherwise. + expect(model.get("df_meta").stats.status).toBe("pending"); + const pendingMeta = model.get("df_meta"); ws.deliver({ ...update(3, "min", false, 2), tier: "scalar" }); await tick(); expect(ws.sent).toEqual([forcedRequest(), forcedRequest()]); - // Still not computed until the final reply says otherwise. - expect(model.get("df_meta").stats.status).toBe("not_computed"); + expect(model.get("df_meta")).toBe(pendingMeta); ws.deliver({ ...update(3, "max", true, 0), tier: "scalar" }); await tick(10_000); @@ -1080,6 +1138,31 @@ describe("wired into WebSocketModel", () => { expect(model.get("df_meta").stats).toMatchObject({ status: "not_computed", reason: "ceiling", gen: 3 }); }); + it("a request the server refuses with not_requestable returns the session to not computed and ends the run", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model); + expect(model.get("df_meta").stats.status).toBe("pending"); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + await tick(10_000); + expect(ws.sent).toEqual([forcedRequest()]); + expect(model.get("df_meta").stats).toEqual(policy({ gen: 3 })); + }); + + it("a run for some columns ends not computed, and the control can ask again", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model, { columns: ["a"] }); + expect(ws.sent).toEqual([forcedRequest({ columns: ["a"] })]); + + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + expect(model.get("df_meta").stats).toEqual(policy({ gen: 3 })); + + expect(forceStats(model)).toBe(true); + expect(ws.sent).toEqual([forcedRequest({ columns: ["a"] }), forcedRequest()]); + expect(ws.sent[1]).not.toHaveProperty("columns"); + }); + it("asks for the demand columns after the first rows and nothing else", async () => { const { ws, model } = makeSocketModel(policy({ gen: 3, demand_columns: ["a"] })); model.set("visible_columns", ["a", "b"]); diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx index 20bee07f4..4f63c9ca3 100644 --- a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -19,7 +19,9 @@ * The NotComputed story (rows-first c5) is a session the server's policy left * without stats: typed columns over schema-tier stats, a main and a summary * view, and the policy in df_meta.stats. The buttons switch the view and the - * reason the stats were not computed; the controls send through `forceStats`. + * reason the stats were not computed (which also puts the session back as the + * server described it). The controls send through `forceStats`, which marks the + * stats pending on the model, as it does for a real session. */ import type { Meta, StoryObj } from "@storybook/react"; import React, { useMemo, useState } from "react"; @@ -275,7 +277,11 @@ const NotComputedInner: React.FC = () => { }); const [operations, setOperations] = useState([]); - const df_meta = useMemo( + // What the server said for the reason picked, and what the page made of it + // since: forceStats marks the stats pending on the model, as a real session + // does, and the story shows that. + const [override, setOverride] = useState(undefined); + const serverMeta = useMemo( () => ({ total_rows: typedData.length, @@ -295,13 +301,16 @@ const NotComputedInner: React.FC = () => { }) as DFMeta, [reason], ); + const df_meta = override ?? serverMeta; // The model the controls send through: it answers get("df_meta") from the - // story's state and logs what it is asked to send. + // story's state, keeps what is set on it and logs what it is asked to send. const model = useMemo( () => ({ get: (key: string) => (key === "df_meta" ? df_meta : undefined), - set: () => {}, + set: (key: string, value: unknown) => { + if (key === "df_meta") setOverride(value as DFMeta); + }, send: (msg: unknown) => setSent((log) => [...log, msg]), }), [df_meta], @@ -335,7 +344,15 @@ const NotComputedInner: React.FC = () => { ))} {(Object.keys(REASONS) as Reason[]).map((r) => ( - ))} From 1737685957af0d81e45dfe5fc1ee1b6667c74597 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 07:07:58 -0400 Subject: [PATCH 15/18] feat(client): not_computed states, compute control and demand units for the stats policy (rows-first c5) The client side of plan 3 phase 5, against the df_meta.stats fields of the policy wire (p33): tier_target, reason, auto_request, requestable, estimate, demand_columns, omitted_keys and approx_keys. - ?caps= carries stats_update and stats_ondemand, from both wiring copies, so a server applies its policy to this client. - A final stats_update may carry status and reason. {final: true, status: "not_computed", reason: "ceiling"} with no payload ends in the ceiling message; a reply with a payload and a status merges it and leaves that status. - The control is forceStats: stats_request {force, tier}, the tier being the smallest requestable above the one reached (scalar before full), with a per-column form that names columns. It marks the stats pending, and the scheduler continues the run, one request per reply that is not final. - With auto_request false the scheduler asks only for demand_columns, as one scoped request per gen and list of columns. - The status bar says why the stats are missing and offers the control unless the ceiling refused them. The summary view shows an empty state with the reason, the size and the control, and a column picker, in place of its grid. Nothing changes for a session whose df_meta.stats is absent. Needs a buckaroo-js-core release and a tallyman bump. Co-Authored-By: Claude Sonnet 5.5 --- .../src/components/BuckarooWidgetInfinite.tsx | 52 ++-- .../src/components/StatsEmptyState.tsx | 108 +++++++- .../src/components/StatusBar.tsx | 36 ++- .../src/components/WidgetTypes.tsx | 58 ++++- packages/buckaroo-js-core/src/index.ts | 4 +- .../src/server/BuckarooView.tsx | 11 +- .../src/server/StateOrchestrator.test.ts | 4 +- .../src/server/StateOrchestrator.ts | 238 +++++++++++++++--- .../src/server/StatsChannel.ts | 66 +++-- .../buckaroo-js-core/src/style/dcf-npm.css | 27 ++ packages/js/standalone.tsx | 9 +- 11 files changed, 518 insertions(+), 95 deletions(-) diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index c6f382ca8..df9547471 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -5,6 +5,7 @@ import { ColumnsEditor } from "./ColumnsEditor"; import { DFData } from "./DFViewerParts/DFWhole"; import { StatusBar } from "./StatusBar"; +import { StatsColumnOption, StatsEmptyState } from "./StatsEmptyState"; import { BuckarooState } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; import { DFMeta, getStatsStatus } from "./WidgetTypes"; @@ -12,6 +13,7 @@ import { CommandConfigT } from "./CommandUtils"; import { Operation } from "./OperationUtils"; import { getDs, + getFieldVal, IDisplayArgs, } from "./DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "./DFViewerParts/displayArgsUtils"; @@ -229,6 +231,19 @@ export function BuckarooInfiniteWidget({ [cDisp, df_data_dict, mainDs, df_meta.total_rows], ); + // A view that lists stats (the summary view, whose rows are not the + // data's) has nothing to show while the stats are not computed. It shows + // why instead, with the control that asks for them. + const statsEmptyState = df_meta.stats?.status === "not_computed" && cDisp.data_key !== "main"; + const statsColumns = useMemo( + () => + (cDisp.df_viewer_config?.column_config ?? []).map((cc) => ({ + field: getFieldVal(cc), + label: "header_name" in cc ? String(cc.header_name) : cc.col_path.join(" / "), + })), + [cDisp], + ); + // Operations minus filter-like ops (search, OnlyOutliers — Python // marks them with meta.quick_command=true). These don't change row // identity; quick_command_args already carries the same info, so @@ -354,20 +369,29 @@ export function BuckarooInfiniteWidget({ componentConfig={effectiveDisplayArgs['main']?.df_viewer_config?.component_config as Record | undefined} onComputeStats={on_compute_stats} /> - + {statsEmptyState && df_meta.stats !== undefined ? ( + + ) : ( + + )}
{buckaroo_state.show_commands ? ( void; themeConfig?: ThemeConfig; } -// Stub: the real functions come with the fix. -export function formatStatsEstimate(_estimate: DFMetaStats["estimate"]): string { - return ""; +const countFormat = new Intl.NumberFormat("en-US"); +const compactFormat = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); + +const plural = (n: number, noun: string): string => (n === 1 ? `1 ${noun}` : `${countFormat.format(n)} ${noun}s`); + +/** + * The size the server's decision rested on, as "12.4M rows x 44 columns". A + * million rows and over read as a compact count. Empty when the server sent no + * estimate. + */ +export function formatStatsEstimate(estimate: DFMetaStats["estimate"]): string { + const parts: string[] = []; + if (typeof estimate?.rows === "number") { + parts.push(estimate.rows >= 1_000_000 ? `${compactFormat.format(estimate.rows)} rows` : plural(estimate.rows, "row")); + } + if (typeof estimate?.cols === "number") parts.push(plural(estimate.cols, "column")); + return parts.join(" x "); +} + +/** Why the stats were not computed, in a sentence: the server's reason for a + * session that is not computed, with the size when it sent one. A session at + * the server's ceiling reads as over the size limit, however the server put it. */ +export function notComputedMessage(stats: DFMetaStats): string { + const size = formatStatsEstimate(stats.estimate); + const sizeSuffix = size === "" ? "" : ` (${size})`; + switch (statsOverCeiling(stats) ? "ceiling" : stats.reason) { + case "ceiling": + return `Summary stats are not available for this table: it is over the size limit${sizeSuffix}.`; + case "size": + return `Summary stats were not computed for this table because it is large${sizeSuffix}.`; + case "host": + return `Summary stats were turned off for this table${sizeSuffix}.`; + case "cost": + return `Summary stats were paused because computing them was taking too long${sizeSuffix}.`; + default: + return `Summary stats are not computed for this table${sizeSuffix}.`; + } } -export function StatsEmptyState(_props: StatsEmptyStateProps) { - return null; +const TIER_DETAILS: Record = { + scalar: { label: "Compute basic stats", title: "Null counts, min, max, mean and std for each column" }, + full: { label: "Compute full stats", title: "Every summary stat, with histograms and value counts. Slower." }, +}; + +/** + * What the summary view shows while the stats are not computed: why, and the + * control that asks for them. The control is one button for the next tier the + * server allows (basic before full), with a picker that limits it to one column. + * A session the server's ceiling refused gets the message and nothing else. + */ +export function StatsEmptyState({ stats, columns = [], onComputeStats, themeConfig }: StatsEmptyStateProps) { + const osColorScheme = useColorScheme(); + const scheme = resolveColorScheme(osColorScheme, themeConfig); + const colors = resolveThemeColors(scheme, themeConfig); + const [picked, setPicked] = useState(""); + + const tier = nextRequestTier(stats); + const canCompute = onComputeStats !== undefined && canRequestStats(stats); + const column = columns.some((c) => c.field === picked) ? picked : ""; + const details = TIER_DETAILS[tier ?? ""] ?? TIER_DETAILS.full; + const label = stats.reason === "cost" ? "Continue computing stats" : details.label; + + return ( +
+

{notComputedMessage(stats)}

+ {canCompute ? ( +
+ {columns.length > 0 ? ( + + ) : null} + +
+ ) : null} +
+ ); } diff --git a/packages/buckaroo-js-core/src/components/StatusBar.tsx b/packages/buckaroo-js-core/src/components/StatusBar.tsx index 50894140a..57e5bc763 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.tsx @@ -4,7 +4,8 @@ import * as _ from "lodash-es"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { ColDef, GridApi, GridOptions } from "ag-grid-community"; import { basicIntFormatter } from "./DFViewerParts/Displayer"; -import { DFMeta, DFMetaStats } from "./WidgetTypes"; +import { DFMeta, DFMetaStats, canRequestStats, statsOverCeiling } from "./WidgetTypes"; +import { notComputedMessage } from "./StatsEmptyState"; import { BuckarooOptions } from "./WidgetTypes"; import { BuckarooState, BKeys } from "./WidgetTypes"; import { CustomCellEditorProps } from 'ag-grid-react'; @@ -312,13 +313,27 @@ export const SearchEditor = memo(({ value, onValueChange, stopEditing }: Custom * loading while they are pending, a control to ask for them while they are not * computed, the reason when they failed. The cell always renders one line in a * fixed-width column, so changing status moves nothing. + * + * A session that is not computed offers the control unless the server's ceiling + * refused the stats or nothing is left to ask for. The control's label reads + * Continue when a run was paused on cost. It calls the host's callback with no + * arguments; the host asks for the tier df_meta.stats allows (see forceStats). */ -export const StatsStatusCell = function (params: { value?: DFMetaStats; context?: { onComputeStats?: (opts?: { columns?: string[] }) => void } }) { +export const StatsStatusCell = function (params: { + value?: DFMetaStats; + context?: { onComputeStats?: (opts?: { columns?: string[] }) => void }; +}) { const stats = params.value; if (stats === undefined) return null; const onComputeStats = params.context?.onComputeStats; const cell = (content: React.ReactNode, extra: React.HTMLAttributes = {}) => ( - + {content} ); @@ -331,16 +346,21 @@ export const StatsStatusCell = function (params: { value?: DFMetaStats; context? , { role: "status", "aria-live": "polite" }, ); - case "not_computed": - return onComputeStats ? ( + case "not_computed": { + const message = notComputedMessage(stats); + if (statsOverCeiling(stats)) { + return cell("Summary stats unavailable: size limit", { title: message }); + } + return onComputeStats && canRequestStats(stats) ? ( cell( - , ) ) : ( - cell("Summary stats not computed") + cell("Summary stats not computed", { title: message }) ); + } case "error": { const text = stats.reason ? `Stats error: ${stats.reason}` : "Stats error"; return cell(text, { title: text }); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index e01074a29..370a25ce9 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -48,13 +48,57 @@ export interface DFMeta { export const getStatsStatus = (meta: DFMeta | undefined): StatsStatus => meta?.stats?.status ?? "complete"; -// Stubs: the real helpers come with the fix. -export const statsRequestable = (_stats: DFMetaStats | undefined): string[] => []; -export const statsAutoRequest = (_stats: DFMetaStats | undefined): boolean => false; -export const nextRequestTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; -export const canRequestStats = (_stats: DFMetaStats | undefined): boolean => false; -export const demandTier = (_stats: DFMetaStats | undefined): StatsTier | undefined => undefined; -export const statsOverCeiling = (_stats: DFMetaStats | undefined): boolean => false; +// What a server that leaves a policy field out means by it (see DFMetaStats). +const DEFAULT_REQUESTABLE: readonly string[] = ["full"]; + +export const statsRequestable = (stats: DFMetaStats | undefined): string[] => + stats?.requestable ?? [...DEFAULT_REQUESTABLE]; + +export const statsAutoRequest = (stats: DFMetaStats | undefined): boolean => stats?.auto_request !== false; + +const tierRank = (tier: string | undefined): number => { + const rank = STATS_TIERS.indexOf(tier as StatsTier); + return rank === -1 ? 0 : rank; +}; + +/** + * The tier a request for more stats should ask for: the smallest tier in + * `requestable` above the one reached, so scalar goes before full. Names that + * are not tiers are ignored. Undefined when there is nothing left to ask for. + */ +export const nextRequestTier = (stats: DFMetaStats | undefined): StatsTier | undefined => { + if (stats === undefined) return undefined; + const requestable = statsRequestable(stats); + const reached = tierRank(stats.tier); + return STATS_TIERS.find((tier) => requestable.includes(tier) && tierRank(tier) > reached); +}; + +/** + * Whether the server's ceiling keeps the stats from being computed. It says so + * with reason "ceiling" when the ceiling cut a request down. When the server + * sized the session at the ceiling itself the reason is "size" and there is + * nothing above it to ask for, which comes to the same thing for the user. + */ +export const statsOverCeiling = (stats: DFMetaStats | undefined): boolean => + stats?.status === "not_computed" && + (stats.reason === "ceiling" || + (stats.reason === "size" && stats.requestable !== undefined && nextRequestTier(stats) === undefined)); + +/** Whether the "Compute summary stats" control applies: the stats are not + * computed, the server's ceiling did not refuse them, and a tier is left. */ +export const canRequestStats = (stats: DFMetaStats | undefined): boolean => + stats?.status === "not_computed" && !statsOverCeiling(stats) && nextRequestTier(stats) !== undefined; + +/** + * The tier of a request for the columns styling needs stats for (`demand_columns`): + * the smallest tier from scalar up that the policy allows, the target included, + * since scalar is where min and max come from. Undefined when it allows none. + */ +export const demandTier = (stats: DFMetaStats | undefined): StatsTier | undefined => { + if (stats === undefined) return undefined; + const allowed = [...statsRequestable(stats), ...(stats.tier_target === undefined ? [] : [stats.tier_target])]; + return STATS_TIERS.find((tier) => tierRank(tier) >= tierRank("scalar") && allowed.includes(tier)); +}; export interface BuckarooOptions { sampled: string[]; diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 85923ae9d..f3969d455 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -20,7 +20,7 @@ import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; import { makeLatestDictDecoder } from './server/latestDictDecoder'; import { withStatsCapability } from './server/StatsChannel'; -import { StateOrchestrator, requestStats, setVisibleColumns } from './server/StateOrchestrator'; +import { StateOrchestrator, forceStats, requestStats, setVisibleColumns } from './server/StateOrchestrator'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -67,6 +67,7 @@ export default { withStatsCapability, StateOrchestrator, requestStats, + forceStats, setVisibleColumns, }; @@ -101,6 +102,7 @@ export { withStatsCapability, StateOrchestrator, requestStats, + forceStats, setVisibleColumns, }; diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 88788cac1..855920862 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -10,7 +10,7 @@ import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; -import { requestStats, setVisibleColumns } from "./StateOrchestrator"; +import { forceStats, setVisibleColumns } from "./StateOrchestrator"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -285,10 +285,11 @@ export function BuckarooView({ model.save_changes(); }, [model]); - // The status bar's Compute summary stats button, shown while the server - // reports the stats as not computed. - const onComputeStats = React.useCallback(() => { - requestStats(model, { force: true }); + // The Compute summary stats control, shown while the server reports the + // stats as not computed: the status bar's button, and the summary view's + // button and per-column form (which pass their columns). + const onComputeStats = React.useCallback((opts?: { columns?: string[] }) => { + forceStats(model, opts); }, [model]); // The columns the grid shows, kept on the model for stats requests to carry diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index bed3d241a..1a17b41c2 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -785,7 +785,9 @@ describe("a forced run is continued by the scheduler (rows-first c5)", () => { it("drops the run when the gen moves on, and does not continue a late reply", async () => { const model = await startForced(); - model.frame({ df_meta: meta(notComputed({ gen: 6 })), df_data_dict: dict() }); + // The next gen's frame, with the stats pending as the run left them: only + // the gen differs. + model.frame({ df_meta: meta(notComputed({ gen: 6, status: "pending" })), df_data_dict: dict() }); await tick(); // A reply for the old run arrives after the new frame. model.set("df_data_dict", dict([statRow("min")])); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts index d43b07bd4..72a782eba 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts @@ -19,11 +19,20 @@ * A server that does not know either field ignores them, runs the whole run and * answers with a final reply, which ends the run the same way. * + * A session the server's policy left without stats says so with status + * "not_computed" and the fields of its policy (see DFMetaStats). The scheduler + * asks for nothing on such a session, with two exceptions. With `auto_request` + * false it asks for the columns in `demand_columns`, whose styling needs stats + * now, as one scoped request per gen (see demand runs below). And it continues + * a run the user started with the "Compute summary stats" control (see + * forceStats): each reply that is not final is answered with the same request. + * * It watches the model and sends through it, so any IModel works: * * change:df_meta the status and stats_gen of the state on screen * change:df_data_dict a reply merged into all_stats (StatsChannel's job) * change:buckaroo_state a state change made on this client + * change:stats_forced a run the user started (forceStats) * msg:custom an infinite_resp, the first rows * * The columns the grid shows are read from the model's `visible_columns` key @@ -43,7 +52,15 @@ * `stats_gen` is the server's token for the state a reply describes. A reply for * another gen is dropped by StatsChannel, so no client-side token is kept here. */ -import { BuckarooState, DFMeta } from "../components/WidgetTypes"; +import { + BuckarooState, + DFMeta, + DFMetaStats, + StatsTier, + demandTier, + nextRequestTier, + statsAutoRequest, +} from "../components/WidgetTypes"; import { IModel } from "./IModel"; /** What the scheduler needs of a model. */ @@ -76,38 +93,75 @@ export interface StatsRequestOptions { force?: boolean; /** The tier asked for. */ tier?: string; - /** The columns the request is for. */ + /** The columns the request is for. A forced request names the columns it + * is for or none, so the grid's columns are not added to it. A request + * that is not forced carries them as a hint when it names none. */ columns?: string[]; } -/** The model key that records a forced run, for the scheduler to continue. */ -export const FORCED_RUN_KEY = "stats_forced"; - -// Stub: the real function comes with the fix. -export function forceStats(_model: Pick, _opts: { columns?: string[] } = {}): boolean { - return false; -} - /** * Send a time-boxed `stats_request` for the stats_gen of the state the model - * shows, with the columns the grid shows as the hint when they are known. - * Returns false, and sends nothing, when its df_meta carries no stats.gen. + * shows. The scheduler's requests carry the columns the grid shows as the hint, + * when they are known; a request that names its columns carries those instead, + * and a forced one carries only those. Returns false, and sends nothing, when + * the model's df_meta carries no stats.gen. */ export function requestStats(model: Pick, opts: StatsRequestOptions = {}): boolean { const gen = (model.get("df_meta") as DFMeta | undefined)?.stats?.gen; if (typeof gen !== "number") return false; - const columns = model.get(VISIBLE_COLUMNS_KEY) as string[] | undefined; + const columns = opts.columns ?? (opts.force ? undefined : (model.get(VISIBLE_COLUMNS_KEY) as string[] | undefined)); model.send({ type: "stats_request", stats_gen: gen, scope: "raw", incremental: true, + ...(opts.tier === undefined ? {} : { tier: opts.tier }), ...(Array.isArray(columns) && columns.length > 0 ? { columns } : {}), ...(opts.force ? { force: true } : {}), }); return true; } +/** The model key that records a run the user started, for the scheduler to + * continue. */ +export const FORCED_RUN_KEY = "stats_forced"; + +/** What forceStats records under FORCED_RUN_KEY. */ +export interface ForcedRun { + gen: number; + tier: string; + columns?: string[]; +} + +/** + * The "Compute summary stats" control: ask for stats the server did not plan to + * compute, `stats_request {force: true, tier}`. The tier is the smallest the + * server allows above the one reached (scalar before full), and `opts.columns` + * is the per-column form. Sends nothing, and returns false, when no tier is + * left to ask for or df_meta carries no stats.gen. + * + * The stats are marked pending in a new df_meta. The server sends no frame to a + * capable client, so without this the loading text and the placeholder rows + * would wait for the first reply, and the control would stay on screen to be + * clicked again. A final reply, a refusal or a frame from the server puts the + * status it names in place of it. + * + * The run is recorded on the model, where a scheduler started on it picks it up + * and answers each reply that is not final with the same request. + */ +export function forceStats(model: Pick, opts: { columns?: string[] } = {}): boolean { + const meta = model.get("df_meta") as DFMeta | undefined; + const stats = meta?.stats; + const tier = nextRequestTier(stats); + if (meta === undefined || stats === undefined || tier === undefined || typeof stats.gen !== "number") return false; + const columns = opts.columns !== undefined && opts.columns.length > 0 ? opts.columns : undefined; + if (!requestStats(model, { force: true, tier, columns })) return false; + const run: ForcedRun = { gen: stats.gen, tier, ...(columns === undefined ? {} : { columns }) }; + model.set("df_meta", { ...meta, stats: { ...stats, status: "pending" } }); + model.set(FORCED_RUN_KEY, run); + return true; +} + export interface OrchestratorOptions { model: StatsModel; /** Lower bound on the delay before asking for a new state's stats. Default 200 ms. */ @@ -126,6 +180,34 @@ export interface OrchestratorOptions { type Timer = ReturnType; +// What the scheduler is driving. "auto" is the run of a pending session. A +// "demand" run asks for the columns whose styling needs stats, on a session +// that does not auto-request: a scoped request, at the tier that carries min and +// max. A "forced" run is one the user started; its first request was sent by +// forceStats and the scheduler sends the rest. +type Run = + | { kind: "auto" } + | { kind: "demand"; columns: string[]; tier: StatsTier } + | { kind: "forced"; gen: number; tier: string; columns?: string[] }; + +// The parts of df_meta.stats a demand or forced run is watching: a change in +// any of them means the reply to the run's request has arrived and ended it. +// A new df_meta with the same three is a full frame for the same state. +interface StatsBasis { + status: string; + reason?: string; + tier?: string; +} + +const basisOf = (stats: DFMetaStats | undefined): StatsBasis => ({ + status: stats?.status ?? "", + reason: stats?.reason, + tier: stats?.tier, +}); + +const sameBasis = (a: StatsBasis, b: StatsBasis): boolean => + a.status === b.status && a.reason === b.reason && a.tier === b.tier; + export class StateOrchestrator { private readonly model: StatsModel; private readonly minDebounceMs: number; @@ -135,8 +217,14 @@ export class StateOrchestrator { private readonly firstPaintTimeoutMs: number; private started = false; - // The stats_gen being driven; undefined while nothing is pending. - private gen: number | undefined; + // The run being driven and the key that identifies it (the gen, and for a + // demand run the columns and tier); undefined while nothing is. + private run: Run | undefined; + private runKey: string | undefined; + // The stats a demand or forced run started from. + private basis: StatsBasis | undefined; + // A demand run that has ended, which is not started again. + private doneKey: string | undefined; // A request is out and its reply has not been seen. private inFlight = false; // Delay before the next request: 0 for the first state and for each reply @@ -153,6 +241,7 @@ export class StateOrchestrator { private seenMeta: unknown; private seenDict: unknown; private seenState: BuckarooState | undefined; + private seenForced: unknown; constructor(opts: OrchestratorOptions) { this.model = opts.model; @@ -170,9 +259,12 @@ export class StateOrchestrator { this.seenMeta = this.model.get("df_meta"); this.seenDict = this.model.get("df_data_dict"); this.seenState = this.model.get("buckaroo_state"); + // A run recorded before this scheduler started is not its to continue. + this.seenForced = this.model.get(FORCED_RUN_KEY); this.model.on("change:df_meta", this.onModelChange); this.model.on("change:df_data_dict", this.onModelChange); this.model.on("change:buckaroo_state", this.onState); + this.model.on(`change:${FORCED_RUN_KEY}`, this.onModelChange); this.model.on("msg:custom", this.onMessage); this.sync(); this.began = true; @@ -185,8 +277,10 @@ export class StateOrchestrator { this.model.off("change:df_meta", this.onModelChange); this.model.off("change:df_data_dict", this.onModelChange); this.model.off("change:buckaroo_state", this.onState); + this.model.off(`change:${FORCED_RUN_KEY}`, this.onModelChange); this.model.off("msg:custom", this.onMessage); this.standDown(); + this.doneKey = undefined; this.began = false; } @@ -216,8 +310,8 @@ export class StateOrchestrator { private readonly onState = (next?: BuckarooState): void => { const prev = this.seenState; this.seenState = next ?? this.model.get("buckaroo_state"); - if (this.gen === undefined || !touchesDataflow(prev, this.seenState)) return; - this.begin(this.gen); + if (this.run === undefined || this.run.kind === "forced" || !touchesDataflow(prev, this.seenState)) return; + this.begin(this.runKey!, this.run, this.basis); }; private readonly onMessage = (msg?: { type?: string }): void => { @@ -233,27 +327,97 @@ export class StateOrchestrator { this.seenDict = dict; const stats = meta?.stats; - if (stats?.status !== "pending" || typeof stats.gen !== "number") { - // Complete, not computed, an error, or a server that reports no stats. + if (this.adoptForced(stats)) return; + if (this.run?.kind === "forced") { + this.syncForced(stats, metaChanged, dictChanged); + if (this.run !== undefined) return; + // The run is over; what it left is handled like any other state. + } + + const desired = this.desiredRun(stats); + if (desired === undefined) { + // Complete, an error, a session that asks for nothing, or a server + // that reports no stats. + this.standDown(); + } else if (desired.key === this.runKey) { + if (this.inFlight && dictChanged && !metaChanged) { + // A new df_data_dict under the same df_meta is a stats_update that + // is not final (a full frame for this state replaces both). One + // request per reply: ask again. + this.inFlight = false; + this.noteRequestTime(); + this.arm(); + } else if (this.inFlight && metaChanged && desired.run.kind === "demand" && this.basisChanged(stats)) { + // A refusal, or a final reply that left the session in another + // state: the run is over and is not asked for again. + this.doneKey = this.runKey; + this.standDown(); + } + } else if (desired.key !== this.doneKey) { + this.begin(desired.key, desired.run, basisOf(stats)); + } + } + + // The run the stats ask for, if any. A pending session is run whole, unless + // the server says not to auto-request, and then only the columns whose + // styling needs stats are asked for. A not computed session is asked for + // nothing, on the same condition. + private desiredRun(stats: DFMetaStats | undefined): { key: string; run: Run } | undefined { + if (stats === undefined || typeof stats.gen !== "number") return undefined; + if (stats.status !== "pending" && stats.status !== "not_computed") return undefined; + if (statsAutoRequest(stats)) { + return stats.status === "pending" ? { key: JSON.stringify(["auto", stats.gen]), run: { kind: "auto" } } : undefined; + } + const columns = stats.demand_columns; + const tier = demandTier(stats); + if (!Array.isArray(columns) || columns.length === 0 || tier === undefined) return undefined; + return { key: JSON.stringify(["demand", stats.gen, tier, columns]), run: { kind: "demand", columns, tier } }; + } + + // A run the user started has been recorded on the model, and its first + // request has gone out: continue it, and give way to what the scheduler was + // driving. Returns whether it took one over. + private adoptForced(stats: DFMetaStats | undefined): boolean { + const record = this.model.get(FORCED_RUN_KEY) as ForcedRun | undefined; + if (record === this.seenForced) return false; + this.seenForced = record; + if (record === undefined || record.gen !== stats?.gen) return false; + this.standDown(); + this.runKey = JSON.stringify(["forced", record.gen]); + this.run = { kind: "forced", ...record }; + this.basis = basisOf(stats); + this.inFlight = true; + this.sentAt = Date.now(); + return true; + } + + private syncForced(stats: DFMetaStats | undefined, metaChanged: boolean, dictChanged: boolean): void { + const run = this.run; + if (run?.kind !== "forced") return; + if (stats?.gen !== run.gen || (metaChanged && this.basisChanged(stats))) { + // A final reply, a refusal or a new state ends the run. this.standDown(); - } else if (stats.gen !== this.gen) { - this.begin(stats.gen); } else if (this.inFlight && dictChanged && !metaChanged) { - // A new df_data_dict under the same df_meta is a stats_update that - // is not final (a full frame for this state replaces both). One - // request per reply: ask again. + // A reply that is not final: ask again. this.inFlight = false; this.noteRequestTime(); this.arm(); } } - // Wait for rows, then ask, for `gen`. The state the model starts with asks as - // soon as rows are up; every later one is a state change and waits out the - // debounce. - private begin(gen: number): void { + private basisChanged(stats: DFMetaStats | undefined): boolean { + return this.basis === undefined || !sameBasis(this.basis, basisOf(stats)); + } + + // Wait for rows, then ask, for the run `key`. The state the model starts + // with asks as soon as rows are up; every later one is a state change and + // waits out the debounce. + private begin(key: string, run: Run, basis: StatsBasis | undefined): void { this.clearTimers(); - this.gen = gen; + this.runKey = key; + this.run = run; + this.basis = basis; + this.doneKey = undefined; this.inFlight = false; this.delayMs = this.began ? this.computeDebounce() : 0; this.paintTimer = setTimeout(() => { @@ -265,7 +429,9 @@ export class StateOrchestrator { private standDown(): void { if (this.inFlight) this.noteRequestTime(); this.clearTimers(); - this.gen = undefined; + this.run = undefined; + this.runKey = undefined; + this.basis = undefined; this.inFlight = false; } @@ -279,7 +445,7 @@ export class StateOrchestrator { } private arm(): void { - if (this.inFlight || this.gen === undefined || this.requestTimer !== undefined) return; + if (this.inFlight || this.run === undefined || this.requestTimer !== undefined) return; this.requestTimer = setTimeout(() => { this.requestTimer = undefined; this.fire(); @@ -287,7 +453,15 @@ export class StateOrchestrator { } private fire(): void { - if (requestStats(this.model)) { + const run = this.run; + if (run === undefined) return; + const opts: StatsRequestOptions = + run.kind === "demand" + ? { columns: run.columns, tier: run.tier } + : run.kind === "forced" + ? { force: true, tier: run.tier, columns: run.columns } + : {}; + if (requestStats(this.model, opts)) { this.inFlight = true; this.sentAt = Date.now(); this.delayMs = 0; diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts index 9567fa526..4c3d2eb7b 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -8,6 +8,11 @@ * stats_update {stats_gen, scope, tier, final, payload, elapsed_ms} * stats_aborted {stats_gen, current_gen?, scope, reason} * + * A final `stats_update` may also carry `status` and `reason` (rows-first c5): + * `{final: true, status: "not_computed", reason: "ceiling"}`, with no payload, + * answers a request the server refused, and the session is left in that state. + * Without a `status` a final update completes the session. + * * `payload` is an inline wide DFEnvelope holding `all_stats`. `stats_gen` is the * server's counter for the state the stats describe; it rides on every * `initial_state` as `df_meta.stats.gen`, and a reply for any other gen is for @@ -22,12 +27,19 @@ import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; import { DFMeta, DFMetaStats, StatsStatus } from "../components/WidgetTypes"; import { IModel } from "./IModel"; -/** The capability this client advertises, as one value of `?caps=` on the +/** A capability this client advertises, as one value of `?caps=` on the * WebSocket URL: it merges `stats_update` messages. The server records it per * connection when the socket opens, since it sends the first message before * the client can say anything. */ export const STATS_UPDATE_CAP = "stats_update"; +/** The second capability: this client can show a session whose stats are not + * computed, with the reason, and send tiered requests (see forceStats). A + * server applies its stats policy only to a client that has it. */ +export const STATS_ONDEMAND_CAP = "stats_ondemand"; + +const CLIENT_CAPS = [STATS_UPDATE_CAP, STATS_ONDEMAND_CAP]; + const decodeQueryValue = (value: string): string => { try { return decodeURIComponent(value); @@ -36,10 +48,11 @@ const decodeQueryValue = (value: string): string => { } }; -/** `wsUrl` with `caps=stats_update` added: a new query on a bare URL, a new - * parameter after an existing query, or a comma-joined value when the host - * already passes `caps`. The fragment stays last and other parameters are - * left as the host wrote them. */ +/** `wsUrl` with `caps=stats_update,stats_ondemand` added: a new query on a bare + * URL, a new parameter after an existing query, or a comma-joined value when + * the host already passes `caps`, to which only the capabilities it lacks are + * added. The fragment stays last and other parameters are left as the host + * wrote them. */ export function withStatsCapability(wsUrl: string): string { const hashAt = wsUrl.indexOf("#"); const fragment = hashAt === -1 ? "" : wsUrl.slice(hashAt); @@ -50,13 +63,15 @@ export function withStatsCapability(wsUrl: string): string { const capsAt = params.findIndex((p) => p === "caps" || p.startsWith("caps=")); if (capsAt === -1) { - params.push(`caps=${STATS_UPDATE_CAP}`); + params.push(`caps=${CLIENT_CAPS.join(",")}`); } else { const caps = decodeQueryValue(params[capsAt].slice("caps=".length)) .split(",") .map((cap) => cap.trim()) .filter((cap) => cap !== ""); - if (!caps.includes(STATS_UPDATE_CAP)) caps.push(STATS_UPDATE_CAP); + for (const cap of CLIENT_CAPS) { + if (!caps.includes(cap)) caps.push(cap); + } params[capsAt] = `caps=${caps.join(",")}`; } return `${path}?${params.join("&")}${fragment}`; @@ -126,6 +141,19 @@ const genOf = (meta: DFMeta | undefined): number | undefined => { return typeof gen === "number" ? gen : undefined; }; +/** + * `df_meta.stats` after a final update. With no status it is complete at the + * update's tier, and the policy fields, which only describe what is left to + * ask for, go. With a status (a refusal, or a run for some columns only) the + * session keeps its stats and takes the status and the reason, if the update + * names one; the update's tier is what was asked for, not what was reached. + */ +function finalStats(stats: DFMetaStats, msg: StatsUpdateMessage): DFMetaStats { + const status = msg.status ?? "complete"; + if (status === "complete") return { status, tier: msg.tier ?? stats.tier, gen: stats.gen }; + return { ...stats, status, ...(msg.reason === undefined ? {} : { reason: msg.reason }) }; +} + /** What the channel needs of a model. */ type StatsModel = Pick; @@ -172,20 +200,24 @@ export class StatsChannel { } private async applyUpdate(msg: StatsUpdateMessage): Promise { - const update = await decodeDFData(msg.payload); + // A reply that did not run (a request over the ceiling) has no payload + // and nothing to merge. + const update = msg.payload === undefined ? undefined : await decodeDFData(msg.payload); for (;;) { // A frame may have moved the gen on while something decoded. if (msg.stats_gen !== this.expectedGen) return; - const dict: Record | null | undefined = this.model.get("df_data_dict"); - // The dict is decoded when the seed built it and raw when a later - // initial_state did. - const base = await decodeDFData(dict?.all_stats); - // A frame replaced the dict (or moved the gen) while it decoded: - // start over from the new one. - if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; - this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + if (update !== undefined) { + const dict: Record | null | undefined = this.model.get("df_data_dict"); + // The dict is decoded when the seed built it and raw when a later + // initial_state did. + const base = await decodeDFData(dict?.all_stats); + // A frame replaced the dict (or moved the gen) while it decoded: + // start over from the new one. + if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; + this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + } if (msg.final) { - this.replaceStats((stats) => ({ status: "complete", tier: msg.tier ?? stats.tier, gen: stats.gen })); + this.replaceStats((stats) => finalStats(stats, msg)); } return; } diff --git a/packages/buckaroo-js-core/src/style/dcf-npm.css b/packages/buckaroo-js-core/src/style/dcf-npm.css index f7f689283..11660c290 100644 --- a/packages/buckaroo-js-core/src/style/dcf-npm.css +++ b/packages/buckaroo-js-core/src/style/dcf-npm.css @@ -453,6 +453,33 @@ div.dependent-tabs ul.tabs li.active { padding: 0 6px; cursor: pointer; } +/* The summary view while the stats are not computed (rows-first c5): the reason, + * and the control that asks for them, in place of the grid. */ +.bk-stats-empty { + box-sizing: border-box; + min-height: 120px; + padding: 16px; + font-size: 13px; + border: 1px solid rgba(128, 128, 128, 0.4); + border-top: none; +} +.bk-stats-empty-message { + margin: 0 0 12px; +} +.bk-stats-empty-control { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} +.bk-stats-empty-control select, +.bk-stats-empty-control button { + font-size: 12px; + padding: 2px 8px; +} +.bk-stats-empty-control button { + cursor: pointer; +} .status-bar .ag-center-cols-viewport { min-height: unset !important; } diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index bb59ad2ea..29f982316 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -252,10 +252,11 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.save_changes(); }, [model]); - // The status bar's Compute summary stats button, shown while the server - // reports the stats as not computed. - const onComputeStats = React.useCallback(() => { - srt.requestStats(model, { force: true }); + // The Compute summary stats control, shown while the server reports the + // stats as not computed: the status bar's button, and the summary view's + // button and per-column form (which pass their columns). + const onComputeStats = React.useCallback((opts?: { columns?: string[] }) => { + srt.forceStats(model, opts); }, [model]); // The columns the grid shows, kept on the model for stats requests to carry From 7b30aad1e163d4dca68abeeddedc950879e2c47f Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 08:24:43 -0400 Subject: [PATCH 16/18] fix(client): keep the status bar's scrollbar overlay from taking the control's clicks (rows-first c5) The Storybook Playwright job failed on the c5 fix commit: clicking "Compute summary stats" in the status bar timed out with ".ag-body-horizontal-scroll-container from .ag-body-horizontal-scroll.ag-scrollbar-scrolling.ag-scrollbar-invisible subtree intercepts pointer events". The control's button was 20px high in a 20px row, so the browser scrolled the grid to reveal it, AG Grid flagged the horizontal scrollbar as scrolling, and its overlay (positioned over the last 16px of the one-row grid, not drawn) sat above the button. - The button is 16px high and its container fills the row, so it fits inside it. - The status bar's invisible scrollbar overlay takes no pointer events. A scrollbar the browser draws in its own space (no ag-scrollbar-invisible) is left alone, and the strip still scrolls with the wheel. Checked with the overlay held in its scrolling state: a click on the control fails with the CI message on the previous CSS and lands with this one. Co-Authored-By: Claude Sonnet 5.5 --- .../buckaroo-js-core/src/style/dcf-npm.css | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/buckaroo-js-core/src/style/dcf-npm.css b/packages/buckaroo-js-core/src/style/dcf-npm.css index 11660c290..93994c79b 100644 --- a/packages/buckaroo-js-core/src/style/dcf-npm.css +++ b/packages/buckaroo-js-core/src/style/dcf-npm.css @@ -436,10 +436,15 @@ div.dependent-tabs ul.tabs li.active { .bk-status-inflight-dot { animation: none; opacity: 1; } } /* Summary stats status in the status bar (rows-first c4). One line in a - * fixed-width column, whatever the status, so a status change moves nothing. */ + * fixed-width column, whatever the status, so a status change moves nothing. + * The status bar's row is about 20px high, so the control has to fit inside it + * with room to spare: a button that overflows the row makes the browser scroll + * the grid to reveal it, and the grid's scrollbar overlay then covers the + * button and takes its clicks. */ .bk-stats-status { - display: inline-flex; + display: flex; align-items: center; + height: 100%; gap: 6px; max-width: 100%; font-size: 12px; @@ -449,7 +454,7 @@ div.dependent-tabs ul.tabs li.active { } .bk-stats-status button { font-size: 11px; - line-height: 16px; + line-height: 12px; padding: 0 6px; cursor: pointer; } @@ -483,6 +488,14 @@ div.dependent-tabs ul.tabs li.active { .status-bar .ag-center-cols-viewport { min-height: unset !important; } +/* The status bar is a single row, so the grid's overlay scrollbar (positioned + * over the last 16px of the grid, and not drawn) covers most of it. It sits + * above the cells while the grid is scrolling and takes the clicks meant for + * the control. The strip still scrolls with the wheel or a trackpad, and a + * scrollbar the browser draws in its own space (not an overlay) is left alone. */ +.status-bar .ag-body-horizontal-scroll.ag-scrollbar-invisible { + pointer-events: none; +} .status-bar .ag-root-wrapper { /* the status-bar is always displayed with the df-viewer */ From 30bf7c02677b63c91118dc8a97da648c39267a66 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 09:34:32 -0400 Subject: [PATCH 17/18] test(client): failing tests for showing the stats a run for some columns computed (rows-first c5) A scoped run (the per-column form, or the demand columns) ends with status not_computed and a payload that StatsChannel merges into all_stats. The summary view shows the empty state for as long as the status is not_computed, so those stats are not shown anywhere. The tests pin a df_meta.stats.computed_columns list that StatsChannel sets when a run ends not computed, and the summary view showing its grid once the list is not empty. DFMetaStats carries the field as a type only, so the failures are assertions. Two assertions of this branch's own tests that read the end state of a scoped run now expect the list. Co-Authored-By: Claude Sonnet 5.5 --- .../BuckarooInfiniteWidget.flash.test.tsx | 77 +++++++++++++++++ .../src/components/WidgetTypes.tsx | 5 ++ .../src/server/StateOrchestrator.test.ts | 15 +++- .../src/server/StatsChannel.test.ts | 84 ++++++++++++++++++- 4 files changed, 178 insertions(+), 3 deletions(-) diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index b949f63ef..a92870221 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -14,6 +14,7 @@ import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridS import { BuckarooState, BuckarooOptions, DFMeta } from "./WidgetTypes"; import { DFViewerConfig } from "./DFViewerParts/DFWhole"; import { IDisplayArgs } from "./DFViewerParts/gridUtils"; +import { StatsChannel } from "../server/StatsChannel"; jest.mock("ag-grid-react", () => require("../test-utils/agGridSpy").agGridReactMockFactory(), @@ -1037,4 +1038,80 @@ describe("BuckarooInfiniteWidget summary view, stats not computed (rows-first c5 expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); expect(getSpyCalls().mountCount).toBe(1); }); + + // A run for some columns (the per-column form, or the demand columns) ends + // with the session still not computed, and the stats it merged are in + // all_stats. The summary view lists them instead of the empty state. + describe("after a run that computed some columns", () => { + const ranArgs: Record = { + ...baseDisplayArgs, + summary: { + data_key: "empty", + df_viewer_config: { + ...baseConfig, + pinned_rows: [{ primary_key_val: "min", displayer_args: { displayer: "obj" } }], + }, + summary_stats_key: "all_stats", + }, + }; + const ranProps = (meta: DFMeta, allStats: any[]) => + widgetProps({ df_display_args: ranArgs, df_meta: meta, df_data_dict: { empty: [], all_stats: allStats } }); + const lastPinnedRows = (): any[] => { + const sets = getSpyCalls().setGridOption.filter(([k]) => k === "pinnedTopRowData"); + return sets[sets.length - 1][1] as any[]; + }; + const minRows = [ + { index: "dtype", a: "int64" }, + { index: "min", a: 1 }, + ]; + + it("shows the grid, with the stats the run merged, in place of the empty state", () => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + expect(lastPinnedRows()).toEqual([minRows[1]]); + }); + + it("an empty list of columns keeps the empty state", () => { + render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(0); + }); + + it("gives the empty state back when a new gen's frame drops the columns", () => { + const props = ranProps(notComputed({ computed_columns: ["a"] }), minRows); + const { rerender } = render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + + rerender(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + }); + + it("shows the grid for the session a scoped reply leaves, as StatsChannel records it", async () => { + const state: Record = { + df_meta: notComputed(), + df_data_dict: { empty: [], all_stats: [minRows[0]] }, + }; + const channel = new StatsChannel({ get: (key) => state[key], set: (key, value) => { state[key] = value; } }); + const before = state.df_meta; + const reply = { + type: "stats_update", + stats_gen: 1, + scope: "raw", + tier: "scalar", + final: true, + status: "not_computed", + payload: { format: "json", layout: "wide", data: [{ index: "min", level_0: "min", a: 1 }] }, + }; + channel.handle(reply); + await waitFor(() => expect(state.df_meta).not.toBe(before)); + expect(state.df_meta.stats.status).toBe("not_computed"); + + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + expect(lastPinnedRows().map((r) => r.index)).toEqual(["min"]); + expect(lastPinnedRows()[0].a).toBe(1); + }); + }); }); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index 370a25ce9..10a96f43c 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -33,6 +33,11 @@ export interface DFMetaStats { // Stat keys left out of the run, and keys computed from a sample. omitted_keys?: string[]; approx_keys?: string[]; + // Unlike the fields above, the server never sends this one. StatsChannel + // sets it when a run ends with the session still not computed (a request + // for some columns, or for the demand columns): the grid's own names of + // the columns whose stats the run merged into all_stats. + computed_columns?: string[]; } export interface DFMeta { diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts index 1a17b41c2..efe4efcf0 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.test.ts @@ -1158,7 +1158,8 @@ describe("wired into WebSocketModel", () => { ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); await tick(10_000); expect(ws.sent).toHaveLength(1); - expect(model.get("df_meta").stats).toEqual(policy({ gen: 3 })); + // The session is not computed again, and says which column the run filled. + expect(model.get("df_meta").stats).toEqual({ ...policy({ gen: 3 }), computed_columns: ["a"] }); expect(forceStats(model)).toBe(true); expect(ws.sent).toEqual([forcedRequest({ columns: ["a"] }), forcedRequest()]); @@ -1182,5 +1183,17 @@ describe("wired into WebSocketModel", () => { expect(model.get("df_meta").stats.status).toBe("not_computed"); expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "min"]); }); + + it("a demand run's final reply names the columns it filled", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3, demand_columns: ["a"] })); + rowsFromServer(ws); + await tick(); + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(model.get("df_meta").stats).toEqual({ + ...policy({ gen: 3, demand_columns: ["a"] }), + computed_columns: ["a"], + }); + }); }); }); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts index ee8df46e8..a8c4ece8b 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -330,15 +330,95 @@ describe("a final stats_update with a status", () => { expect(model.get("df_meta").stats).toEqual({ ...policyStats, reason: "cost" }); }); - it("merges a payload it carries, and leaves the session not computed (a reply for some columns only)", async () => { + it("merges a payload it carries, and leaves the session not computed, naming the columns it filled (a reply for some columns only)", async () => { const { ws, model, events } = makePolicyModel(); ws.deliver(update(3, [row("min", { a: 1 }), row("max", { a: 9 })], { tier: "scalar", status: "not_computed" })); await settle(); expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "min", "max"]); - expect(model.get("df_meta").stats).toEqual(policyStats); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); expect(events).toEqual(["df_data_dict", "df_meta"]); }); + it("counts a column only where a cell has a value: a null fills nothing", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1, b: null })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["a"]); + }); + + it("names the columns every reply of a run filled, and a reply that is not final leaves df_meta alone", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1, b: null })], { tier: "scalar", final: false })); + await settle(); + expect(model.get("df_meta").stats).toEqual(policyStats); + + ws.deliver(update(3, [row("max", { b: 9, a: null })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a", "b"] }); + }); + + it("a final reply with no payload names the columns the run's earlier replies filled", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "scalar", final: true, status: "not_computed" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); + }); + + it("keeps the columns of an earlier run when a later one ends not computed", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", status: "not_computed" })); + await settle(); + ws.deliver(update(3, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["a", "b"]); + }); + + it("a new gen starts over: the replies for the gen the client left are not counted", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ + type: "initial_state", + df_meta: metaFor({ ...policyStats, gen: 4 }), + df_data_dict: { all_stats: schemaStats() }, + }); + ws.deliver(update(4, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, gen: 4, computed_columns: ["b"] }); + }); + + it("a frame that replaced the stats under the same gen takes the earlier replies' columns with it", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ + type: "initial_state", + df_meta: metaFor(policyStats), + df_data_dict: { all_stats: schemaStats() }, + }); + ws.deliver(update(3, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["b"]); + }); + + it("a refusal after replies that filled columns names them too", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); + }); + + it("an error does not name columns: the session is not a not computed one", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "error" }); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, status: "error", reason: "stats_failed" }); + }); + it("a final update with no status still completes the session and drops the policy fields", async () => { const { ws, model } = makePolicyModel(); ws.deliver(update(3, [row("mean", { a: 2 })])); From 9992b2e2bba159115b24864f164b5dfc5ac8e2c8 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sun, 4 Oct 2026 09:38:18 -0400 Subject: [PATCH 18/18] fix(client): show the stats a run for some columns computed in the summary view (rows-first c5) A run for some columns ends with status not_computed and a payload, and StatsChannel merges the payload into all_stats. The summary view showed the empty state for as long as the status was not_computed, so the stats of the column the user picked, or of the demand columns, were not shown. StatsChannel now keeps the columns the replies of a run filled (a column counts where some stat row has a value for it) and a final reply or a not_requestable refusal that leaves the session not computed puts them in df_meta.stats.computed_columns. A new gen, or a frame that replaced the stats, starts the list over. The summary view shows its grid, not the empty state, when the list is not empty. Co-Authored-By: Claude Sonnet 5.5 --- .../src/components/BuckarooWidgetInfinite.tsx | 9 ++- .../src/server/StatsChannel.ts | 73 +++++++++++++++++-- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index df9547471..d47ffdff4 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -233,8 +233,13 @@ export function BuckarooInfiniteWidget({ // A view that lists stats (the summary view, whose rows are not the // data's) has nothing to show while the stats are not computed. It shows - // why instead, with the control that asks for them. - const statsEmptyState = df_meta.stats?.status === "not_computed" && cDisp.data_key !== "main"; + // why instead, with the control that asks for them. A run for some + // columns leaves the session not computed with those columns' stats in + // all_stats (computed_columns names them), and the grid lists those. + const statsEmptyState = + df_meta.stats?.status === "not_computed" && + cDisp.data_key !== "main" && + (df_meta.stats.computed_columns?.length ?? 0) === 0; const statsColumns = useMemo( () => (cDisp.df_viewer_config?.column_config ?? []).map((cc) => ({ diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts index 4c3d2eb7b..cca73245e 100644 --- a/packages/buckaroo-js-core/src/server/StatsChannel.ts +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -11,7 +11,10 @@ * A final `stats_update` may also carry `status` and `reason` (rows-first c5): * `{final: true, status: "not_computed", reason: "ceiling"}`, with no payload, * answers a request the server refused, and the session is left in that state. - * Without a `status` a final update completes the session. + * Without a `status` a final update completes the session. A run for some + * columns ends the same way, with a payload, and the columns its replies filled + * go in `df_meta.stats.computed_columns`, which the summary view reads to show + * what the run computed. * * `payload` is an inline wide DFEnvelope holding `all_stats`. `stats_gen` is the * server's counter for the state the stats describe; it rides on every @@ -141,17 +144,42 @@ const genOf = (meta: DFMeta | undefined): number | undefined => { return typeof gen === "number" ? gen : undefined; }; +/** The columns of `rows` that hold a value in some stat row. The wide pivot + * fills a stat a column did not carry in a message with null, so a null says + * nothing about the column. */ +function columnsWithValues(rows: DFData): string[] { + const columns = new Set(); + for (const row of rows) { + for (const [column, value] of Object.entries(row)) { + if (column !== "index" && column !== "level_0" && value != null) columns.add(column); + } + } + return Array.from(columns); +} + +/** `stats` with `columns` added to its `computed_columns`, or `stats` itself + * when there is nothing to add. */ +function withComputedColumns(stats: DFMetaStats, columns: string[]): DFMetaStats { + const all = Array.from(new Set([...(stats.computed_columns ?? []), ...columns])); + return all.length === 0 ? stats : { ...stats, computed_columns: all }; +} + /** * `df_meta.stats` after a final update. With no status it is complete at the * update's tier, and the policy fields, which only describe what is left to * ask for, go. With a status (a refusal, or a run for some columns only) the * session keeps its stats and takes the status and the reason, if the update * names one; the update's tier is what was asked for, not what was reached. + * `filled` lists the columns the run's replies merged, and a session left not + * computed adds them to `computed_columns`. */ -function finalStats(stats: DFMetaStats, msg: StatsUpdateMessage): DFMetaStats { +function finalStats(stats: DFMetaStats, msg: StatsUpdateMessage, filled: string[]): DFMetaStats { const status = msg.status ?? "complete"; if (status === "complete") return { status, tier: msg.tier ?? stats.tier, gen: stats.gen }; - return { ...stats, status, ...(msg.reason === undefined ? {} : { reason: msg.reason }) }; + return withComputedColumns( + { ...stats, status, ...(msg.reason === undefined ? {} : { reason: msg.reason }) }, + filled, + ); } /** What the channel needs of a model. */ @@ -162,6 +190,15 @@ export class StatsChannel { // wrote, so a later update cannot overwrite an earlier one's merge. private applying: Promise = Promise.resolve(); + // The columns the replies of the run in progress filled, which its final + // reply puts in df_meta.stats.computed_columns when it leaves the session + // not computed. They are for one gen and for the df_data_dict this channel + // last wrote: a new gen, or a frame that replaced the dict, took what the + // run had merged, so the columns start over. + private filled = new Set(); + private filledGen: number | undefined; + private filledDict: unknown; + constructor(private model: StatsModel) {} /** @@ -215,9 +252,11 @@ export class StatsChannel { // start over from the new one. if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + this.noteFilled(msg.stats_gen, dict, update); } if (msg.final) { - this.replaceStats((stats) => finalStats(stats, msg)); + const filled = this.takeFilled(msg.stats_gen); + this.replaceStats((stats) => finalStats(stats, msg, filled)); } return; } @@ -229,11 +268,35 @@ export class StatsChannel { // new gen; taking `current_gen` without that frame would merge stats // for a state the client has not seen into the one it shows. if (msg.stats_gen !== this.expectedGen) return; + const filled = this.takeFilled(msg.stats_gen); if (msg.reason === "error") { this.replaceStats((stats) => ({ ...stats, status: "error", reason: "stats_failed" })); } else if (msg.reason === "not_requestable") { - this.replaceStats((stats) => ({ ...stats, status: "not_computed" })); + this.replaceStats((stats) => withComputedColumns({ ...stats, status: "not_computed" }, filled)); + } + } + + /** Note the columns `update` filled in the dict this channel has just + * written over `before`. */ + private noteFilled(gen: number, before: unknown, update: DFData): void { + if (this.filledGen !== gen || (this.filledDict !== undefined && this.filledDict !== before)) { + this.filled = new Set(); } + this.filledGen = gen; + for (const column of columnsWithValues(update)) this.filled.add(column); + this.filledDict = this.model.get("df_data_dict"); + } + + /** The columns the run's replies filled, as of the reply that ends it: + * none when the gen or the dict has changed since. The next run starts + * from nothing. */ + private takeFilled(gen: number | undefined): string[] { + const current = this.filledGen === gen && this.filledDict === this.model.get("df_data_dict"); + const columns = current ? Array.from(this.filled) : []; + this.filled = new Set(); + this.filledGen = undefined; + this.filledDict = undefined; + return columns; } /** Replace `df_meta.stats` in a new `df_meta`, the reference that c0a's