diff --git a/packages/buckaroo-js-core/pw-tests/server.spec.ts b/packages/buckaroo-js-core/pw-tests/server.spec.ts index ad9634a05..b4eafb064 100644 --- a/packages/buckaroo-js-core/pw-tests/server.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/server.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; import { loadSession, waitForGrid, getRowCount, getCellText } from './server-helpers'; import { execSync } from 'child_process'; import * as fs from 'fs'; @@ -485,6 +485,427 @@ test.describe('WebSocket data flow', () => { expect(await getCellText(page, COL.name, 0)).toBe('Alice'); }); + test('standalone page advertises stats_update and stats_ondemand on its WebSocket URL', async ({ page, request }) => { + const session = `ws-caps-${Date.now()}`; + await loadSession(request, session, csvPath); + + // The server reads capabilities from ?caps= at open, before the client + // sends anything, so the page has to put them on the URL it connects to. + // stats_ondemand (rows-first c5) says the page can show a session whose + // stats are not computed and send tiered requests. + const socketUrls: string[] = []; + page.on('websocket', (socket) => socketUrls.push(socket.url())); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + const wsUrls = socketUrls.filter((u) => u.includes(`/ws/${session}`)); + expect(wsUrls).toHaveLength(1); + expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update,stats_ondemand'); + }); + + // Rows-first c4: the standalone page's scheduler. No session the server can + // build here defers its stats (that needs a xorq expression), so these + // tests put the pending state on a real session's first frame and answer + // the page's stats_request themselves. Rows, config and the stats payload + // all come from the real server. + async function loadBuckarooSession(request: any, sessionId: string, path: string = csvPath) { + const resp = await request.post(`${BASE}/load`, { + data: { session: sessionId, path, mode: 'buckaroo' }, + }); + expect(resp.ok()).toBe(true); + } + + test('a pending session: the page asks for stats after the first rows, merges the reply and shows them', async ({ page, request }) => { + const session = `ws-sched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const order: string[] = []; + const requests: any[] = []; + let realStats: unknown; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + server.onMessage((message) => { + if (typeof message !== 'string') { + ws.send(message); + return; + } + const msg = JSON.parse(message); + if (msg.type === 'infinite_resp') order.push('infinite_resp'); + if (msg.type === 'initial_state' && realStats === undefined) { + // The first frame as a deferring server sends it: no stats yet. + realStats = msg.df_data_dict.all_stats; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } }; + ws.send(JSON.stringify(msg)); + return; + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + order.push('stats_request'); + requests.push(msg); + ws.send(JSON.stringify({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: msg.scope, tier: 'full', + final: true, payload: realStats, elapsed_ms: 1, + })); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // The stats arrive: the status bar says so and the pinned dtype row has its values. + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 }); + + // One request, for the gen on the first frame, after rows had arrived: an + // incremental step carrying the columns the grid shows (rows-first c4b). This + // harness answers it with a final reply carrying every stat, as a server + // that does not know the field does, and the page takes that as the end. + expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw', incremental: true, columns: ['a', 'b', 'c'] }]); + expect(order.indexOf('infinite_resp')).toBeGreaterThanOrEqual(0); + expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request')); + }); + + test('a session whose stats are not computed: the page asks for nothing until the Compute summary stats button is clicked', async ({ page, request }) => { + const session = `ws-notcomputed-${Date.now()}`; + await loadBuckarooSession(request, session); + + const requests: any[] = []; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + let firstFrame = true; + server.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'initial_state' && firstFrame) { + firstFrame = false; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: { status: 'not_computed', tier: 'schema', gen: 4, reason: 'host' } }; + ws.send(JSON.stringify(msg)); + return; + } + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + requests.push(msg); + return; + } + } + server.send(message); + }); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed'); + // The pinned rows are omitted, not left as placeholders. + await expect(page.locator('.ag-floating-top .ag-row')).toHaveCount(0); + + // Nothing is asked for on its own, however long the page waits. + await page.waitForTimeout(2500); + expect(requests).toEqual([]); + + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + // A server that names no requestable tier can be asked for full; the + // whole-table request names no columns (rows-first c5). + await expect.poll(() => requests).toEqual([ + { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'full' }, + ]); + }); + + // Rows-first c4b: a server on the unit path answers an incremental request + // with a partial stats_update and expects another request. The harness + // answers from the test, with the session's real all_stats as the final + // payload, so no xorq is needed. + async function routeStatsRequests( + page: Page, + session: string, + onRequest: (msg: any, send: (m: object) => void, h: { realStats: unknown; firstFrame: any }) => void, + firstStats: object = { status: 'pending', tier: 'schema', gen: 1 }, + ) { + const h = { requests: [] as any[], realStats: undefined as unknown, firstFrame: undefined as any }; + await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => { + const server = ws.connectToServer(); + server.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'initial_state' && h.realStats === undefined) { + h.realStats = msg.df_data_dict.all_stats; + msg.df_data_dict.all_stats = []; + msg.df_meta = { ...msg.df_meta, stats: firstStats }; + h.firstFrame = msg; + ws.send(JSON.stringify(msg)); + return; + } + } + ws.send(message); + }); + ws.onMessage((message) => { + if (typeof message === 'string') { + const msg = JSON.parse(message); + if (msg.type === 'stats_request') { + h.requests.push(msg); + onRequest(msg, (m) => ws.send(JSON.stringify(m)), h); + return; + } + } + server.send(message); + }); + }); + return h; + } + const partialUpdate = (gen: number, remaining: number) => ({ + type: 'stats_update', stats_gen: gen, scope: 'raw', tier: 'full', final: false, remaining, + payload: { format: 'json', layout: 'wide', data: [{ index: 'dtype', level_0: 'dtype', a: 'object', b: 'int64', c: 'float64' }] }, + elapsed_ms: 1, + }); + + test('partial updates: the page asks again for each one, keeps the stats pending, and completes on the final', async ({ page, request }) => { + const session = `ws-incremental-${Date.now()}`; + await loadBuckarooSession(request, session); + + let releaseFinal!: () => void; + const finalGate = new Promise((resolve) => { releaseFinal = resolve; }); + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length <= 2) { + send(partialUpdate(msg.stats_gen, 3 - harness.requests.length)); + return; + } + void finalGate.then(() => send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + })); + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // Two partial replies, two more requests, and the third waits for its reply. + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(3); + const asked = { type: 'stats_request', stats_gen: 1, scope: 'raw', incremental: true, columns: ['a', 'b', 'c'] }; + expect(h.requests).toEqual([asked, asked, asked]); + // The partial rows are in, and the stats are still pending: no final yet. + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).toHaveText('int64', { timeout: 10_000 }); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); + + releaseFinal(); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); + }); + + test('the columns hint follows the grid viewport: a wide table sends the columns on screen, and the next request the ones scrolled to', async ({ page, request }) => { + const wideCsv = path.join(os.tmpdir(), `buckaroo_e2e_wide_${Date.now()}.csv`); + const names = Array.from({ length: 60 }, (_, i) => `column_${i}`); + fs.writeFileSync(wideCsv, [names.join(','), names.map((_, i) => String(i)).join(','), names.map((_, i) => String(i * 2)).join(',')].join('\n') + '\n'); + try { + const session = `ws-incremental-wide-${Date.now()}`; + await loadBuckarooSession(request, session, wideCsv); + + // Each partial reply waits for the test, so the grid can scroll between requests. + const replies: Array<() => void> = []; + const h = await routeStatsRequests(page, session, (msg, send) => { + replies.push(() => send(partialUpdate(msg.stats_gen, 5))); + }); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + // The data grid's header cells (the status bar is a grid too). + const headerIds = () => page.locator('.df-viewer .ag-header-viewport .ag-header-cell[col-id]').evaluateAll( + (cells) => cells.map((c) => c.getAttribute('col-id') as string)); + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(1); + const first: string[] = h.requests[0].columns; + expect(first).toEqual(expect.any(Array)); + // Only the columns on screen, from the left edge: not all sixty. + expect(first.length).toBeGreaterThan(2); + expect(first.length).toBeLessThan(40); + expect(first[0]).toBe('a'); + // The grid may lay out more columns after the first rows, so the hint is + // the columns it showed at the time: the leading ones. + expect((await headerIds()).filter((id) => id !== 'index').slice(0, first.length)).toEqual(first); + + // Scroll to the right edge, then let the first reply through. The scroll + // event is sent by hand each time: the grid may not have been listening + // for the first one, and setting the same offset again sends none. + await expect.poll(async () => { + await page.locator('.df-viewer .ag-body-horizontal-scroll-viewport').evaluate((el) => { + el.scrollLeft = el.scrollWidth; + el.dispatchEvent(new Event('scroll')); + }); + return (await headerIds()).includes('a'); + }, { timeout: 10_000 }).toBe(false); + const scrolledTo = (await headerIds()).filter((id) => id !== 'index'); + // The grid reports its columns a frame after it renders them: let that pass + // before the next request can be sent. + await page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + replies[0](); + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(2); + const second: string[] = h.requests[1].columns; + expect(second).toEqual(expect.any(Array)); + expect(second).not.toContain('a'); + expect(second).toContain(scrolledTo[scrolledTo.length - 1]); + } finally { + cleanupFile(wideCsv); + } + }); + + test('a gen change in the middle of a run: the old gen is dropped and the page asks again for the new one', async ({ page, request }) => { + const session = `ws-incremental-gen-${Date.now()}`; + await loadBuckarooSession(request, session); + + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length === 1) { + send(partialUpdate(msg.stats_gen, 2)); + } else if (harness.requests.length === 2) { + // The server moved to gen 2 (a state change) with the run for gen 1 + // in flight; its late reply for gen 1 follows the frame. + const frame = { ...harness.firstFrame, df_meta: { ...harness.firstFrame.df_meta, stats: { status: 'pending', tier: 'schema', gen: 2 } } }; + send(frame); + send(partialUpdate(1, 1)); + } else { + send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + }); + } + }); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 15_000 }); + // Two requests for gen 1, then one for gen 2, and none for gen 1 after the change. + expect(h.requests.map((r) => r.stats_gen)).toEqual([1, 1, 2]); + expect(h.requests[2]).toMatchObject({ incremental: true, columns: ['a', 'b', 'c'] }); + }); + + // Rows-first c5: a session the server's policy left without stats. The first + // frame carries the policy fields a capable client is sent, and the harness + // answers the page's requests as a server that serves tiers would. + const policyStats = (over: object = {}) => ({ + status: 'not_computed', tier: 'schema', gen: 4, reason: 'size', tier_target: 'schema', + estimate: { rows: 12_400_000, cols: 3 }, auto_request: false, requestable: ['scalar', 'full'], ...over, + }); + + test('a policy session: the page asks for nothing, the summary view says why, and the control asks for the basic tier', async ({ page, request }) => { + const session = `ws-policy-${Date.now()}`; + await loadBuckarooSession(request, session); + const h = await routeStatsRequests(page, session, () => {}, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed'); + // Nothing is asked for on its own: auto_request is false and no column needs stats. + await page.waitForTimeout(2500); + expect(h.requests).toEqual([]); + + // The summary view has no stats to list: it says why, with the control. + await page.locator('.status-bar select').first().selectOption('summary'); + const empty = page.getByTestId('stats-empty-state'); + await expect(empty).toBeVisible(); + await expect(empty).toContainText('12.4M rows x 3 columns'); + await empty.getByRole('button', { name: 'Compute basic stats' }).click(); + await expect.poll(() => h.requests).toEqual([ + { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }, + ]); + // The page marks the stats pending itself, since the server sends no frame to + // a capable client: the empty state and the control give way to the loading text. + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await expect(page.getByTestId('stats-empty-state')).toHaveCount(0); + await expect(page.getByRole('button', { name: /compute|continue/i })).toHaveCount(0); + }); + + test('a policy session: a request over the ceiling is answered with the ceiling message, and the page offers nothing more', async ({ page, request }) => { + const session = `ws-ceiling-${Date.now()}`; + await loadBuckarooSession(request, session); + const h = await routeStatsRequests(page, session, (msg, send) => { + send({ type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'full', final: true, status: 'not_computed', reason: 'ceiling' }); + }, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + + await expect(page.getByTestId('stats-status')).toContainText('Summary stats unavailable'); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-reason', 'ceiling'); + await expect(page.getByRole('button', { name: 'Compute summary stats' })).toHaveCount(0); + await page.locator('.status-bar select').first().selectOption('summary'); + await expect(page.getByTestId('stats-empty-state')).toContainText('over the size limit'); + await expect(page.getByTestId('stats-empty-state').getByRole('button')).toHaveCount(0); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(1); + }); + + test('a policy session: a forced run is asked for again after each partial reply, stays pending, and ends with the final one', async ({ page, request }) => { + const session = `ws-forced-${Date.now()}`; + await loadBuckarooSession(request, session); + let releaseFinal!: () => void; + const finalGate = new Promise((resolve) => { releaseFinal = resolve; }); + const h = await routeStatsRequests(page, session, (msg, send, harness) => { + if (harness.requests.length < 3) { + send({ ...partialUpdate(msg.stats_gen, 3 - harness.requests.length), tier: 'scalar' }); + return; + } + void finalGate.then(() => send({ + type: 'stats_update', stats_gen: msg.stats_gen, scope: 'raw', tier: 'scalar', final: true, remaining: 0, + payload: harness.realStats, elapsed_ms: 1, + })); + }, policyStats()); + + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await page.getByRole('button', { name: 'Compute summary stats' }).click(); + + // Two partial replies, two more requests, and the third waits for its reply. + await expect.poll(() => h.requests.length, { timeout: 10_000 }).toBe(3); + const forced = { type: 'stats_request', stats_gen: 4, scope: 'raw', incremental: true, force: true, tier: 'scalar' }; + expect(h.requests).toEqual([forced, forced, forced]); + // The partial rows are in and the stats are pending, with no control. + await expect(page.locator('.ag-floating-top [col-id="b"]').first()).toHaveText('int64', { timeout: 10_000 }); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'pending'); + await expect(page.getByRole('button', { name: 'Compute summary stats' })).toHaveCount(0); + + releaseFinal(); + await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 }); + await page.waitForTimeout(500); + expect(h.requests).toHaveLength(3); + }); + + test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => { + const session = `ws-nosched-${Date.now()}`; + await loadBuckarooSession(request, session); + + const sentTypes: string[] = []; + page.on('websocket', (socket) => { + socket.on('framesent', (frame) => { + if (typeof frame.payload === 'string') sentTypes.push(JSON.parse(frame.payload).type); + }); + }); + await page.goto(`${BASE}/s/${session}`); + await waitForGrid(page); + await expect(page.locator('.status-bar')).toBeVisible(); + // Longer than the scheduler would wait for rows that never come. + await page.waitForTimeout(3000); + + expect(sentTypes).toContain('infinite_request'); + expect(sentTypes).not.toContain('stats_request'); + await expect(page.getByTestId('stats-status')).toHaveCount(0); + }); + test('WebSocket receives data for scrolled rows', async ({ page, request }) => { // Create a larger dataset (100 rows) to force infinite scrolling const rows = []; diff --git a/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts new file mode 100644 index 000000000..91c8344e6 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts @@ -0,0 +1,50 @@ +/** + * Playwright test for the client states of the two-message protocol + * (rows-first c0a). + * + * While summary stats have not arrived, `df_meta.stats.status` decides what + * the pinned area shows. The StatsPendingPinnedRows story flips the status and + * supplies the stats on "complete": + * - pending: every valueless pinned key shows a placeholder row with its own + * row id + * - not_computed: valueless pinned keys are omitted + * - complete: the values appear and the color-mapped column restyles + */ +import { test, expect } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-dfviewer-statspendingpinnedrows--primary&globals=&args="; + +test("pinned rows follow df_meta.stats.status: placeholders, omitted, then values and colors", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + const bodyCellA = page.locator('.ag-center-cols-container .ag-row[row-index="0"] [col-id="a"]'); + const backgroundOf = (loc: typeof bodyCellA) => loc.evaluate((el) => getComputedStyle(el).backgroundColor); + + // pending: one placeholder per pinned key, each with its own row id. + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(bodyCellA).toHaveText("1"); + const backgroundBeforeBins = await backgroundOf(bodyCellA); + + // not_computed: the valueless keys are omitted. + await page.getByTestId("status-not_computed").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + + // complete: the values appear, and the color-mapped column restyles now + // that its histogram bins exist. + await page.getByTestId("status-complete").click(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + await expect.poll(() => backgroundOf(bodyCellA), { timeout: 10_000 }).not.toBe(backgroundBeforeBins); + expect(pageErrors).toEqual([]); +}); diff --git a/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts new file mode 100644 index 000000000..bf35699a3 --- /dev/null +++ b/packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts @@ -0,0 +1,259 @@ +/** + * Playwright test for the client states of the stats wire (rows-first c4). + * + * The StatsSchedulerStates story flips df_meta.stats.status. Each status has + * its own text in the status bar's stats column, and the pinned area follows it + * as in c0a. Switching between statuses moves nothing above or beside the grid: + * the status bar keeps its box, and the grid keeps its position and width. The + * grid's height changes only with the pinned area: placeholders hold the height + * of the values that replace them, and omitted rows take theirs away. In + * "not_computed" the status bar offers a Compute summary stats button, which + * sends stats_request {force: true, incremental: true} with the columns the grid + * shows (rows-first c4b). While the stats are pending, a stat row that has + * arrived for some columns leaves the cells of the others empty. + * + * The NotComputed story (rows-first c5) is a session the server's policy left + * without stats. Its main view renders typed columns over the schema-tier + * stats with no blank pinned row, its summary view shows why the stats are + * missing with the control that asks for them (basic before full, with a + * per-column form), and over the ceiling it says so and offers nothing. + */ +import { test, expect, Page } from "@playwright/test"; +import { waitForCells } from "./ag-pw-utils"; + +const STORY_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--primary&globals=&args="; + +const boxes = async (page: Page) => { + const round = (b: { x: number; y: number; width: number; height: number } | null) => + b && { x: Math.round(b.x), y: Math.round(b.y), width: Math.round(b.width), height: Math.round(b.height) }; + return { + statusBar: round(await page.locator(".status-bar").boundingBox()), + grid: round(await page.locator(".df-viewer").boundingBox()), + widget: round(await page.locator(".buckaroo-widget").boundingBox()), + }; +}; + +test("each stats status shows its own text in the status bar and nothing moves between them", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(STORY_URL); + await waitForCells(page); + + const statsCell = page.getByTestId("stats-status"); + const pinnedRows = page.locator(".ag-floating-top .ag-row"); + const distinctPinnedRowIds = async () => { + const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id"))); + return Array.from(new Set(ids)).sort(); + }; + + // pending: the loading text, and a placeholder for each pinned key. + await expect(statsCell).toHaveAttribute("data-stats-status", "pending"); + await expect(statsCell).toContainText("Computing summary stats"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + const reference = await boxes(page); + expect(reference.statusBar).not.toBeNull(); + expect(reference.grid).not.toBeNull(); + // Nothing above or beside the grid moves, whatever the status. + const staysPut = async () => { + const now = await boxes(page); + expect(now.statusBar).toEqual(reference.statusBar); + expect(now.grid).toMatchObject({ x: reference.grid!.x, y: reference.grid!.y, width: reference.grid!.width }); + return now; + }; + + // not_computed: the control, and no pinned rows. + await page.getByTestId("status-not_computed").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + const withoutPinned = await staysPut(); + // The omitted rows take their height with them. + expect(withoutPinned.grid!.height).toBeLessThan(reference.grid!.height); + + // error: the reason, and no pinned rows. + await page.getByTestId("status-error").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "error"); + await expect(statsCell).toContainText("Stats error: stats_failed"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]); + expect((await staysPut()).grid).toEqual(withoutPinned.grid); + + // complete: the values are in the pinned area where the placeholders were, so + // the grid is as tall as it was while pending. + await page.getByTestId("status-complete").click(); + await expect(statsCell).toHaveAttribute("data-stats-status", "complete"); + await expect(statsCell).toContainText("Summary stats ready"); + await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]); + await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64"); + expect(await boxes(page)).toEqual(reference); + + expect(pageErrors).toEqual([]); +}); + +test("the Compute summary stats control sends an incremental stats_request with force and a tier, and no columns hint", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + await page.getByTestId("status-not_computed").click(); + await page.getByRole("button", { name: "Compute summary stats" }).click(); + + // The story's server names no requestable tier, so the one it can be asked for + // is full. The grid shows columns a and b, and the whole-table request does not + // send them: on a forced request `columns` names what it is for (rows-first c5). + const sent = page.getByTestId("sent-log"); + await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); +}); + +test("a stat row that has arrived for one column leaves the other columns' cells empty while the stats are pending", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + await page.getByTestId("status-partial").click(); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByTestId("stats-status")).toContainText("Computing summary stats"); + + const meanCell = (col: string) => page.locator(`.ag-floating-top .ag-row[row-id="main-mean"] [col-id="${col}"]`); + await expect(meanCell("a")).toHaveText("2", { timeout: 10_000 }); + // Column b has no mean yet: the cell is empty. + await expect(meanCell("b")).toHaveText(""); + // dtype came with the schema, for both columns. + await expect(page.locator('.ag-floating-top .ag-row[row-id="main-dtype"] [col-id="b"]')).toHaveText("object"); + + // The stats arrive for column b too: the cell fills in. + await page.getByTestId("status-complete").click(); + await expect(meanCell("b")).toHaveText("N/A", { timeout: 10_000 }); +}); + +test("no control is offered while the stats are pending, in error or complete", async ({ page }) => { + await page.goto(STORY_URL); + await waitForCells(page); + + const control = page.getByRole("button", { name: "Compute summary stats" }); + for (const status of ["pending", "error", "complete"]) { + await page.getByTestId(`status-${status}`).click(); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", status); + await expect(control).toHaveCount(0); + } +}); + +const NOT_COMPUTED_URL = + "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--not-computed&globals=&args="; + +const sentLog = async (page: Page): Promise => JSON.parse((await page.getByTestId("sent-log").textContent()) ?? "[]"); + +test("a not computed session renders typed columns with no blank pinned rows", async ({ page }) => { + const pageErrors: string[] = []; + page.on("pageerror", (e) => pageErrors.push(e.message)); + + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + + // The columns are typed from the schema: integers without decimals, floats + // with three, strings as they are. + const firstRow = (col: string) => page.locator(`.ag-center-cols-container .ag-row[row-index="0"] [col-id="${col}"]`); + await expect(firstRow("a")).toHaveText("1"); + await expect(firstRow("b")).toHaveText("x"); + await expect(firstRow("c")).toHaveText("1.500"); + + // dtype came with the schema. histogram has no value and none is coming, so + // there is one pinned row and no blank one. + const pinnedRows = page.locator(".df-viewer .ag-floating-top .ag-row"); + // A pinned row renders once per column container, with the same id each time. + await expect + .poll(async () => Array.from(new Set(await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id")))))) + .toEqual(["main-dtype"]); + const dtypeCell = (col: string) => page.locator(`.df-viewer .ag-floating-top .ag-row[row-id="main-dtype"] [col-id="${col}"]`); + await expect(dtypeCell("a")).toHaveText("int64"); + await expect(dtypeCell("c")).toHaveText("float64"); + + // The status bar says the stats are not computed and offers the control. + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible(); + expect(pageErrors).toEqual([]); +}); + +test("the summary view shows the empty state, and its control asks for the basic tier, for one column or all", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("view-summary").click(); + + const empty = page.getByTestId("stats-empty-state"); + await expect(empty).toBeVisible(); + await expect(empty).toContainText("not computed"); + await expect(empty).toContainText("12.4M rows x 3 columns"); + // The grid is not shown in its place. + await expect(page.locator(".df-viewer")).toHaveCount(0); + + // The whole table: the smallest tier the server allows, force, no columns. + await empty.getByRole("button", { name: "Compute basic stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + // The stats are marked pending, so the empty state and the control give way to + // the loading text and the grid, and a second click is not possible. + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByTestId("stats-status")).toContainText("Computing summary stats"); + await expect(page.getByTestId("stats-empty-state")).toHaveCount(0); + await expect(page.getByRole("button", { name: /compute|continue/i })).toHaveCount(0); + await expect(page.locator(".df-viewer")).toHaveCount(1); + + // The session as the server described it again: the empty state and its + // per-column form. One column: the same request, naming the column by the + // grid's own name. + await page.getByTestId("policy-size").click(); + await expect(empty).toBeVisible(); + await empty.getByRole("combobox", { name: "Columns to compute" }).selectOption("b"); + await empty.getByRole("button", { name: "Compute basic stats" }).click(); + await expect.poll(async () => (await sentLog(page)).length).toBe(2); + expect((await sentLog(page))[1]).toEqual({ + type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar", columns: ["b"], + }); + + // Back to the main view: its grid is there again. + await page.getByTestId("view-main").click(); + await expect(page.getByTestId("stats-empty-state")).toHaveCount(0); + await waitForCells(page); +}); + +test("the status bar's control asks for the tier too, and the control is gone while the stats are pending", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + + await page.getByRole("button", { name: "Compute summary stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); + await expect(page.getByRole("button", { name: "Compute summary stats" })).toHaveCount(0); +}); + +test("over the ceiling the summary view says so and offers no control, and the status bar offers none either", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("policy-ceiling").click(); + await page.getByTestId("view-summary").click(); + + const empty = page.getByTestId("stats-empty-state"); + await expect(empty).toContainText("over the size limit"); + await expect(page.getByRole("button", { name: /compute|continue/i })).toHaveCount(0); + await expect(page.getByRole("combobox", { name: "Columns to compute" })).toHaveCount(0); + await expect(page.getByTestId("stats-status")).toContainText("Summary stats unavailable"); + expect(await sentLog(page)).toEqual([]); +}); + +test("a paused run offers Continue, which asks for the tier that is left", async ({ page }) => { + await page.goto(NOT_COMPUTED_URL); + await waitForCells(page); + await page.getByTestId("policy-cost").click(); + await page.getByTestId("view-summary").click(); + + await expect(page.getByTestId("stats-empty-state")).toContainText("paused"); + await page.getByTestId("stats-empty-state").getByRole("button", { name: "Continue computing stats" }).click(); + await expect.poll(() => sentLog(page)).toEqual([ + { type: "stats_request", stats_gen: 7, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); + await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "pending"); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx index a117e9816..a92870221 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooInfiniteWidget.flash.test.tsx @@ -7,13 +7,14 @@ * Tests assert CURRENT behavior on main (Option A in docs/rerender-test-plan.md). * Tests tagged "[captures current flash]" are tracking pain, not validating it. */ -import { render } from "@testing-library/react"; +import { render, act, waitFor, screen, fireEvent } from "@testing-library/react"; import { BuckarooInfiniteWidget } from "./BuckarooWidgetInfinite"; import { KeyAwareSmartRowCache } from "./DFViewerParts/SmartRowCache"; import { getSpyCalls, resetSpy, setMockColumnState } from "../test-utils/agGridSpy"; import { BuckarooState, BuckarooOptions, DFMeta } from "./WidgetTypes"; import { DFViewerConfig } from "./DFViewerParts/DFWhole"; import { IDisplayArgs } from "./DFViewerParts/gridUtils"; +import { StatsChannel } from "../server/StatsChannel"; jest.mock("ag-grid-react", () => require("../test-utils/agGridSpy").agGridReactMockFactory(), @@ -21,8 +22,14 @@ jest.mock("ag-grid-react", () => jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); // StatusBar also instantiates AgGridReact; stub it so the spy only counts the data grid. +// The stub records its props so the in-flight tests can read inFlight and drive +// setBuckarooState the way the real status bar does. +let mockStatusBarProps: any = null; jest.mock("./StatusBar", () => ({ - StatusBar: () =>
, + StatusBar: (props: any) => { + mockStatusBarProps = props; + return
; + }, })); // DFViewerInfinite-prop capture for identity-stability assertion. @@ -656,3 +663,455 @@ describe("BuckarooInfiniteWidget — flash matrix (current behavior)", () => { }); }); }); + +// Rows-first c0a: the grid has to behave while summary stats are pending, not +// computed, or arrive after the first paint. df_meta.stats.status drives it; +// absent df_meta.stats means "complete", as servers without the field send. +describe("BuckarooInfiniteWidget — stats not yet available (rows-first c0a)", () => { + const withStats = (status: string): DFMeta => ({ ...baseDfMeta, stats: { status } } as DFMeta); + + const pinnedConfig: DFViewerConfig = { + ...baseConfig, + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "histogram", displayer_args: { displayer: "obj" } }, + ], + }; + const pinnedDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: pinnedConfig, summary_stats_key: "summary_stats" }, + }; + + const widgetProps = (over: Record = {}) => ({ + df_data_dict: { summary_stats: [] as any[] }, + df_display_args: pinnedDisplayArgs, + df_meta: baseDfMeta, + operations: [], + on_operations: jest.fn(), + operation_results: {} as any, + command_config: { argspecs: {}, defaultArgs: {} }, + buckaroo_state: initialState, + on_buckaroo_state: jest.fn(), + buckaroo_options: baseOptions, + src: mkSrc(), + ...over, + }); + + const lastPinnedRows = (): any[] => { + const sets = getSpyCalls().setGridOption.filter(([k]) => k === "pinnedTopRowData"); + return sets[sets.length - 1][1] as any[]; + }; + const pinnedRowIds = (rows: any[]): string[] => { + const { gridOptions, context } = getSpyCalls().lastProps; + return rows.map((data) => gridOptions.getRowId({ data, level: 0, rowPinned: "top", context })); + }; + + describe("pinned rows with no value", () => { + it("pending: shows one placeholder per valueless key, each with its own row id", () => { + render(); + const rows = lastPinnedRows(); + // Each placeholder keeps its key as the row label, so the pinned area + // holds its height and the rows are told apart. + expect(rows.map((r) => r?.index)).toEqual(["dtype", "histogram"]); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: a key that has a value renders it and only the other key gets a placeholder", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + const rows = lastPinnedRows(); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]?.index).toBe("histogram"); + expect(new Set(pinnedRowIds(rows)).size).toBe(2); + }); + + it("pending: placeholders turn into the real rows when stats arrive", () => { + const props = widgetProps({ df_meta: withStats("pending") }); + const { rerender } = render(); + const stats = [ + { index: "dtype", a: "int64" }, + { index: "histogram", a: [{ name: "1-5", population: 100 }] }, + ]; + rerender( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("not_computed: omits the valueless keys", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("not_computed: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("error: omits the valueless keys, since no value is coming", () => { + render(); + expect(lastPinnedRows()).toEqual([]); + }); + + it("error: keeps a key that has a value", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render( + , + ); + expect(lastPinnedRows()).toEqual(stats); + }); + + it("no df_meta.stats (an older server) behaves as complete: valueless keys stay undefined rows", () => { + const stats = [{ index: "dtype", a: "int64" }]; + render(); + const rows = lastPinnedRows(); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual(stats[0]); + expect(rows[1]).toEqual({}); + }); + }); + + describe("color_map columns", () => { + const colorConfig: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + }; + const colorDisplayArgs: Record = { + main: { data_key: "main", df_viewer_config: colorConfig, summary_stats_key: "summary_stats" }, + }; + const colorProps = (stats: any[]) => + widgetProps({ df_display_args: colorDisplayArgs, df_data_dict: { summary_stats: stats } }); + const forcedRefreshes = () => getSpyCalls().refreshCells.filter((p: any) => p.force === true); + + it("restyles the color-mapped columns when histogram bins arrive after the first render", () => { + const props = colorProps([{ index: "dtype", a: "int64", b: "int64" }]); + const { rerender } = render(); + expect(forcedRefreshes()).toHaveLength(0); + + rerender( + , + ); + const refreshes = forcedRefreshes(); + expect(refreshes).toHaveLength(1); + expect(refreshes[0].columns).toEqual(["a"]); + }); + + it("does not refresh when the new stats leave the bins unchanged", () => { + const props = colorProps([{ index: "histogram_bins", a: [1, 2, 3, 4, 5] }]); + const { rerender } = render(); + rerender( + , + ); + expect(forcedRefreshes()).toHaveLength(0); + }); + }); + + describe("compute summary stats control (rows-first c4)", () => { + it("forwards on_compute_stats to the status bar", () => { + const onComputeStats = jest.fn(); + render( + , + ); + expect(mockStatusBarProps.onComputeStats).toBe(onComputeStats); + }); + + it("passes nothing on when the host gave no callback, so the status bar shows no control", () => { + render(); + expect(mockStatusBarProps.onComputeStats).toBeUndefined(); + }); + }); + + describe("in-flight indicator", () => { + const inFlightAttr = () => document.querySelector('[data-testid="status-bar-stub"]')!.getAttribute("data-inflight"); + const dispatchSearch = () => + act(() => { + mockStatusBarProps.setBuckarooState({ ...initialState, quick_command_args: { search: ["x"] } }); + }); + const props = (meta: DFMeta, dict: Record) => widgetProps({ df_meta: meta, df_data_dict: dict }); + + it("a df_data_dict update alone does not clear it when the server reports df_meta.stats", () => { + const meta = withStats("pending"); + const first = props(meta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + // A stats-only update: a new df_data_dict, the same df_meta. + rerender(); + expect(inFlightAttr()).toBe("true"); + }); + + it("clears when a frame with a new df_meta and df_data_dict arrives", () => { + const first = props(withStats("pending"), { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + + rerender( + , + ); + expect(inFlightAttr()).toBe("false"); + }); + + it("without df_meta.stats a new df_data_dict still clears it, as before", () => { + const first = props(baseDfMeta, { summary_stats: [] }); + const { rerender } = render(); + dispatchSearch(); + expect(inFlightAttr()).toBe("true"); + + rerender(); + expect(inFlightAttr()).toBe("false"); + }); + }); +}); + +// Rows-first c4b: the widget hands the grid the callback that reports which +// columns it shows, which the stats scheduler sends as its hint. +describe("BuckarooInfiniteWidget on_visible_columns (rows-first c4b)", () => { + it("passes the callback to the grid, which reports the data columns of its viewport through it", async () => { + const onVisible = jest.fn(); + const { container } = render( + , + ); + + // The grid is mocked, so give it the header cell AG Grid would render. + container.querySelector(".theme-hanger")!.insertAdjacentHTML( + "beforeend", + '
', + ); + const gridProps = getSpyCalls().lastProps; + expect(typeof gridProps.onVirtualColumnsChanged).toBe("function"); + gridProps.onVirtualColumnsChanged({}); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith(["a"])); + }); +}); + +// Rows-first c5: a session whose stats are not computed has nothing for the +// summary view to list. It shows why, with the control that asks for the stats, +// in place of the grid; the main view keeps its grid. +describe("BuckarooInfiniteWidget summary view, stats not computed (rows-first c5)", () => { + const notComputed = (over: Record = {}): DFMeta => + ({ + ...baseDfMeta, + stats: { + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 2 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, + }, + }) as DFMeta; + + const widgetProps = (over: Record = {}) => ({ + df_data_dict: { summary_stats: [] as any[] }, + df_display_args: baseDisplayArgs, + df_meta: notComputed(), + operations: [], + on_operations: jest.fn(), + operation_results: {} as any, + command_config: { argspecs: {}, defaultArgs: {} }, + buckaroo_state: { ...initialState, df_display: "summary" }, + on_buckaroo_state: jest.fn(), + buckaroo_options: baseOptions, + src: mkSrc(), + ...over, + }); + + it("shows the empty state with the compute control in place of the grid", () => { + const onComputeStats = jest.fn(); + render(); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("not computed"); + expect(getSpyCalls().mountCount).toBe(0); + + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("offers a per-column form over the columns the view has, naming the column by the grid's own name", () => { + const onComputeStats = jest.fn(); + render(); + const picker = screen.getByRole("combobox", { name: "Columns to compute" }); + fireEvent.change(picker, { target: { value: "a" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith({ columns: ["a"] }); + }); + + it("ceiling: the message and no control", () => { + render( + , + ); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("over the size limit"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(0); + }); + + it("without a callback from the host it is the message alone", () => { + render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("the main view keeps its grid and shows no empty state", () => { + render( + , + ); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it.each(["pending", "error", "complete"])("a %s session keeps the summary view's grid", (status) => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it("a session that reports no df_meta.stats keeps the summary view's grid, as before", () => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + it("gives the grid back when the stats arrive", () => { + const props = widgetProps(); + const { rerender } = render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + + rerender( + , + ); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + }); + + // A run for some columns (the per-column form, or the demand columns) ends + // with the session still not computed, and the stats it merged are in + // all_stats. The summary view lists them instead of the empty state. + describe("after a run that computed some columns", () => { + const ranArgs: Record = { + ...baseDisplayArgs, + summary: { + data_key: "empty", + df_viewer_config: { + ...baseConfig, + pinned_rows: [{ primary_key_val: "min", displayer_args: { displayer: "obj" } }], + }, + summary_stats_key: "all_stats", + }, + }; + const ranProps = (meta: DFMeta, allStats: any[]) => + widgetProps({ df_display_args: ranArgs, df_meta: meta, df_data_dict: { empty: [], all_stats: allStats } }); + const lastPinnedRows = (): any[] => { + const sets = getSpyCalls().setGridOption.filter(([k]) => k === "pinnedTopRowData"); + return sets[sets.length - 1][1] as any[]; + }; + const minRows = [ + { index: "dtype", a: "int64" }, + { index: "min", a: 1 }, + ]; + + it("shows the grid, with the stats the run merged, in place of the empty state", () => { + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + expect(lastPinnedRows()).toEqual([minRows[1]]); + }); + + it("an empty list of columns keeps the empty state", () => { + render(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(0); + }); + + it("gives the empty state back when a new gen's frame drops the columns", () => { + const props = ranProps(notComputed({ computed_columns: ["a"] }), minRows); + const { rerender } = render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + + rerender(); + expect(screen.getByTestId("stats-empty-state")).toBeInTheDocument(); + }); + + it("shows the grid for the session a scoped reply leaves, as StatsChannel records it", async () => { + const state: Record = { + df_meta: notComputed(), + df_data_dict: { empty: [], all_stats: [minRows[0]] }, + }; + const channel = new StatsChannel({ get: (key) => state[key], set: (key, value) => { state[key] = value; } }); + const before = state.df_meta; + const reply = { + type: "stats_update", + stats_gen: 1, + scope: "raw", + tier: "scalar", + final: true, + status: "not_computed", + payload: { format: "json", layout: "wide", data: [{ index: "min", level_0: "min", a: 1 }] }, + }; + channel.handle(reply); + await waitFor(() => expect(state.df_meta).not.toBe(before)); + expect(state.df_meta.stats.status).toBe("not_computed"); + + render(); + expect(screen.queryByTestId("stats-empty-state")).not.toBeInTheDocument(); + expect(getSpyCalls().mountCount).toBe(1); + expect(lastPinnedRows().map((r) => r.index)).toEqual(["min"]); + expect(lastPinnedRows()[0].a).toBe(1); + }); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx index d126a87c7..d47ffdff4 100644 --- a/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/BuckarooWidgetInfinite.tsx @@ -5,13 +5,15 @@ import { ColumnsEditor } from "./ColumnsEditor"; import { DFData } from "./DFViewerParts/DFWhole"; import { StatusBar } from "./StatusBar"; +import { StatsColumnOption, StatsEmptyState } from "./StatsEmptyState"; import { BuckarooState } from "./WidgetTypes"; import { BuckarooOptions } from "./WidgetTypes"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, getStatsStatus } from "./WidgetTypes"; import { CommandConfigT } from "./CommandUtils"; import { Operation } from "./OperationUtils"; import { getDs, + getFieldVal, IDisplayArgs, } from "./DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "./DFViewerParts/displayArgsUtils"; @@ -134,6 +136,8 @@ export function BuckarooInfiniteWidget({ df_data_dict, df_display_args, df_meta, + on_compute_stats, + on_visible_columns, operations, on_operations, operation_results, @@ -148,6 +152,11 @@ export function BuckarooInfiniteWidget({ df_meta: DFMeta; df_data_dict: Record; df_display_args: Record; + /** Sends a forced stats_request. Server entry points pass it; the Jupyter + * widget does not, and then the status bar offers no control. */ + on_compute_stats?: (opts?: { columns?: string[] }) => void; + /** Called with the data columns the grid shows, now and whenever they change. */ + on_visible_columns?: (columns: string[]) => void; operations: Operation[]; on_operations: (ops: Operation[]) => void; operation_results: OperationResult; @@ -222,6 +231,24 @@ export function BuckarooInfiniteWidget({ [cDisp, df_data_dict, mainDs, df_meta.total_rows], ); + // A view that lists stats (the summary view, whose rows are not the + // data's) has nothing to show while the stats are not computed. It shows + // why instead, with the control that asks for them. A run for some + // columns leaves the session not computed with those columns' stats in + // all_stats (computed_columns names them), and the grid lists those. + const statsEmptyState = + df_meta.stats?.status === "not_computed" && + cDisp.data_key !== "main" && + (df_meta.stats.computed_columns?.length ?? 0) === 0; + const statsColumns = useMemo( + () => + (cDisp.df_viewer_config?.column_config ?? []).map((cc) => ({ + field: getFieldVal(cc), + label: "header_name" in cc ? String(cc.header_name) : cc.col_path.join(" / "), + })), + [cDisp], + ); + // Operations minus filter-like ops (search, OnlyOutliers — Python // marks them with meta.quick_command=true). These don't change row // identity; quick_command_args already carries the same info, so @@ -290,13 +317,23 @@ export function BuckarooInfiniteWidget({ // so we don't accidentally clear on the same dict the user was looking // at when they fired the change. Cleared on the *next* dict reference. const inFlightAtDictRef = useRef(null); + // The df_meta at that moment. A server that reports df_meta.stats can + // send df_data_dict on its own (a stats update for an earlier state), + // and df_meta only comes with a full frame, so for those servers a + // new df_meta is also needed before the answer to the change counts + // as arrived. Servers without df_meta.stats keep the dict-only rule. + const inFlightAtMetaRef = useRef(null); useEffect(() => { if (inFlight && inFlightAtDictRef.current !== null && df_data_dict !== inFlightAtDictRef.current) { + if (df_meta === inFlightAtMetaRef.current && df_meta.stats !== undefined) { + return; + } setInFlight(false); inFlightAtDictRef.current = null; + inFlightAtMetaRef.current = null; } - }, [df_data_dict, inFlight]); + }, [df_data_dict, df_meta, inFlight]); const wrappedOnBuckarooState = useCallback< React.Dispatch> @@ -311,10 +348,11 @@ export function BuckarooInfiniteWidget({ !== JSON.stringify(buckaroo_state.quick_command_args); if (dataflowChanged) { inFlightAtDictRef.current = df_data_dict; + inFlightAtMetaRef.current = df_meta; setInFlight(true); } on_buckaroo_state(next); - }, [buckaroo_state, df_data_dict, on_buckaroo_state]); + }, [buckaroo_state, df_data_dict, df_meta, on_buckaroo_state]); return (
| undefined} + onComputeStats={on_compute_stats} /> - + {statsEmptyState && df_meta.stats !== undefined ? ( + + ) : ( + + )}
{buckaroo_state.show_commands ? (
diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx index 6e77c3cb8..c69836244 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.test.tsx @@ -1,9 +1,13 @@ -import { render } from "@testing-library/react"; +import { render, waitFor } from "@testing-library/react"; import { DFViewerInfinite } from "./DFViewerInfinite"; import { DFViewerConfig } from "./DFWhole"; const setGridOptionMock = jest.fn(); let latestAgGridProps: any = null; +// The ids of the columns the mocked grid has in its viewport, rendered as the +// header cells AG Grid renders for them (jest.mock factories may read +// variables whose names start with "mock"). +let mockVirtualColumnIds: string[] = []; jest.mock("ag-grid-react", () => { const React = require("react"); @@ -24,7 +28,15 @@ jest.mock("ag-grid-react", () => { }); }, [props]); - return
; + return ( +
+
+ {mockVirtualColumnIds.map((id) => ( +
+ ))} +
+
+ ); }), }; }); @@ -47,6 +59,7 @@ describe("DFViewerInfinite", () => { beforeEach(() => { setGridOptionMock.mockClear(); latestAgGridProps = null; + mockVirtualColumnIds = []; }); it("renders error_info and custom class name", () => { @@ -131,3 +144,69 @@ describe("DFViewerInfinite", () => { expect(latestAgGridProps.datasource.rowCount).toBe(50); }); }); + +// Rows-first c4b: the stats scheduler asks the server for the units covering +// the columns the user is looking at first, so the grid reports them. +describe("DFViewerInfinite on_visible_columns (rows-first c4b)", () => { + const wideConfig: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + { col_name: "c", header_name: "c", displayer_args: { displayer: "obj" } }, + { col_name: "d", header_name: "d", displayer_args: { displayer: "obj" } }, + ], + }; + const viewer = (onVisible?: (columns: string[]) => void) => ( + + ); + // The grid renders its header cells a frame after the event that moves them. + const aFrame = () => new Promise((resolve) => requestAnimationFrame(resolve)); + + beforeEach(() => { + mockVirtualColumnIds = []; + }); + + it("reports the data columns in the grid's viewport once the grid is ready, not the index column", async () => { + mockVirtualColumnIds = ["index", "a", "b"]; + const onVisible = jest.fn(); + render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith(["a", "b"])); + await aFrame(); + expect(onVisible).toHaveBeenCalledTimes(1); + }); + + it("reports again when the viewport's columns change, and not when they stay the same", async () => { + mockVirtualColumnIds = ["index", "a", "b"]; + const onVisible = jest.fn(); + const { rerender } = render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledTimes(1)); + expect(typeof latestAgGridProps.onVirtualColumnsChanged).toBe("function"); + + // The same columns again, as a resize can produce. + latestAgGridProps.onVirtualColumnsChanged({}); + await aFrame(); + expect(onVisible).toHaveBeenCalledTimes(1); + + // The user scrolls right: the grid swaps its header cells and says so. + mockVirtualColumnIds = ["index", "c", "d"]; + rerender(viewer(onVisible)); + latestAgGridProps.onVirtualColumnsChanged({}); + await waitFor(() => expect(onVisible).toHaveBeenCalledTimes(2)); + expect(onVisible).toHaveBeenLastCalledWith(["c", "d"]); + }); + + it("reports an empty list when no data column is in the viewport (a grid that is not laid out yet)", async () => { + mockVirtualColumnIds = ["index"]; + const onVisible = jest.fn(); + render(viewer(onVisible)); + await waitFor(() => expect(onVisible).toHaveBeenCalledWith([])); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx index 7c8fa079e..8ad80e038 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/DFViewerInfinite.tsx @@ -4,10 +4,12 @@ import { useEffect, useRef, } from "react"; +import type { RefObject } from "react"; import * as _ from "lodash-es"; import { DFData, DFDataRow, DFViewerConfig, SDFT } from "./DFWhole"; -import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT } from "./gridUtils"; +import { getCellRendererSelector, dfToAgrid, extractPinnedRows, extractSDFT, getFieldVal } from "./gridUtils"; +import type { StatsStatus } from "../WidgetTypes"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { @@ -21,6 +23,7 @@ import { CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -45,6 +48,9 @@ ModuleRegistry.registerModules([ CellStyleModule, ColumnAutoSizeModule, PinnedRowModule, + // api.refreshCells lives here. Without it the call logs AG Grid error 200 + // and does nothing. + RenderApiModule, RowSelectionModule, TooltipModule, TextFilterModule, @@ -150,6 +156,8 @@ export function DFViewerInfinite({ max_rows_in_configs, view_name, data_key, + stats_status, + on_visible_columns, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -173,6 +181,12 @@ export function DFViewerInfinite({ // a rowId, even though their `index` values overlap (row 0 in main is a // different record than row 0 in summary). data_key?: string; + // df_meta.stats.status. While "pending" a pinned key with no value shows a + // placeholder row; when "not_computed" it is omitted. Undefined behaves + // as "complete". + stats_status?: StatsStatus; + // Called with the data columns in the grid's viewport, now and whenever they change. + on_visible_columns?: (columns: string[]) => void; }) { /* The idea is to do some pre-setup here for @@ -181,6 +195,8 @@ export function DFViewerInfinite({ //console.log("137renderStartTime"); return Date.now(); } , []); + // The element that holds the grid, where the columns in its viewport are read. + const hangerRef = useRef(null); const totalRows=5; const compConfig = df_viewer_config?.component_config; @@ -219,6 +235,7 @@ export function DFViewerInfinite({
{error_info ? 
{error_info}
: null}
) @@ -250,6 +270,9 @@ export function DFViewerInfiniteInner({ effectiveScheme, view_name, data_key, + stats_status, + on_visible_columns, + grid_container, }: { data_wrapper: DatasourceOrRaw; df_viewer_config: DFViewerConfig; @@ -266,6 +289,9 @@ export function DFViewerInfiniteInner({ effectiveScheme?: 'light' | 'dark'; view_name?: string; data_key?: string; + stats_status?: StatsStatus; + on_visible_columns?: (columns: string[]) => void; + grid_container?: RefObject; }) { /* const lastProps = useRef(null); @@ -344,8 +370,8 @@ export function DFViewerInfiniteInner({ // Always re-extract; upstream may mutate summary in-place without changing identity // Memoize to ensure it updates when summary_stats_data changes const topRowData = useMemo( - () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : []) as DFDataRow[], - [summary_stats_data, pinned_rows] + () => extractPinnedRows(summary_stats_data, pinned_rows ? pinned_rows : [], stats_status) as DFDataRow[], + [summary_stats_data, pinned_rows, stats_status] ); // Pinned rows are extracted and ready @@ -417,6 +443,33 @@ export function DFViewerInfiniteInner({ [data_wrapper, gridOptions, hs]); // Use grid API to set pinned rows imperatively, avoiding a full React prop update that can flash const gridRef = useRef | null>(null); + // Report the data columns in the grid's viewport (the index column is not + // one) when the grid is ready and whenever they change. They are read + // from the header cells the grid has rendered, since the grid's own + // column API (getAllDisplayedVirtualColumns) is not registered here and + // registering it would also switch on the column-state calls above. The + // cells follow the event by a frame, so the read waits for one. An + // unchanged list is not reported again, so a resize or a vertical + // scroll costs nothing. + const dataFields = useMemo( + () => new Set(df_viewer_config.column_config.map(getFieldVal)), + [df_viewer_config.column_config], + ); + const lastVisibleRef = useRef(undefined); + const reportVisibleColumns = useCallback(() => { + if (on_visible_columns === undefined) return; + window.requestAnimationFrame(() => { + const container = grid_container?.current; + if (!container) return; + const ids = Array.from(container.querySelectorAll(".ag-header-viewport .ag-header-cell[col-id]")) + .map((cell) => cell.getAttribute("col-id") ?? "") + .filter((id) => dataFields.has(id)); + const sig = JSON.stringify(ids); + if (sig === lastVisibleRef.current) return; + lastVisibleRef.current = sig; + on_visible_columns(ids); + }); + }, [on_visible_columns, dataFields, grid_container]); // Keep latest pinned rows in a ref so onGridReady can apply them once API is ready const topRowsRef = useRef(null); // Build a content signature based on visible fields and pinned values, @@ -442,6 +495,32 @@ export function DFViewerInfiniteInner({ // ignore until grid ready } }, [pinnedSig]); + + // color_map reads histogram_bins from the grid context when a cell is + // painted, so cells that rendered before the bins arrived keep the + // neutral style. Repaint the color-mapped columns when their bins + // change after the first render. Bins that are already there at mount + // paint correctly, so the first run only records the signature. + const colorMapCols = useMemo( + () => df_viewer_config.column_config.flatMap((cc) => + cc.color_map_config?.color_rule === "color_map" + ? [{ field: getFieldVal(cc), statsCol: cc.color_map_config.val_column }] + : []), + [df_viewer_config.column_config], + ); + const colorMapSig = useMemo(() => { + if (colorMapCols.length === 0) return ""; + const stats = extractSDFT(summary_stats_data); + return JSON.stringify(colorMapCols.map( + ({ field, statsCol }) => [field, statsCol === undefined ? undefined : stats[statsCol]?.histogram_bins])); + }, [colorMapCols, summary_stats_data]); + const colorMapSigRef = useRef(colorMapSig); + useEffect(() => { + if (colorMapSigRef.current === colorMapSig) return; + colorMapSigRef.current = colorMapSig; + if (colorMapCols.length === 0) return; + gridRef.current?.api?.refreshCells({ force: true, columns: colorMapCols.map((c) => c.field) }); + }, [colorMapSig, colorMapCols]); // Force update rowData when Raw data changes const rawDataSig = useMemo(() => { @@ -574,7 +653,9 @@ export function DFViewerInfiniteInner({ // Ensure pinned rows are applied once API is ready params.api.setGridOption('pinnedTopRowData', topRowsRef.current || []); } catch (_e) {} + reportVisibleColumns(); }} + onVirtualColumnsChanged={reportVisibleColumns} context={{ outside_df_params, ...extra_context }} > ); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx index 907fef642..bfbfad3aa 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/SeriesSummaryTooltip.tsx @@ -24,11 +24,16 @@ export const getSimpleTooltip = (tooltipField:string) => { // This should be possible with the tooltipValueGetter, but that // wasn't working for some reason - if (props.data.index === "histogram") { + if (props.data?.index === "histogram") { return; } - const val = props.data[tooltipField].toString() - return
{val}
; + // A pinned row whose stats have not arrived has no value for the + // column, and a row that has not loaded has no data: show nothing. + const raw = props.data?.[tooltipField]; + if (raw === undefined || raw === null) { + return; + } + return
{raw.toString()}
; }; return simpleTooltip; } diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx index ef83da6ae..fcdd0baff 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/Styler.tsx @@ -64,13 +64,14 @@ export function colorMap(cmr: ColorMapRules) { const summarys = params.context?.histogram_stats; const statsCol = cmr.val_column; // || col_name; + // No bins is the normal state until the summary stats arrive, so + // fall back to the neutral style quietly; the grid repaints the + // column when they do. if (statsCol === undefined || summarys === undefined){ - console.log("66 couldn't find stats_col") return baseReturn; } const summary_stats_cell = summarys[statsCol]; if (summary_stats_cell === undefined || summary_stats_cell.histogram_bins === undefined ) { - console.log("69 couldn't find summary_stats"); return baseReturn } const histogram_edges = summary_stats_cell.histogram_bins; diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts index 01697ba2e..a32f4a7db 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.test.ts @@ -15,7 +15,8 @@ import { import * as _ from "lodash-es"; import { DFData, DFViewerConfig, NormalColumnConfig, MultiIndexColumnConfig, PinnedRowConfig, ColumnConfig, FormatterArgs } from "./DFWhole"; import { getFormatter, getFloatFormatter, getCompactNumberFormatter, formatDuration, formatIsoDuration, getDurationFormatter } from './Displayer'; -import { ColDef, ICellRendererParams, ValueFormatterParams } from 'ag-grid-community'; +import { CellClassParams, ColDef, ICellRendererParams, ITooltipParams, ValueFormatterParams } from 'ag-grid-community'; +import { getSimpleTooltip } from './SeriesSummaryTooltip'; describe("testing utility functions in gridUtils ", () => { // mostly sanity checks to help develop gridUtils @@ -199,6 +200,18 @@ describe("testing utility functions in gridUtils ", () => { ]); }); + it("omits a required key with no value when the stats status is error, and keeps one that has a value", () => { + // An error is final for the state on screen, so no value is coming for + // the keys that are missing (rows-first c4). + const data: DFData = [{ index: "row1", value: 1 }]; + const pinnedConfig: PinnedRowConfig[] = [ + { primary_key_val: "row1", displayer_args: { displayer: "obj" } }, + { primary_key_val: "missing", displayer_args: { displayer: "obj" } } + ]; + expect(extractPinnedRows(data, pinnedConfig, "error")).toStrictEqual([{ index: "row1", value: 1 }]); + expect(extractPinnedRows([], pinnedConfig, "error")).toStrictEqual([]); + }); + it("includes an optional `?`-prefixed pinned row when the unprefixed key exists in data", () => { const data: DFData = [ { index: "histogram_bins", value: 1 }, @@ -732,6 +745,70 @@ describe("testing multi index organiztion ", () => { expect(children.length).toBe(2); }); +}); + +// Rows-first c0a: while summary stats are pending or not computed, pinned +// cells have no value, and color_map has no histogram bins to read. +describe("pinned cells without stats values (rows-first c0a)", () => { + it("the simple tooltip returns nothing for a valueless pinned cell instead of throwing", () => { + const tooltip = getSimpleTooltip("a"); + // A pinned row with no value for the column: only the row label exists. + expect(() => tooltip({ data: { index: "dtype" } } as ITooltipParams)).not.toThrow(); + expect(tooltip({ data: { index: "dtype" } } as ITooltipParams)).toBeUndefined(); + // A null cell and a row that has not loaded (data undefined) are also valueless. + expect(() => tooltip({ data: { index: "mean", a: null } } as ITooltipParams)).not.toThrow(); + expect(() => tooltip({ data: undefined } as unknown as ITooltipParams)).not.toThrow(); + }); + + it("the simple tooltip still renders a cell that has a value", () => { + const tooltip = getSimpleTooltip("a"); + const el = tooltip({ data: { index: 0, a: 5 } } as ITooltipParams); + expect(el).toBeDefined(); + expect((el as any).props.children).toBe("5"); + }); + + describe("color_map without histogram bins", () => { + const config: DFViewerConfig = { + pinned_rows: [], + left_col_configs: [], + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + }, + ], + }; + const cellStyleFor = (context: any, value: any = 3) => { + const colDef = dfToAgrid(config)[0] as ColDef; + const cellStyle = colDef.cellStyle as (p: CellClassParams) => Record; + return cellStyle({ context, data: { index: 0, a: value }, value, node: { rowPinned: undefined } } as unknown as CellClassParams); + }; + + let logSpy: jest.SpyInstance; + beforeEach(() => { logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); }); + afterEach(() => { logSpy.mockRestore(); }); - + it("returns the neutral style without logging when the stats column has no entry", () => { + expect(cellStyleFor({ histogram_stats: {} })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the entry has no histogram_bins", () => { + expect(cellStyleFor({ histogram_stats: { a: { histogram_log_bins: [1, 2] } } })).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("returns the neutral style without logging when the context carries no histogram_stats", () => { + expect(cellStyleFor({})).toEqual({ backgroundColor: "inherit" }); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it("colors the cell once bins exist", () => { + const style = cellStyleFor({ histogram_stats: { a: { histogram_bins: [1, 2, 3, 4, 5] } } }); + expect(style.backgroundColor).not.toBe("inherit"); + expect(style.backgroundColor).toBeDefined(); + }); + }); }); diff --git a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts index b963123e4..8d765f84e 100644 --- a/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts +++ b/packages/buckaroo-js-core/src/components/DFViewerParts/gridUtils.ts @@ -38,6 +38,7 @@ import { getFormatterFromArgs, getCellRenderer, objFormatter, getFormatter } fro import { CSSProperties, Dispatch, SetStateAction } from "react"; import { CommandConfigT } from "../CommandUtils"; import { KeyAwareSmartRowCache, PayloadArgs } from "./SmartRowCache"; +import type { StatsStatus } from "../WidgetTypes"; // for now colDef stuff with less than 3 implementantions should stay in this file @@ -97,13 +98,33 @@ export function stripOptionalPinnedKey(key: string): string { return isOptionalPinnedKey(key) ? key.slice(1) : key; } -export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[]) { +// Marks a pinned row that stands in for stats that have not arrived. The row +// carries its key as `index`, so it keeps its label and gets a row id of its +// own, and the cell renderer selector leaves its value cells empty. +export const PENDING_STAT_ROW_KEY = "__stat_pending"; + +const PendingStatCell = () => null; +const pendingStatRenderer: CellRendererSelectorResult = { component: PendingStatCell }; + +// `statsStatus` is df_meta.stats.status. A required key with no value is, by +// status: +// "pending" a placeholder row, which holds the pinned area's height +// "not_computed", "error" omitted, since no value is coming +// anything else undefined, as it was before df_meta.stats existed +export function extractPinnedRows(sdf: DFData, prc: PinnedRowConfig[], statsStatus?: StatsStatus) { const result: (DFData[number] | undefined)[] = []; for (const cfg of prc) { const raw = cfg.primary_key_val; - const found = _.find(sdf, { index: stripOptionalPinnedKey(raw) }); - if (found === undefined && isOptionalPinnedKey(raw)) { - continue; + const key = stripOptionalPinnedKey(raw); + const found = _.find(sdf, { index: key }); + if (found === undefined) { + if (isOptionalPinnedKey(raw) || statsStatus === "not_computed" || statsStatus === "error") { + continue; + } + if (statsStatus === "pending") { + result.push({ index: key, [PENDING_STAT_ROW_KEY]: true }); + continue; + } } result.push(found); } @@ -340,6 +361,9 @@ export function getCellRendererSelector(pinned_rows: PinnedRowConfig[], column_c if (pk === undefined) { return anyRenderer; // default renderer } + if (_.get(params.node.data, PENDING_STAT_ROW_KEY) === true && params.column?.getColId() !== "index") { + return pendingStatRenderer; // a stat that has not arrived: leave the cell empty + } const maybePrc: PinnedRowConfig | undefined = _.find( pinned_rows, (cfg) => stripOptionalPinnedKey(cfg.primary_key_val) === pk, diff --git a/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx new file mode 100644 index 000000000..c7113736e --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatsEmptyState.test.tsx @@ -0,0 +1,161 @@ +/** + * StatsEmptyState — the summary view of a session whose stats are not computed + * (rows-first c5). + * + * The summary view lists stats, and there are none to list, so it shows why + * and offers the control that asks for them: one button for the smallest tier + * the server allows (basic before full), with a per-column form. When the + * server's ceiling refuses the stats the view says so and offers nothing. + */ +import "@testing-library/jest-dom"; +import { render, screen, fireEvent, within } from "@testing-library/react"; + +jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); + +import { StatsEmptyState, StatsColumnOption, formatStatsEstimate } from "./StatsEmptyState"; +import { DFMetaStats } from "./WidgetTypes"; + +const policy = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, +}); + +const columns: StatsColumnOption[] = [ + { field: "a", label: "name" }, + { field: "b", label: "age" }, +]; + +const renderState = (stats: DFMetaStats, onComputeStats?: jest.Mock, cols: StatsColumnOption[] = columns) => + render(); + +describe("StatsEmptyState", () => { + it("size: says the stats were not computed because the table is large, and how large", () => { + renderState(policy(), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveAttribute("data-stats-reason", "size"); + expect(root).toHaveTextContent("not computed"); + expect(root).toHaveTextContent("12.4M rows x 44 columns"); + }); + + it("offers one button for the smallest tier the server allows, and it calls the handler with no arguments", () => { + const onComputeStats = jest.fn(); + renderState(policy(), onComputeStats); + expect(screen.getAllByRole("button")).toHaveLength(1); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("names full when it is the only tier left to ask for", () => { + renderState(policy({ requestable: ["full"] }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Compute basic stats" })).not.toBeInTheDocument(); + }); + + it("names full once the basic tier has been reached", () => { + renderState(policy({ tier: "scalar" }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + }); + + it("has a per-column form: the button asks for the column picked, and all columns by default", () => { + const onComputeStats = jest.fn(); + renderState(policy(), onComputeStats); + const picker = screen.getByRole("combobox", { name: "Columns to compute" }); + expect(within(picker).getAllByRole("option").map((o) => o.textContent)).toEqual(["All columns", "name", "age"]); + + fireEvent.change(picker, { target: { value: "b" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith({ columns: ["b"] }); + + fireEvent.change(picker, { target: { value: "" } }); + fireEvent.click(screen.getByRole("button", { name: "Compute basic stats" })); + expect(onComputeStats).toHaveBeenLastCalledWith(); + }); + + it("has no per-column picker when it knows no columns", () => { + renderState(policy(), jest.fn(), []); + expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Compute basic stats" })).toBeInTheDocument(); + }); + + it("ceiling: says the table is over the size limit, and offers no control even with a handler", () => { + renderState(policy({ reason: "ceiling", requestable: [] }), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveAttribute("data-stats-reason", "ceiling"); + expect(root).toHaveTextContent("over the size limit"); + expect(root).toHaveTextContent("12.4M rows x 44 columns"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox")).not.toBeInTheDocument(); + }); + + it("a size the server chose with nothing left above it reads as over the limit, with no control", () => { + renderState(policy({ requestable: [] }), jest.fn()); + const root = screen.getByTestId("stats-empty-state"); + expect(root).toHaveTextContent("over the size limit"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("host with nothing requestable keeps its own message, with no control", () => { + renderState(policy({ reason: "host", requestable: [] }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("turned off"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("ceiling: offers no control whatever requestable lists", () => { + renderState(policy({ reason: "ceiling", requestable: ["scalar", "full"] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("host: says the stats were turned off, and offers the control", () => { + renderState(policy({ reason: "host" }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("turned off"); + expect(screen.getByRole("button", { name: "Compute basic stats" })).toBeInTheDocument(); + }); + + it("cost: says the run was paused, and the button reads Continue", () => { + const onComputeStats = jest.fn(); + renderState(policy({ reason: "cost" }), onComputeStats); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("paused"); + fireEvent.click(screen.getByRole("button", { name: "Continue computing stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("with no handler it is the message alone", () => { + renderState(policy()); + expect(screen.getByTestId("stats-empty-state")).toHaveTextContent("not computed"); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("leaves the size out when the server sent no estimate", () => { + renderState(policy({ estimate: undefined }), jest.fn()); + expect(screen.getByTestId("stats-empty-state")).not.toHaveTextContent("rows"); + }); + + it("an older server's not_computed (no policy fields) gets the message and a full-stats button", () => { + renderState({ status: "not_computed", tier: "schema", gen: 1, reason: "host" }, jest.fn()); + expect(screen.getByRole("button", { name: "Compute full stats" })).toBeInTheDocument(); + }); +}); + +describe("formatStatsEstimate", () => { + it.each([ + [{ rows: 12_400_000, cols: 44 }, "12.4M rows x 44 columns"], + [{ rows: 78_000_000 }, "78M rows"], + [{ rows: 1_000_000 }, "1M rows"], + [{ rows: 999_999 }, "999,999 rows"], + [{ rows: 1 }, "1 row"], + [{ rows: 3, cols: 1 }, "3 rows x 1 column"], + [{ cols: 44 }, "44 columns"], + [{}, ""], + [undefined, ""], + ])("%j reads %j", (estimate, expected) => { + expect(formatStatsEstimate(estimate)).toBe(expected); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx b/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx new file mode 100644 index 000000000..652e797b5 --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatsEmptyState.tsx @@ -0,0 +1,122 @@ +import { useState } from "react"; +import { resolveColorScheme, resolveThemeColors } from "./DFViewerParts/gridUtils"; +import type { ThemeConfig } from "./DFViewerParts/gridUtils"; +import { useColorScheme } from "./useColorScheme"; +import { DFMetaStats, canRequestStats, nextRequestTier, statsOverCeiling } from "./WidgetTypes"; + +/** A column the empty state can compute stats for on its own. */ +export interface StatsColumnOption { + /** The grid's own column name (a, b, c), which a request names. */ + field: string; + /** What the column is called on screen. */ + label: string; +} + +export interface StatsEmptyStateProps { + stats: DFMetaStats; + /** The columns of the table, for the per-column form. Without them the form + * is the whole-table button alone. */ + columns?: StatsColumnOption[]; + /** Asks the server for the stats: no argument for the whole table, `columns` + * for the per-column form. Without it the state is the message alone. */ + onComputeStats?: (opts?: { columns?: string[] }) => void; + themeConfig?: ThemeConfig; +} + +const countFormat = new Intl.NumberFormat("en-US"); +const compactFormat = new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }); + +const plural = (n: number, noun: string): string => (n === 1 ? `1 ${noun}` : `${countFormat.format(n)} ${noun}s`); + +/** + * The size the server's decision rested on, as "12.4M rows x 44 columns". A + * million rows and over read as a compact count. Empty when the server sent no + * estimate. + */ +export function formatStatsEstimate(estimate: DFMetaStats["estimate"]): string { + const parts: string[] = []; + if (typeof estimate?.rows === "number") { + parts.push(estimate.rows >= 1_000_000 ? `${compactFormat.format(estimate.rows)} rows` : plural(estimate.rows, "row")); + } + if (typeof estimate?.cols === "number") parts.push(plural(estimate.cols, "column")); + return parts.join(" x "); +} + +/** Why the stats were not computed, in a sentence: the server's reason for a + * session that is not computed, with the size when it sent one. A session at + * the server's ceiling reads as over the size limit, however the server put it. */ +export function notComputedMessage(stats: DFMetaStats): string { + const size = formatStatsEstimate(stats.estimate); + const sizeSuffix = size === "" ? "" : ` (${size})`; + switch (statsOverCeiling(stats) ? "ceiling" : stats.reason) { + case "ceiling": + return `Summary stats are not available for this table: it is over the size limit${sizeSuffix}.`; + case "size": + return `Summary stats were not computed for this table because it is large${sizeSuffix}.`; + case "host": + return `Summary stats were turned off for this table${sizeSuffix}.`; + case "cost": + return `Summary stats were paused because computing them was taking too long${sizeSuffix}.`; + default: + return `Summary stats are not computed for this table${sizeSuffix}.`; + } +} + +const TIER_DETAILS: Record = { + scalar: { label: "Compute basic stats", title: "Null counts, min, max, mean and std for each column" }, + full: { label: "Compute full stats", title: "Every summary stat, with histograms and value counts. Slower." }, +}; + +/** + * What the summary view shows while the stats are not computed: why, and the + * control that asks for them. The control is one button for the next tier the + * server allows (basic before full), with a picker that limits it to one column. + * A session the server's ceiling refused gets the message and nothing else. + */ +export function StatsEmptyState({ stats, columns = [], onComputeStats, themeConfig }: StatsEmptyStateProps) { + const osColorScheme = useColorScheme(); + const scheme = resolveColorScheme(osColorScheme, themeConfig); + const colors = resolveThemeColors(scheme, themeConfig); + const [picked, setPicked] = useState(""); + + const tier = nextRequestTier(stats); + const canCompute = onComputeStats !== undefined && canRequestStats(stats); + const column = columns.some((c) => c.field === picked) ? picked : ""; + const details = TIER_DETAILS[tier ?? ""] ?? TIER_DETAILS.full; + const label = stats.reason === "cost" ? "Continue computing stats" : details.label; + + return ( +
+

{notComputedMessage(stats)}

+ {canCompute ? ( +
+ {columns.length > 0 ? ( + + ) : null} + +
+ ) : null} +
+ ); +} diff --git a/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx new file mode 100644 index 000000000..eeb448e24 --- /dev/null +++ b/packages/buckaroo-js-core/src/components/StatusBar.stats.test.tsx @@ -0,0 +1,229 @@ +/** + * StatusBar — summary stats status (rows-first c4). + * + * A session that reports df_meta.stats gets one extra, fixed-width column in + * the status bar showing where the stats stand: loading ("pending"), not + * computed (with a control that asks for them), error (with the reason) or + * ready. A session that does not report df_meta.stats gets the status bar it + * always had. + * + * AG Grid is stubbed to capture the props the status bar hands it; the cell + * renderer is rendered on its own. + */ +import "@testing-library/jest-dom"; +import { render, screen, fireEvent } from "@testing-library/react"; + +// The props the status bar gave AG Grid on its last render. +const mockGrid: { props: any } = { props: null }; +jest.mock("ag-grid-react", () => ({ + AgGridReact: (props: any) => { + // React also calls this stub once with no props; keep the last real ones. + if (props) mockGrid.props = props; + return
; + }, +})); +jest.mock("./useColorScheme", () => ({ useColorScheme: () => "light" })); + +import { StatusBar, StatsStatusCell } from "./StatusBar"; +import { BuckarooOptions, BuckarooState, DFMeta, DFMetaStats } from "./WidgetTypes"; + +const baseMeta: DFMeta = { total_rows: 378, columns: 7, filtered_rows: 297, rows_shown: 297 }; +const options: BuckarooOptions = { + sampled: [], + cleaning_method: ["", "clean1"], + post_processing: ["", "post1"], + df_display: ["main", "summary"], + show_commands: ["0", "1"], +}; +const bState: BuckarooState = { + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, +}; + +const renderBar = (dfMeta: DFMeta, onComputeStats?: (opts?: { columns?: string[] }) => void) => + render( + {}} + buckarooOptions={options} + onComputeStats={onComputeStats} + />, + ); + +const fields = (): string[] => mockGrid.props.columnDefs.map((c: any) => c.field); + +describe("StatusBar stats column", () => { + beforeEach(() => { + mockGrid.props = null; + }); + + it("adds no column and no row field when df_meta has no stats (every session today)", () => { + renderBar(baseMeta); + expect(fields()).not.toContain("stats"); + expect(mockGrid.props.rowData[0]).not.toHaveProperty("stats"); + }); + + it("adds a fixed-width stats column after the summary-view selector when df_meta.stats is present", () => { + const stats: DFMetaStats = { status: "pending", tier: "schema", gen: 1 }; + renderBar({ ...baseMeta, stats }); + const names = fields(); + expect(names.indexOf("stats")).toBe(names.indexOf("df_display") + 1); + + const column = mockGrid.props.columnDefs[names.indexOf("stats")]; + expect(column.cellRenderer).toBe(StatsStatusCell); + // A fixed width, so the status changing never moves the other columns. + expect(typeof column.width).toBe("number"); + expect(column.flex).toBeUndefined(); + expect(mockGrid.props.rowData[0].stats).toBe(stats); + }); + + it("keeps the column for every status", () => { + for (const status of ["pending", "not_computed", "error", "complete"] as const) { + const { unmount } = renderBar({ ...baseMeta, stats: { status, gen: 1 } }); + expect(fields()).toContain("stats"); + unmount(); + } + }); + + it("hands the compute callback to the cell renderer through the grid context", () => { + const onComputeStats = jest.fn(); + renderBar({ ...baseMeta, stats: { status: "not_computed", gen: 1 } }, onComputeStats); + expect(mockGrid.props.context.onComputeStats).toBe(onComputeStats); + }); +}); + +describe("StatsStatusCell", () => { + const cell = (value: DFMetaStats | undefined, onComputeStats?: (opts?: { columns?: string[] }) => void) => + render(); + + it("pending: says the stats are being computed", () => { + cell({ status: "pending", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "pending"); + expect(root).toHaveTextContent("Computing summary stats"); + expect(root).toHaveAttribute("role", "status"); + }); + + it("not_computed: offers a Compute summary stats button that calls the handler", () => { + const onComputeStats = jest.fn(); + cell({ status: "not_computed", gen: 1 }, onComputeStats); + expect(screen.getByTestId("stats-status")).toHaveAttribute("data-stats-status", "not_computed"); + + fireEvent.click(screen.getByRole("button", { name: "Compute summary stats" })); + // Called with no arguments, not with the click event. + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("not_computed: with no handler there is no button, only the label", () => { + cell({ status: "not_computed", gen: 1 }); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); + }); + + it("error: shows the reason", () => { + cell({ status: "error", gen: 1, reason: "stats_failed" }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "error"); + expect(root).toHaveTextContent("Stats error: stats_failed"); + }); + + it("error: without a reason still says so", () => { + cell({ status: "error", gen: 1 }); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Stats error"); + }); + + it("complete: says the stats are ready", () => { + cell({ status: "complete", tier: "full", gen: 1 }); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "complete"); + expect(root).toHaveTextContent("Summary stats ready"); + }); + + it("renders nothing without stats", () => { + const { container } = cell(undefined); + expect(container).toBeEmptyDOMElement(); + }); +}); + +// The server's policy fields say why the stats were not computed and what may +// still be asked for (rows-first c5). +describe("StatsStatusCell, not computed by policy (rows-first c5)", () => { + const policy = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, + }); + const cell = (value: DFMetaStats | undefined, onComputeStats?: (opts?: { columns?: string[] }) => void) => + render(); + + it("size: offers the control, and says how large the table is in its title", () => { + const onComputeStats = jest.fn(); + cell(policy(), onComputeStats); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-status", "not_computed"); + expect(root).toHaveAttribute("data-stats-reason", "size"); + + const button = screen.getByRole("button", { name: "Compute summary stats" }); + expect(button).toHaveAttribute("title", expect.stringContaining("12.4M rows")); + fireEvent.click(button); + // The whole table, with no arguments: the wiring picks the tier from df_meta. + expect(onComputeStats).toHaveBeenCalledTimes(1); + expect(onComputeStats).toHaveBeenCalledWith(); + }); + + it("host: offers the control", () => { + cell(policy({ reason: "host" }), jest.fn()); + expect(screen.getByRole("button", { name: "Compute summary stats" })).toBeInTheDocument(); + }); + + it("cost: the control reads Continue, since the run was paused", () => { + const onComputeStats = jest.fn(); + cell(policy({ reason: "cost" }), onComputeStats); + expect(screen.queryByRole("button", { name: "Compute summary stats" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Continue computing stats" })); + expect(onComputeStats).toHaveBeenCalledTimes(1); + }); + + it("ceiling: a message and no control, even with a handler and a requestable list", () => { + cell(policy({ reason: "ceiling" }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + const root = screen.getByTestId("stats-status"); + expect(root).toHaveAttribute("data-stats-reason", "ceiling"); + expect(root).toHaveTextContent("Summary stats unavailable"); + expect(root).toHaveAttribute("title", expect.stringContaining("size limit")); + }); + + it("nothing requestable for a size the server chose: it is at the ceiling, so the message and no control", () => { + cell(policy({ requestable: [] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats unavailable"); + }); + + it("nothing requestable for a host's choice: the label only", () => { + cell(policy({ reason: "host", requestable: [] }), jest.fn()); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats not computed"); + }); + + it("an older server's not_computed (no policy fields) still offers the control", () => { + cell({ status: "not_computed", tier: "schema", gen: 1, reason: "host" }, jest.fn()); + expect(screen.getByRole("button", { name: "Compute summary stats" })).toBeInTheDocument(); + }); + + it("keeps the label for the other statuses whatever the reason says", () => { + cell({ status: "complete", tier: "full", gen: 1, reason: "ceiling" } as DFMetaStats, jest.fn()); + expect(screen.getByTestId("stats-status")).toHaveTextContent("Summary stats ready"); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/StatusBar.tsx b/packages/buckaroo-js-core/src/components/StatusBar.tsx index 1a08dedd8..57e5bc763 100644 --- a/packages/buckaroo-js-core/src/components/StatusBar.tsx +++ b/packages/buckaroo-js-core/src/components/StatusBar.tsx @@ -4,7 +4,8 @@ import * as _ from "lodash-es"; import { AgGridReact } from "ag-grid-react"; // the AG Grid React Component import { ColDef, GridApi, GridOptions } from "ag-grid-community"; import { basicIntFormatter } from "./DFViewerParts/Displayer"; -import { DFMeta } from "./WidgetTypes"; +import { DFMeta, DFMetaStats, canRequestStats, statsOverCeiling } from "./WidgetTypes"; +import { notComputedMessage } from "./StatsEmptyState"; import { BuckarooOptions } from "./WidgetTypes"; import { BuckarooState, BKeys } from "./WidgetTypes"; import { CustomCellEditorProps } from 'ag-grid-react'; @@ -307,6 +308,68 @@ export const SearchEditor = memo(({ value, onValueChange, stopEditing }: Custom ); }); +/** + * Where the summary stats stand, as the server reports it in df_meta.stats: + * loading while they are pending, a control to ask for them while they are not + * computed, the reason when they failed. The cell always renders one line in a + * fixed-width column, so changing status moves nothing. + * + * A session that is not computed offers the control unless the server's ceiling + * refused the stats or nothing is left to ask for. The control's label reads + * Continue when a run was paused on cost. It calls the host's callback with no + * arguments; the host asks for the tier df_meta.stats allows (see forceStats). + */ +export const StatsStatusCell = function (params: { + value?: DFMetaStats; + context?: { onComputeStats?: (opts?: { columns?: string[] }) => void }; +}) { + const stats = params.value; + if (stats === undefined) return null; + const onComputeStats = params.context?.onComputeStats; + const cell = (content: React.ReactNode, extra: React.HTMLAttributes = {}) => ( + + {content} + + ); + switch (stats.status) { + case "pending": + return cell( + <> +
diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts new file mode 100644 index 000000000..2493c3a08 --- /dev/null +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.test.ts @@ -0,0 +1,155 @@ +/** + * df_meta.stats helpers (rows-first c5). + * + * The server reports its stats policy to a client that advertises + * stats_ondemand, and leaves a field out when it holds the documented default: + * `auto_request` absent means true, `requestable` absent means ["full"]. These + * helpers read the fields the way the control and the scheduler need them. + */ +import { + DFMetaStats, + canRequestStats, + demandTier, + nextRequestTier, + statsAutoRequest, + statsOverCeiling, + statsRequestable, +} from "./WidgetTypes"; + +const stats = (over: Partial = {}): DFMetaStats => ({ + status: "not_computed", + tier: "schema", + gen: 1, + ...over, +}); + +describe("defaults for fields the server leaves out", () => { + it("requestable is ['full'] when absent, and the server's list when present", () => { + expect(statsRequestable(stats())).toEqual(["full"]); + expect(statsRequestable(undefined)).toEqual(["full"]); + expect(statsRequestable(stats({ requestable: ["scalar", "full"] }))).toEqual(["scalar", "full"]); + expect(statsRequestable(stats({ requestable: [] }))).toEqual([]); + }); + + it("auto_request is true when absent, and false only when the server says so", () => { + expect(statsAutoRequest(stats())).toBe(true); + expect(statsAutoRequest(undefined)).toBe(true); + expect(statsAutoRequest(stats({ auto_request: true }))).toBe(true); + expect(statsAutoRequest(stats({ auto_request: false }))).toBe(false); + }); +}); + +describe("nextRequestTier", () => { + it.each([ + // The smallest tier the policy allows goes first: scalar before full. + [["scalar", "full"], "schema", "scalar"], + [["full", "scalar"], "schema", "scalar"], + [["full"], "schema", "full"], + [["scalar"], "schema", "scalar"], + // After scalar has been reached the next one up is offered. + [["scalar", "full"], "scalar", "full"], + [["scalar"], "scalar", undefined], + // Nothing above the target: a ceiling left nothing to ask for. + [[], "schema", undefined], + ])("requestable %j with %s reached asks for %s", (requestable, tier, expected) => { + expect(nextRequestTier(stats({ requestable, tier }))).toBe(expected); + }); + + it("asks for full when the server reports no requestable (its default)", () => { + expect(nextRequestTier(stats())).toBe("full"); + }); + + it("treats a session with no tier as having reached schema", () => { + expect(nextRequestTier({ status: "not_computed", gen: 1, requestable: ["scalar"] })).toBe("scalar"); + }); + + it("ignores names that are not tiers", () => { + expect(nextRequestTier(stats({ requestable: ["columns", "scalar"] }))).toBe("scalar"); + expect(nextRequestTier(stats({ requestable: ["columns"] }))).toBeUndefined(); + }); + + it("is undefined without stats", () => { + expect(nextRequestTier(undefined)).toBeUndefined(); + }); +}); + +describe("canRequestStats", () => { + it("is true for a not_computed session with a tier left to ask for", () => { + expect(canRequestStats(stats())).toBe(true); + expect(canRequestStats(stats({ requestable: ["scalar", "full"], reason: "size" }))).toBe(true); + expect(canRequestStats(stats({ reason: "cost" }))).toBe(true); + expect(canRequestStats(stats({ reason: "host" }))).toBe(true); + }); + + it("is false at the ceiling, whatever requestable lists", () => { + expect(canRequestStats(stats({ reason: "ceiling" }))).toBe(false); + expect(canRequestStats(stats({ reason: "ceiling", requestable: ["scalar", "full"] }))).toBe(false); + }); + + it("is false when requestable is empty", () => { + expect(canRequestStats(stats({ requestable: [] }))).toBe(false); + }); + + it.each(["pending", "complete", "error"] as const)("is false when the status is %s", (status) => { + expect(canRequestStats(stats({ status }))).toBe(false); + }); + + it("is false without stats", () => { + expect(canRequestStats(undefined)).toBe(false); + }); +}); + +describe("demandTier", () => { + it("is the smallest tier that carries min and max the policy allows", () => { + expect(demandTier(stats({ requestable: ["scalar", "full"] }))).toBe("scalar"); + expect(demandTier(stats({ requestable: ["scalar"] }))).toBe("scalar"); + expect(demandTier(stats({ requestable: ["full"] }))).toBe("full"); + }); + + it("counts the target tier as allowed", () => { + expect(demandTier(stats({ tier_target: "scalar", requestable: [] }))).toBe("scalar"); + }); + + it("is undefined when the policy allows nothing above schema", () => { + expect(demandTier(stats({ tier_target: "schema", requestable: [] }))).toBeUndefined(); + expect(demandTier(undefined)).toBeUndefined(); + }); +}); + +// The server's ceiling keeps the stats from being computed. It says so with +// reason "ceiling" when the ceiling cut a request down; when the server sized the +// session at the ceiling itself, the reason is "size" and nothing is left to ask +// for. +describe("statsOverCeiling", () => { + it("is true for reason ceiling, whatever requestable lists", () => { + expect(statsOverCeiling(stats({ reason: "ceiling" }))).toBe(true); + expect(statsOverCeiling(stats({ reason: "ceiling", requestable: ["scalar", "full"] }))).toBe(true); + }); + + it("is true for a session the server sized with nothing left above it to ask for", () => { + expect(statsOverCeiling(stats({ reason: "size", requestable: [] }))).toBe(true); + expect(statsOverCeiling(stats({ reason: "size", tier: "scalar", requestable: ["scalar"] }))).toBe(true); + }); + + it("is false while a tier is left to ask for", () => { + expect(statsOverCeiling(stats({ reason: "size", requestable: ["scalar", "full"] }))).toBe(false); + expect(statsOverCeiling(stats({ reason: "size", tier: "scalar", requestable: ["scalar", "full"] }))).toBe(false); + }); + + it("is false when the server named no requestable (its default is full)", () => { + expect(statsOverCeiling(stats({ reason: "size" }))).toBe(false); + }); + + it("is false for the reasons that are not about size", () => { + expect(statsOverCeiling(stats({ reason: "host", requestable: [] }))).toBe(false); + expect(statsOverCeiling(stats({ reason: "cost", requestable: [] }))).toBe(false); + }); + + it.each(["pending", "complete", "error"] as const)("is false when the status is %s", (status) => { + expect(statsOverCeiling(stats({ status, reason: "ceiling" }))).toBe(false); + }); + + it("is false without stats", () => { + expect(statsOverCeiling(undefined)).toBe(false); + }); +}); diff --git a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx index e7f63015c..10a96f43c 100644 --- a/packages/buckaroo-js-core/src/components/WidgetTypes.tsx +++ b/packages/buckaroo-js-core/src/components/WidgetTypes.tsx @@ -1,11 +1,110 @@ +// Where the summary stats stand, as the server reports it in df_meta.stats. +// "pending": a stats_update is expected. "not_computed": none will be sent +// unless the user asks. A missing df_meta.stats means "complete", which is +// what servers that predate the field send. +export type StatsStatus = "complete" | "pending" | "not_computed" | "error"; + +// The tiers a session can reach, lowest first. "schema" is dtypes and identity +// keys, "scalar" adds the cheap per-column stats (min, max, mean, std, null +// count), "full" is every stat. +export type StatsTier = "schema" | "scalar" | "full"; +export const STATS_TIERS: readonly StatsTier[] = ["schema", "scalar", "full"]; + +export interface DFMetaStats { + status: StatsStatus; + // The tier reached so far. + tier?: string; + // Why the stats are not computed: "size", "host", "cost" or "ceiling". + reason?: string; + gen?: number; + // The fields below are sent to a client that advertises stats_ondemand, + // and only where they differ from the default noted on each. + // The tier the session is headed for. + tier_target?: string; + // The numbers the server's decision rested on. + estimate?: { rows?: number; cols?: number; bytes?: number }; + // Whether the client should ask for the stats on its own. Absent means true. + auto_request?: boolean; + // Tiers above the target an explicit request may still be granted. Absent + // means ["full"]. + requestable?: string[]; + // Columns whose styling needs stats now (a color_map reads min and max). + demand_columns?: string[]; + // Stat keys left out of the run, and keys computed from a sample. + omitted_keys?: string[]; + approx_keys?: string[]; + // Unlike the fields above, the server never sends this one. StatsChannel + // sets it when a run ends with the session still not computed (a request + // for some columns, or for the demand columns): the grid's own names of + // the columns whose stats the run merged into all_stats. + computed_columns?: string[]; +} + export interface DFMeta { // static, total_rows: number; columns: number; filtered_rows: number; rows_shown: number; + // Absent when the server predates the two-message protocol. + stats?: DFMetaStats; } +export const getStatsStatus = (meta: DFMeta | undefined): StatsStatus => + meta?.stats?.status ?? "complete"; + +// What a server that leaves a policy field out means by it (see DFMetaStats). +const DEFAULT_REQUESTABLE: readonly string[] = ["full"]; + +export const statsRequestable = (stats: DFMetaStats | undefined): string[] => + stats?.requestable ?? [...DEFAULT_REQUESTABLE]; + +export const statsAutoRequest = (stats: DFMetaStats | undefined): boolean => stats?.auto_request !== false; + +const tierRank = (tier: string | undefined): number => { + const rank = STATS_TIERS.indexOf(tier as StatsTier); + return rank === -1 ? 0 : rank; +}; + +/** + * The tier a request for more stats should ask for: the smallest tier in + * `requestable` above the one reached, so scalar goes before full. Names that + * are not tiers are ignored. Undefined when there is nothing left to ask for. + */ +export const nextRequestTier = (stats: DFMetaStats | undefined): StatsTier | undefined => { + if (stats === undefined) return undefined; + const requestable = statsRequestable(stats); + const reached = tierRank(stats.tier); + return STATS_TIERS.find((tier) => requestable.includes(tier) && tierRank(tier) > reached); +}; + +/** + * Whether the server's ceiling keeps the stats from being computed. It says so + * with reason "ceiling" when the ceiling cut a request down. When the server + * sized the session at the ceiling itself the reason is "size" and there is + * nothing above it to ask for, which comes to the same thing for the user. + */ +export const statsOverCeiling = (stats: DFMetaStats | undefined): boolean => + stats?.status === "not_computed" && + (stats.reason === "ceiling" || + (stats.reason === "size" && stats.requestable !== undefined && nextRequestTier(stats) === undefined)); + +/** Whether the "Compute summary stats" control applies: the stats are not + * computed, the server's ceiling did not refuse them, and a tier is left. */ +export const canRequestStats = (stats: DFMetaStats | undefined): boolean => + stats?.status === "not_computed" && !statsOverCeiling(stats) && nextRequestTier(stats) !== undefined; + +/** + * The tier of a request for the columns styling needs stats for (`demand_columns`): + * the smallest tier from scalar up that the policy allows, the target included, + * since scalar is where min and max come from. Undefined when it allows none. + */ +export const demandTier = (stats: DFMetaStats | undefined): StatsTier | undefined => { + if (stats === undefined) return undefined; + const allowed = [...statsRequestable(stats), ...(stats.tier_target === undefined ? [] : [stats.tier_target])]; + return STATS_TIERS.find((tier) => tierRank(tier) >= tierRank("scalar") && allowed.includes(tier)); +}; + export interface BuckarooOptions { sampled: string[]; cleaning_method: string[]; diff --git a/packages/buckaroo-js-core/src/index.ts b/packages/buckaroo-js-core/src/index.ts index 21e422a46..f3969d455 100644 --- a/packages/buckaroo-js-core/src/index.ts +++ b/packages/buckaroo-js-core/src/index.ts @@ -18,6 +18,9 @@ import { BuckarooStaticTable } from './components/BuckarooStaticTable'; import { BuckarooServerView, buckarooWsUrl } from './server/BuckarooServerView'; import { BuckarooView } from './server/BuckarooView'; import { WebSocketModel } from './server/WebSocketModel'; +import { makeLatestDictDecoder } from './server/latestDictDecoder'; +import { withStatsCapability } from './server/StatsChannel'; +import { StateOrchestrator, forceStats, requestStats, setVisibleColumns } from './server/StateOrchestrator'; import { HistogramCell } from "./components/DFViewerParts/HistogramCell"; import { InfiniteEx } from "./components/DFViewerParts/TableInfinite"; @@ -60,6 +63,12 @@ export default { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, + withStatsCapability, + StateOrchestrator, + requestStats, + forceStats, + setVisibleColumns, }; // Named exports for direct imports @@ -89,6 +98,12 @@ export { BuckarooView, buckarooWsUrl, WebSocketModel, + makeLatestDictDecoder, + withStatsCapability, + StateOrchestrator, + requestStats, + forceStats, + setVisibleColumns, }; export type { IModel } from './server/IModel'; diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx new file mode 100644 index 000000000..2b12a393b --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.caps.test.tsx @@ -0,0 +1,85 @@ +/** + * BuckarooServerView — capability advertisement (rows-first c2). + * + * The server records a client's capabilities from `?caps=` on the WebSocket + * URL, because it sends the first message before the client says anything. A + * client that merges `stats_update` must put it there, whatever URL the host + * passes in. So must one that can show a session whose stats are not computed + * and send tiered requests, which is `stats_ondemand` (rows-first c5). + */ +import { render, cleanup, waitFor } from "@testing-library/react"; +import { BuckarooServerView } from "./BuckarooServerView"; + +const capturedViewProps: any[] = []; + +jest.mock("./BuckarooView", () => ({ + BuckarooView: (props: any) => { + capturedViewProps.push(props); + return
; + }, + pickMode: (m: unknown) => (m === "buckaroo" ? "buckaroo" : "viewer"), +})); + +jest.mock("./WebSocketModel", () => ({ + WebSocketModel: class { constructor(_ws: any, _state: any) {} }, +})); + +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + binaryType = "arraybuffer"; + onopen: (() => void) | null = null; + onerror: ((e: any) => void) | null = null; + private listeners: Record void>> = {}; + constructor(public url: string) { + FakeWebSocket.instances.push(this); + setTimeout(() => { + this.onopen?.(); + setTimeout(() => { + const initial = { + type: "initial_state", + df_meta: { total_rows: 4, columns: 2, filtered_rows: 4, rows_shown: 4 }, + df_data_dict: {}, + df_display_args: {}, + mode: "viewer", + }; + this.listeners["message"]?.forEach((h) => h({ data: JSON.stringify(initial) } as any)); + }, 0); + }, 0); + } + addEventListener(ev: string, h: (e: any) => void) { + (this.listeners[ev] ??= new Set()).add(h); + } + removeEventListener(ev: string, h: (e: any) => void) { + this.listeners[ev]?.delete(h); + } + close() {} +} + +const origWebSocket = (globalThis as any).WebSocket; + +beforeAll(() => { + (globalThis as any).WebSocket = FakeWebSocket; +}); +afterAll(() => { + (globalThis as any).WebSocket = origWebSocket; +}); +afterEach(() => { + capturedViewProps.length = 0; + FakeWebSocket.instances.length = 0; + cleanup(); +}); + +describe("BuckarooServerView advertises stats_update and stats_ondemand", () => { + it("opens the socket with ?caps=stats_update,stats_ondemand", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?caps=stats_update,stats_ondemand"); + }); + + it("keeps the query string the host passed", async () => { + render(); + await waitFor(() => expect(capturedViewProps.length).toBeGreaterThan(0)); + expect(FakeWebSocket.instances[0].url).toBe("ws://x/ws/s?token=abc&caps=stats_update,stats_ondemand"); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx index 49fedbe3a..1687dddaa 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooServerView.tsx @@ -2,6 +2,7 @@ import * as React from "react"; import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; import { BuckarooView, BuckarooServerMetadata, @@ -106,7 +107,8 @@ export function BuckarooServerView({ (async () => { try { - ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + ws = new WebSocket(withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; await new Promise((resolve, reject) => { diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx new file mode 100644 index 000000000..4c4d95661 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/BuckarooView.stats.test.tsx @@ -0,0 +1,146 @@ +/** + * BuckarooView — the Compute summary stats control (rows-first c4). + * + * While df_meta.stats says the stats are not computed, the status bar offers a + * control that asks the server for them. BuckarooView hands the widget the + * callback, and the callback sends `stats_request {force: true, tier}` through + * whatever IModel the host gave it, with the tier taken from df_meta.stats + * (rows-first c5). The grid reports the columns it shows, and BuckarooView + * keeps them on the model, where the scheduler's requests read their hint. + */ +import { render, cleanup, act } from "@testing-library/react"; +import { BuckarooView } from "./BuckarooView"; +import type { IModel } from "./IModel"; + +const mockWidgetProps: any[] = []; +jest.mock("../components/BuckarooWidgetInfinite", () => ({ + BuckarooInfiniteWidget: (props: any) => { + mockWidgetProps.push(props); + return
; + }, + DFViewerInfiniteDS: () =>
, + getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), +})); + +function makeFakeModel(state: Record): { model: IModel; sent: any[] } { + const sent: any[] = []; + const model: IModel = { + send: (msg) => { sent.push(msg); }, + get: (k) => state[k], + set: (k, v) => { state[k] = v; }, + save_changes: () => {}, + on: () => {}, + off: () => {}, + }; + return { model, sent }; +} + +const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, +}; +const metaWith = (stats?: Record) => ({ + total_rows: 3, columns: 1, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const mountBuckaroo = async (state: Record) => { + const { model, sent } = makeFakeModel(state); + await act(async () => { + render(); + }); + return { model, sent, props: () => mockWidgetProps[mockWidgetProps.length - 1] }; +}; + +afterEach(() => { + mockWidgetProps.length = 0; + cleanup(); +}); + +describe("BuckarooView on_compute_stats (rows-first c4)", () => { + it("hands the widget a callback that sends a forced stats_request for the gen on screen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + + props().on_compute_stats(); + // A time-boxed step, like the scheduler's (rows-first c4b), for the one + // tier a server that names none can be asked for (rows-first c5). + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); + }); + + it("takes the tier from df_meta.stats.requestable: scalar before full", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, requestable: ["scalar", "full"] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats(); + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "scalar" }, + ]); + }); + + it("hands the per-column form's columns on to the request", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, requestable: ["scalar", "full"] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats({ columns: ["b"] }); + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "scalar", columns: ["b"] }, + ]); + }); + + it("sends nothing when the server left no tier to ask for", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9, reason: "ceiling", requestable: [] }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + props().on_compute_stats(); + expect(sent).toEqual([]); + }); + + it("records the columns the grid shows on the model, and the whole-table control does not send them", async () => { + const { model, sent, props } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_visible_columns).toBe("function"); + + props().on_visible_columns(["a", "b"]); + expect(model.get("visible_columns")).toEqual(["a", "b"]); + props().on_compute_stats(); + // On a forced request `columns` names what it is for, so the hint stays off. + expect(sent).toEqual([ + { type: "stats_request", stats_gen: 9, scope: "raw", incremental: true, force: true, tier: "full" }, + ]); + }); + + it("sends nothing when the model's df_meta carries no stats.gen", async () => { + const { sent, props } = await mountBuckaroo({ + df_meta: metaWith(), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(typeof props().on_compute_stats).toBe("function"); + props().on_compute_stats(); + expect(sent).toEqual([]); + }); + + it("sends no request on its own: asking is the scheduler's job, and a session that is not pending is left alone", async () => { + const { sent } = await mountBuckaroo({ + df_meta: metaWith({ status: "not_computed", tier: "schema", gen: 9 }), + df_data_dict: {}, + df_display_args: displayArgs, + }); + expect(sent).toEqual([]); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx index 8a38aa6cf..796e1db75 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.test.tsx @@ -9,18 +9,34 @@ import { render, cleanup, act } from "@testing-library/react"; import { BuckarooView } from "./BuckarooView"; import type { IModel } from "./IModel"; +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; // Stub the heavy widget surfaces — this test exercises the injection // wiring, not AG-Grid. The widget components instantiate AgGridReact which // is fragile under jsdom; the stub keeps the test focused on the model // contract. +// +// The viewer stub records the props BuckarooView hands it, so the rows-first +// tests below can see which df_meta / df_data_dict reached the widget. +const mockViewerProps: any[] = []; jest.mock("../components/BuckarooWidgetInfinite", () => ({ BuckarooInfiniteWidget: () =>
, - DFViewerInfiniteDS: () =>
, + DFViewerInfiniteDS: (props: any) => { + mockViewerProps.push(props); + return
; + }, getKeySmartRowCache: jest.fn(() => ({ __stub: "row-cache" })), })); -function makeFakeModel(): { model: IModel; events: Map>; sent: any[] } { +// Wrap decodeDFDataDict in a jest.fn that defaults to the real decoder, so the +// existing tests run unchanged and the rows-first tests can count and delay +// decodes. +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { ...actual, decodeDFDataDict: jest.fn(actual.decodeDFDataDict) }; +}); + +function makeFakeModel(): { model: IModel; events: Map>; sent: any[]; state: Record } { const events = new Map>(); const state: Record = {}; const sent: any[] = []; @@ -35,10 +51,13 @@ function makeFakeModel(): { model: IModel; events: Map>; s }, off: (e, h) => { events.get(e)?.delete(h); }, }; - return { model, events, sent }; + return { model, events, sent, state }; } -afterEach(() => cleanup()); +afterEach(() => { + mockViewerProps.length = 0; + cleanup(); +}); describe("BuckarooView (injectable IModel — #759)", () => { it("renders the viewer widget when given a fake IModel + initialState — no WebSocket needed", async () => { @@ -102,3 +121,110 @@ describe("BuckarooView (injectable IModel — #759)", () => { expect(onMetadata).toHaveBeenCalledWith({ path: "/data/sales.parquet", rows: 42 }, "tell me about sales"); }); }); + +// Rows-first c0a: a second message can follow the first at any moment, so the +// view has to cope with changes that land while it is still wiring itself up. +describe("BuckarooView two-message hardening (rows-first c0a)", () => { + const mockDecode = decodeDFDataDict as jest.Mock; + const realDecode = jest.requireActual("../components/DFViewerParts/resolveDFData").decodeDFDataDict; + + const displayArgs = { + main: { df_viewer_config: { pinned_rows: [], left_col_configs: [], column_config: [] }, summary_stats_key: "all_stats" }, + }; + const metaWith = (total_rows: number) => ({ total_rows, columns: 1, filtered_rows: total_rows, rows_shown: total_rows }); + const emit = (events: Map>, name: string, ...args: unknown[]) => { + for (const h of Array.from(events.get(name) ?? [])) h(...args); + }; + const lastProps = () => mockViewerProps[mockViewerProps.length - 1]; + + beforeEach(() => { + mockDecode.mockReset(); + mockDecode.mockImplementation(realDecode); + }); + + it("applies a change that reached the model before the effect subscribed", async () => { + // initialState is what the host held when it built the view. A second + // initial_state then landed on the model while React was committing, + // so its change:* events had no listener yet. + const { model, state } = makeFakeModel(); + state.df_meta = metaWith(99); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_meta.total_rows).toBe(99); + }); + + it("decodes a df_data_dict that reached the model before the effect subscribed", async () => { + const { model, state } = makeFakeModel(); + const raw = { format: "mock", id: "N" }; + state.df_data_dict = { main: raw }; + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + + await act(async () => { + render(); + }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "N" }]); + }); + + it("decodes one initial_state that carries metadata once, and renders its df_data_dict once", async () => { + const { model, events, state } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + const dictsBefore = new Set(mockViewerProps.map((p) => p.df_data_dict)); + mockDecode.mockClear(); + mockDecode.mockImplementation(async (dict: any) => ({ main: [{ index: 0, from: dict.main.id }] })); + + // The order WebSocketModel emits for a full frame: one change per key, + // then "metadata". The model state already holds the new values. + const frame = { + df_meta: metaWith(7), + df_data_dict: { main: { format: "mock", id: "F" } }, + df_display_args: displayArgs, + metadata: { path: "/data/f.parquet", rows: 7 }, + }; + Object.assign(state, frame); + await act(async () => { + emit(events, "change:df_meta", frame.df_meta); + emit(events, "change:df_data_dict", frame.df_data_dict); + emit(events, "change:df_display_args", frame.df_display_args); + emit(events, "metadata", frame.metadata, undefined); + }); + + expect(mockDecode).toHaveBeenCalledTimes(1); + const dictsAfter = new Set(mockViewerProps.map((p) => p.df_data_dict)); + expect(dictsAfter.size - dictsBefore.size).toBe(1); + expect(lastProps().df_meta.total_rows).toBe(7); + }); + + it("applies the newer df_data_dict when decodes complete out of order", async () => { + const { model, events } = makeFakeModel(); + const initialState = { df_meta: metaWith(1), df_data_dict: {}, df_display_args: displayArgs }; + await act(async () => { + render(); + }); + + const finish: Record void> = {}; + mockDecode.mockImplementation( + (dict: any) => + new Promise((resolve) => { + finish[dict.main.id] = () => resolve({ main: [{ index: 0, from: dict.main.id }] }); + }), + ); + await act(async () => { + emit(events, "change:df_data_dict", { main: { format: "mock", id: "older" } }); + emit(events, "change:df_data_dict", { main: { format: "mock", id: "newer" } }); + }); + // The newer decode finishes first, then the stale one. + await act(async () => { finish["newer"](); }); + await act(async () => { finish["older"](); }); + + expect(lastProps().df_data_dict.main).toEqual([{ index: 0, from: "newer" }]); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/BuckarooView.tsx b/packages/buckaroo-js-core/src/server/BuckarooView.tsx index 3ede8298e..855920862 100644 --- a/packages/buckaroo-js-core/src/server/BuckarooView.tsx +++ b/packages/buckaroo-js-core/src/server/BuckarooView.tsx @@ -1,15 +1,16 @@ import * as React from "react"; import { BuckarooInfiniteWidget, DFViewerInfiniteDS, getKeySmartRowCache } from "../components/BuckarooWidgetInfinite"; -import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; import { DFMeta, BuckarooState, BuckarooOptions } from "../components/WidgetTypes"; import { CommandConfigT } from "../components/CommandUtils"; import { Operation } from "../components/OperationUtils"; import { OperationResult, baseOperationResults } from "../components/DependentTabs"; -import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFData } from "../components/DFViewerParts/DFWhole"; import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; import { stampLayoutType, isFitContentLayout } from "../components/DFViewerParts/displayArgsUtils"; import { IModel } from "./IModel"; +import { makeLatestDictDecoder, RawDFDataDict } from "./latestDictDecoder"; +import { forceStats, setVisibleColumns } from "./StateOrchestrator"; export type BuckarooServerMode = "viewer" | "buckaroo"; @@ -161,6 +162,22 @@ export function BuckarooView({ const onMetadataRef = React.useRef(onMetadata); React.useEffect(() => { onMetadataRef.current = onMetadata; }, [onMetadata]); + // Every df_data_dict that reaches the view (the seed, change:df_data_dict, + // the metadata handler, the catch-up on subscribe) goes through one + // decoder, so a frame decodes once and the newest decode wins. The seed is + // already applied when it needs no resolution. + const dictDecoderRef = React.useRef<((raw: RawDFDataDict) => void) | null>(null); + if (dictDecoderRef.current === null) { + dictDecoderRef.current = makeLatestDictDecoder( + (d) => { + setDfDataDict(d as Record); + setDataReady(true); + }, + initialNeedsResolution ? undefined : (initialState.df_data_dict as RawDFDataDict), + ); + } + const loadDfDataDict = dictDecoderRef.current; + // Resolve any parquet-encoded payloads in df_data_dict. Pre-resolved // dicts (e.g. when BuckarooServerView already ran decodeDFDataDict) // pass through unchanged, so this is cheap in the common case. Skip @@ -168,19 +185,13 @@ export function BuckarooView({ // for the BuckarooServerView path. React.useEffect(() => { if (!initialNeedsResolution) return; - let cancelled = false; - const dict = initialState.df_data_dict as Record | undefined; + const dict = initialState.df_data_dict as RawDFDataDict; if (!dict) { setDataReady(true); return; } - decodeDFDataDict(dict).then((d) => { - if (cancelled) return; - setDfDataDict(d as Record); - setDataReady(true); - }); - return () => { cancelled = true; }; - }, [initialState, initialNeedsResolution]); + loadDfDataDict(dict); + }, [initialState, initialNeedsResolution, loadDfDataDict]); // Fire onMetadata for the initial payload, matching BuckarooServerView's // pre-split behavior. @@ -202,8 +213,7 @@ export function BuckarooView({ const onMeta = (metadata: BuckarooServerMetadata, prompt?: string) => { onMetadataRef.current?.(metadata, prompt); setDfMeta((model.get("df_meta") as DFMeta | undefined) ?? { ...DEFAULT_DF_META, total_rows: metadata?.rows ?? 0 }); - decodeDFDataDict((model.get("df_data_dict") as Record | undefined) ?? {}) - .then((d) => setDfDataDict(d as Record)); + loadDfDataDict(model.get("df_data_dict") as RawDFDataDict); setDfDisplayArgs((model.get("df_display_args") as Record | undefined) ?? {}); setBuckarooStateLocal((model.get("buckaroo_state") as BuckarooState | undefined) ?? DEFAULT_BUCKAROO_STATE); setBuckarooOptions((model.get("buckaroo_options") as BuckarooOptions | undefined) ?? DEFAULT_BUCKAROO_OPTIONS); @@ -212,9 +222,7 @@ export function BuckarooView({ setOperations((model.get("operations") as Operation[] | undefined) ?? []); }; const onDfMeta = (v: DFMeta) => setDfMeta(v); - const onDfDataDict = (v: Record) => { - decodeDFDataDict(v).then((d) => setDfDataDict(d as Record)); - }; + const onDfDataDict = (v: RawDFDataDict) => loadDfDataDict(v); const onDfDisplayArgs = (v: Record) => setDfDisplayArgs(v); const onBState = (v: BuckarooState) => setBuckarooStateLocal(v); const onBOpts = (v: BuckarooOptions) => setBuckarooOptions(v); @@ -232,6 +240,25 @@ export function BuckarooView({ model.on("change:operation_results", onOpRes); model.on("change:operations", onOps); + // A change:* emitted after the model was built and before this effect + // ran had no listener. The model holds the latest value of every key, + // so read each one now that the handlers are in place. A value that is + // the one already held is a no-op (same reference). + const catchUp: Array<[string, (v: any) => void]> = [ + ["df_meta", onDfMeta], + ["df_data_dict", onDfDataDict], + ["df_display_args", onDfDisplayArgs], + ["buckaroo_state", onBState], + ["buckaroo_options", onBOpts], + ["command_config", onCmdCfg], + ["operation_results", onOpRes], + ["operations", onOps], + ]; + for (const [key, apply] of catchUp) { + const v = model.get(key); + if (v !== undefined) apply(v); + } + return () => { model.off("metadata", onMeta); model.off("change:df_meta", onDfMeta); @@ -243,7 +270,7 @@ export function BuckarooView({ model.off("change:operation_results", onOpRes); model.off("change:operations", onOps); }; - }, [model]); + }, [model, loadDfDataDict]); const onBuckarooState = React.useCallback>>((newState) => { const resolved = typeof newState === "function" @@ -258,6 +285,19 @@ export function BuckarooView({ model.save_changes(); }, [model]); + // The Compute summary stats control, shown while the server reports the + // stats as not computed: the status bar's button, and the summary view's + // button and per-column form (which pass their columns). + const onComputeStats = React.useCallback((opts?: { columns?: string[] }) => { + forceStats(model, opts); + }, [model]); + + // The columns the grid shows, kept on the model for stats requests to carry + // as their hint. + const onVisibleColumns = React.useCallback((columns: string[]) => { + setVisibleColumns(model, columns); + }, [model]); + // gridUtils honors component_config.layoutType. Stamp it per entry so the // prop wins when provided. autoHeight=undefined → server value left intact. const effectiveDisplayArgs = React.useMemo( @@ -296,6 +336,8 @@ export function BuckarooView({ on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} + on_visible_columns={onVisibleColumns} /> ) : ( >(); + + constructor(public state: Record) {} + + get(key: string) { + return this.state[key]; + } + set(key: string, value: any) { + this.state[key] = value; + this.emit(`change:${key}`, value); + } + send(msg: any) { + this.sent.push(msg); } - last(): Record { - return JSON.parse(this.sent[this.sent.length - 1]); + on(event: string, handler: (...args: any[]) => void) { + if (!this.handlers.has(event)) this.handlers.set(event, new Set()); + this.handlers.get(event)!.add(handler); } - byType(type: string): Record[] { - return this.sent - .map((s) => JSON.parse(s)) - .filter((m) => m.type === type); + off(event: string, handler: (...args: any[]) => void) { + this.handlers.get(event)?.delete(handler); + } + emit(event: string, ...args: any[]) { + for (const h of Array.from(this.handlers.get(event) ?? [])) h(...args); + } + listenerCount() { + return Array.from(this.handlers.values()).reduce((n, set) => n + set.size, 0); + } + /** A full frame: each key lands and fires its change event in turn, as + * WebSocketModel's initial_state branch does. */ + frame(msg: Record) { + for (const [key, value] of Object.entries(msg)) this.set(key, value); } } -describe("StateOrchestrator", () => { - let ws: FakeWs; - let orch: StateOrchestrator; +const meta = (stats?: Record) => ({ + total_rows: 3, columns: 2, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); +const complete = (gen: number) => ({ status: "complete", tier: "full", gen }); +const dict = (rows: any[] = []) => ({ all_stats: rows }); +const statRow = (stat: string) => ({ index: stat, level_0: stat, a: 1 }); +// Typed loosely: the server's buckaroo_state also carries keys (search_string) +// that BuckarooState does not declare. +const bState = (over: Record = {}): any => ({ + sampled: false, cleaning_method: false, quick_command_args: {}, post_processing: false, + df_display: "main", show_commands: false, ...over, +}); + +// Every request is a time-boxed step (rows-first c4b); the earlier phases sent +// the whole run, so this helper had no `incremental`. +const request = (gen: number, extra: Record = {}) => ({ + type: "stats_request", stats_gen: gen, scope: "raw", incremental: true, ...extra, +}); + +const makeModel = (stats?: Record) => + new FakeModel({ df_meta: meta(stats), df_data_dict: dict(), buckaroo_state: bState() }); + +/** The first rows reached the client: WebSocketModel emits msg:custom once it + * has paired an infinite_resp with its parquet frame. */ +const rowsArrived = (model: FakeModel) => + model.emit("msg:custom", { type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }, []); + +const start = (model: FakeModel, opts: Record = {}) => { + const orchestrator = new StateOrchestrator({ model, ...opts }); + orchestrator.start(); + return orchestrator; +}; + +// With the defaults, a state change waits 2 x 250 ms before it asks again. +const DEBOUNCE = 500; +const FIRST_PAINT_TIMEOUT = 1500; + +// Runs due timers and every promise continuation they leave behind. +const tick = (ms = 0) => jest.advanceTimersByTimeAsync(ms); + +beforeEach(() => { + jest.useFakeTimers(); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +describe("when nothing is pending", () => { + it("requests nothing for a session whose df_meta has no stats (every session today)", async () => { + const model = makeModel(); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["complete", "not_computed", "error"])("requests nothing when the status is %s", async (status) => { + const model = makeModel({ status, tier: "schema", gen: 3 }); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); +}); + +describe("the first request", () => { + it("goes out after the first infinite_resp, not before", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(100); + expect(model.sent).toEqual([]); + + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); + }); + + it("goes out anyway when no rows come (an empty frame, the summary view)", async () => { + const model = makeModel(pending(3)); + start(model); + await tick(FIRST_PAINT_TIMEOUT - 1); + expect(model.sent).toEqual([]); + + await tick(2); + expect(model.sent).toEqual([request(3)]); + }); + + it("carries no force flag", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + await tick(); + expect(model.sent[0]).not.toHaveProperty("force"); + }); + + it("is sent once, however many row responses follow", async () => { + const model = makeModel(pending(3)); + start(model); + rowsArrived(model); + rowsArrived(model); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); +}); + +describe("one request per reply", () => { + const afterFirstRequest = async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; + + it("asks again for each reply that leaves the stats pending, and stops at the final one", async () => { + const { model } = await afterFirstRequest(); + expect(model.sent).toEqual([request(3)]); + + // No reply yet: nothing more goes out, however long the server takes. + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + + // A partial reply is a new df_data_dict under the same df_meta, which is + // what StatsChannel does for a stats_update that is not final. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + + model.set("df_data_dict", dict([statRow("mean"), statRow("std")])); + await tick(); + expect(model.sent).toHaveLength(3); + + // The final reply also sets df_meta.stats to complete. + model.set("df_data_dict", dict([statRow("mean"), statRow("std"), statRow("max")])); + model.set("df_meta", meta(complete(3))); + await tick(10_000); + expect(model.sent).toHaveLength(3); + }); + + it("reads a reply the same way whichever of the two events comes first", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta(complete(3))); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); + + it.each([ + ["df_meta then df_data_dict", ["df_meta", "df_data_dict"]], + ["df_data_dict then df_meta", ["df_data_dict", "df_meta"]], + ])("does not take a full frame for the same state as a reply (%s)", async (_label, order) => { + // A search term that changes only the highlight comes back as a full + // initial_state for the same stats_gen, with a new df_meta and a new dict. + const { model } = await afterFirstRequest(); + const full: Record = { df_meta: meta(pending(3)), df_data_dict: dict([statRow("dtype")]) }; + model.frame(Object.fromEntries(order.map((k) => [k, full[k]]))); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); + + it("stops when the server reports an error", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); + + it("stops when the server says the stats are not computed", async () => { + const { model } = await afterFirstRequest(); + model.set("df_meta", meta({ status: "not_computed", tier: "schema", gen: 3 })); + await tick(10_000); + expect(model.sent).toEqual([request(3)]); + }); +}); + +describe("incremental requests (rows-first c4b)", () => { + const sendFirst = async (model: FakeModel) => { + start(model); + rowsArrived(model); + await tick(); + }; + + it("asks for a time-boxed step, with no columns hint while the grid has not reported its columns", async () => { + const model = makeModel(pending(3)); + await sendFirst(model); + expect(model.sent).toEqual([{ type: "stats_request", stats_gen: 3, scope: "raw", incremental: true }]); + expect(model.sent[0]).not.toHaveProperty("columns"); + expect(model.sent[0]).not.toHaveProperty("force"); + }); + + it("sends the columns the grid shows as the hint", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["b", "c"]; + await sendFirst(model); + expect(model.sent).toEqual([request(3, { columns: ["b", "c"] })]); + }); + + it("sends no hint for an empty list of visible columns", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = []; + await sendFirst(model); + expect(model.sent).toEqual([request(3)]); + }); + + it("reads the hint again for each request of a run, so a column scrolled into view goes out next", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + + model.set("visible_columns", ["e", "f"]); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + model.set("df_data_dict", dict([statRow("mean"), statRow("std")])); + await tick(); + expect(model.sent).toEqual([ + request(3, { columns: ["a", "b"] }), + request(3, { columns: ["e", "f"] }), + request(3, { columns: ["e", "f"] }), + ]); + }); + + it("sends the same shape for the first request of the next gen after a state change", async () => { + const model = makeModel(pending(3)); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + model.frame({ df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([request(3, { columns: ["a", "b"] }), request(4, { columns: ["a", "b"] })]); + }); +}); + +describe("a state change", () => { + const pendingAt = async (gen: number) => { + const model = makeModel(pending(gen)); + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return { model, orchestrator }; + }; + + // The server answers a dataflow change with a frame for the next stats_gen, + // then the grid's refetch brings rows. They are separate messages, so the + // scheduler has read the frame by the time the rows arrive. + const nextFrame = async (model: FakeModel, gen: number) => { + model.frame({ df_meta: meta(pending(gen)), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + }; + + it("stops the requests for the old state, then asks once for the new one", async () => { + const { model } = await pendingAt(3); + expect(model.sent).toEqual([request(3)]); - beforeEach(() => { - jest.useFakeTimers(); - ws = new FakeWs(); - orch = new StateOrchestrator({ ws, minDebounceMs: 10, maxDebounceMs: 5000 }); + // The user changes a dataflow field while the request is out. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(10); + // The reply to the old request lands. It is not followed by another. + model.set("df_data_dict", dict([statRow("mean")])); + await tick(10); + await nextFrame(model, 4); + + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("reads a frame whose dict comes before its df_meta as the next state, not as a reply", async () => { + const { model } = await pendingAt(3); + // The frame for the next gen carries its dict before its df_meta. + model.frame({ df_data_dict: dict(), df_meta: meta(pending(4)) }); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([request(3), request(4)]); }); - afterEach(() => { - orch.dispose(); - jest.useRealTimers(); + it.each([ + ["post_processing", { post_processing: "log_scale" }], + ["cleaning_method", { cleaning_method: "aggressive" }], + ["quick_command_args", { quick_command_args: { search: ["x"] } }], + ])("a %s change cancels a request that is waiting out its delay", async (_field, change) => { + const { model } = await pendingAt(3); + await nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(300); + // 600 ms in: the request that was due at 500 ms never went out. + expect(model.sent).toEqual([request(3)]); + + await nextFrame(model, 5); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(5)]); }); - it("starts with token 0", () => { - expect(orch.currentToken).toBe(0); + it.each([ + ["search_string (the #1015 path)", { search_string: "x" }], + ["df_display", { df_display: "summary" }], + ["show_commands", { show_commands: "1" }], + ["sampled", { sampled: "sample" }], + ])("a %s-only change is skipped", async (_label, change) => { + const { model } = await pendingAt(3); + await nextFrame(model, 4); + await tick(300); + + model.set("buckaroo_state", bState(change)); + await tick(200); + // The request is still due at 500 ms, as if nothing had changed. + expect(model.sent).toEqual([request(3), request(4)]); }); - it("onStateChange bumps token and ships state_change", () => { - orch.onStateChange({ search_string: "x" }); - expect(orch.currentToken).toBe(1); - const msg = ws.last(); - expect(msg.type).toBe("state_change"); - expect(msg.state_token).toBe(1); - expect((msg.new_state as Record).search_string).toBe("x"); + it("a search_string-only change does not interrupt a chain of replies", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ search_string: "x" })); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); }); - it("schedules compute_stat_group after the debounce", () => { - orch.onStateChange({ search_string: "PIZZA" }); - // Only the immediate state_change has been sent so far. - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("a frame that repeats the same dataflow state is not a state change", async () => { + // Every full frame carries buckaroo_state back as the client sent it. + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState()); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(model.sent).toEqual([request(3), request(3)]); + }); - // Default baseline is 500ms × 2× = 1000ms; clamped to maxDebounceMs=5000. - jest.advanceTimersByTime(999); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("waits out the delay for a state change made after the earlier stats completed", async () => { + const { model } = await pendingAt(3); + await tick(250); // a request that took as long as the default assumes, so the delay is DEBOUNCE + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); - jest.advanceTimersByTime(1); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].scope).toBe("filt"); - expect(reqs[0].group).toBe("aggregate"); - expect(reqs[0].state_token).toBe(1); + // The first state asked at once; this one is a change to it. + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([request(3)]); + await tick(1); + expect(model.sent).toEqual([request(3), request(4)]); }); - it("back-to-back state_changes cancel the previous debounce timer", () => { - orch.onStateChange({ search_string: "P" }); - jest.advanceTimersByTime(500); - orch.onStateChange({ search_string: "PI" }); - // The first timer would have fired at t=1000ms; the second - // resets it so at t=999ms from the SECOND call (= 1499 overall) - // no compute_stat_group has fired yet. - jest.advanceTimersByTime(998); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("waits out the delay for the first state change of a model that started with its stats complete", async () => { + const model = makeModel(complete(3)); + start(model); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([]); - // The second timer fires. - jest.advanceTimersByTime(2); - const reqs = ws.byType("compute_stat_group"); - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(2); // second state_change's token + model.set("buckaroo_state", bState({ quick_command_args: { search: ["a"] } })); + await nextFrame(model, 4); + await tick(DEBOUNCE - 1); + expect(model.sent).toEqual([]); + await tick(1); + expect(model.sent).toEqual([request(4)]); }); - it("rapid typing produces just one aggregate request, with the latest token", () => { - // Simulate 5 keystrokes at 100ms intervals — typical typing cadence. - for (let i = 0; i < 5; i++) { - orch.onStateChange({ search_string: "P".repeat(i + 1) }); - jest.advanceTimersByTime(100); + it("asks again for the same state when the server never answers the change with a frame", async () => { + const { model } = await pendingAt(3); + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await tick(FIRST_PAINT_TIMEOUT + DEBOUNCE); + expect(model.sent).toEqual([request(3), request(3)]); + }); + + it("waits 2 x the last request's time, within the limits, before the next state's request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + expect(orchestrator.computeDebounce()).toBe(500); + + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); // the reply, 400 ms after the request + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + await nextFrame(model, 4); + await tick(799); + expect(model.sent).toHaveLength(2); + await tick(1); + expect(model.sent).toHaveLength(3); + expect(model.sent[2]).toEqual(request(4)); + }); + + it("measures a request once: later events do not stretch the delay", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 20_000 }); + rowsArrived(model); + await tick(400); + model.set("df_data_dict", dict([statRow("mean")])); + model.set("df_meta", meta(complete(3))); // the final reply + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + + // A full frame for the finished state, long afterwards. + await tick(10_000); + model.frame({ df_meta: meta(complete(3)), df_data_dict: dict() }); + await tick(); + expect(orchestrator.computeDebounce()).toBe(800); + }); + + it.each([ + [10, 100], + [400, 800], + [5000, 2000], + ])("computeDebounce after a %i ms request is %i ms (floor 100, ceiling 2000)", async (elapsed, expected) => { + const model = makeModel(pending(3)); + const orchestrator = start(model, { minDebounceMs: 100, maxDebounceMs: 2000 }); + rowsArrived(model); + await tick(elapsed); + model.set("df_data_dict", dict([statRow("mean")])); + await tick(); + expect(orchestrator.computeDebounce()).toBe(expected); + }); +}); + +// A session whose stats the server did not plan to compute says so in +// df_meta.stats (rows-first c5): the tier it is headed for, whether to ask for +// it on its own (`auto_request`, absent means true) and the columns whose +// styling needs stats now (`demand_columns`). With auto_request false the +// scheduler asks for those columns and nothing else. +const policy = (over: Record = {}) => ({ + status: "not_computed", + tier: "schema", + gen: 3, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + ...over, +}); + +describe("auto_request false: only the demand columns are asked for (rows-first c5)", () => { + // The request for demand columns: scoped to them, at the tier that carries + // min and max, and not forced (it is the scheduler's, not the user's). + const demand = (gen: number, columns: string[], tier = "scalar") => request(gen, { columns, tier }); + const sendFirst = async (model: FakeModel) => { + const orchestrator = start(model); + rowsArrived(model); + await tick(); + return orchestrator; + }; + + it.each(["pending", "not_computed"])("asks for nothing when the status is %s and no column needs stats", async (status) => { + const model = makeModel(policy({ status })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["pending", "not_computed"])("asks for nothing when demand_columns is empty (status %s)", async (status) => { + const model = makeModel(policy({ status, demand_columns: [] })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it.each(["pending", "not_computed"])("sends one scoped request for the demand columns (status %s)", async (status) => { + const model = makeModel(policy({ status, demand_columns: ["a", "c"] })); + model.state.visible_columns = ["a", "b"]; + await sendFirst(model); + // The visible columns are not what it asks for, and nothing is forced. + expect(model.sent).toEqual([demand(3, ["a", "c"])]); + expect(model.sent[0]).not.toHaveProperty("force"); + }); + + it("goes out after the first rows, not before, and once however many row responses follow", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + start(model); + await tick(100); + expect(model.sent).toEqual([]); + rowsArrived(model); + rowsArrived(model); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("asks for the next step of the demand for each reply that is not final, and stops at the final one", async () => { + const model = makeModel(policy({ demand_columns: ["a", "c"] })); + await sendFirst(model); + expect(model.sent).toHaveLength(1); + + // A partial reply is a new df_data_dict under the same df_meta. + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([demand(3, ["a", "c"]), demand(3, ["a", "c"])]); + + // The final reply says the session is still not computed (it ran for some + // columns only), in a new df_meta. + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + model.set("df_meta", meta(policy({ demand_columns: ["a", "c"] }))); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("does not ask again for the same demand after a refusal or a reply that carried no stats", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + // stats_aborted not_requestable and a ceiling reply both replace df_meta only. + model.set("df_meta", meta(policy({ demand_columns: ["a"], reason: "ceiling" }))); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + + // A full frame for the same state (a search highlight) does not restart it. + model.frame({ df_meta: meta(policy({ demand_columns: ["a"], reason: "ceiling" })), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("a full frame for the same state does not end it: the reply to the request in flight is still answered", async () => { + const model = makeModel(policy({ demand_columns: ["a", "c"] })); + await sendFirst(model); + // A search highlight comes back as a full frame with the same stats. + model.frame({ df_meta: meta(policy({ demand_columns: ["a", "c"] })), df_data_dict: dict([statRow("dtype")]) }); + await tick(10_000); + expect(model.sent).toEqual([demand(3, ["a", "c"])]); + + model.set("df_data_dict", dict([statRow("dtype"), statRow("min")])); + await tick(); + expect(model.sent).toEqual([demand(3, ["a", "c"]), demand(3, ["a", "c"])]); + }); + + it("asks for the next gen's demand after a state change", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + model.set("df_meta", meta(policy({ demand_columns: ["a"] }))); + await tick(); + + model.set("buckaroo_state", bState({ post_processing: "log_scale" })); + model.frame({ df_meta: meta(policy({ gen: 4, demand_columns: ["b"] })), df_data_dict: dict() }); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([demand(3, ["a"]), demand(4, ["b"])]); + }); + + it("asks for a changed list of demand columns on the same gen", async () => { + const model = makeModel(policy({ demand_columns: ["a"] })); + await sendFirst(model); + model.set("df_meta", meta(policy({ demand_columns: ["a"] }))); + await tick(); + + model.set("df_meta", meta(policy({ demand_columns: ["a", "d"] }))); + await tick(); + rowsArrived(model); + await tick(DEBOUNCE); + expect(model.sent).toEqual([demand(3, ["a"]), demand(3, ["a", "d"])]); + }); + + it("asks at the tier the policy allows: full when scalar is not requestable", async () => { + const model = makeModel(policy({ demand_columns: ["a"], requestable: ["full"] })); + await sendFirst(model); + expect(model.sent).toEqual([demand(3, ["a"], "full")]); + }); + + it("asks for nothing when the policy allows no tier above schema (a ceiling)", async () => { + const model = makeModel(policy({ demand_columns: ["a"], requestable: [], reason: "ceiling" })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("a pending session with auto_request false is not run whole", async () => { + const model = makeModel(policy({ status: "pending", demand_columns: ["a"] })); + await sendFirst(model); + expect(model.sent).toEqual([demand(3, ["a"])]); + }); + + it("leaves a session that does auto-request alone: demand_columns do not change its whole run", async () => { + for (const over of [{ auto_request: true }, { auto_request: undefined }]) { + const model = makeModel({ status: "pending", tier: "schema", gen: 3, demand_columns: ["a"], ...over }); + model.state.visible_columns = ["b"]; + await sendFirst(model); + expect(model.sent).toEqual([request(3, { columns: ["b"] })]); } - // Default debounce = 1000ms after the last keystroke. No aggregate yet. - expect(ws.byType("compute_stat_group")).toHaveLength(0); - - jest.advanceTimersByTime(1000); - const reqs = ws.byType("compute_stat_group"); - // Exactly one aggregate request, with the 5th (final) token. - expect(reqs).toHaveLength(1); - expect(reqs[0].state_token).toBe(5); - }); - - it("onStatGroupResult with matching token updates the baseline", () => { - orch.onStateChange({ search_string: "x" }); - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, - scope: "filt", - group: "aggregate", - elapsed_ms: 7500, - }); - expect(applied).toBe(true); - - // Next debounce is 2× 7500 = 15000ms, clamped to maxDebounceMs=5000. - expect(orch.computeDebounce("filt")).toBe(5000); - }); - - it("onStatGroupResult with stale token is silently dropped", () => { - orch.onStateChange({ search_string: "x" }); - orch.onStateChange({ search_string: "xy" }); - // Token is now 2. - const applied = orch.onStatGroupResult({ - type: "stat_group_result", - state_token: 1, // stale - scope: "filt", - group: "aggregate", - elapsed_ms: 9999, - }); - expect(applied).toBe(false); - // Baseline unchanged — debounce stays at the default. - expect(orch.computeDebounce("filt")).toBe(1000); }); - it("computeDebounce respects minDebounceMs floor", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 500, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 10, // 2× 10 = 20, well under floor - }); - expect(o.computeDebounce("filt")).toBe(500); + it("does not ask when the policy says not computed and auto_request is true (nothing is served yet)", async () => { + const model = makeModel(policy({ auto_request: true, demand_columns: ["a"] })); + await sendFirst(model); + await tick(10_000); + expect(model.sent).toEqual([]); + }); +}); + +describe("forceStats: the control's request (rows-first c5)", () => { + const notComputed = (over: Record = {}) => policy({ gen: 5, ...over }); + + it("asks for the smallest tier the policy allows: scalar before full", () => { + const model = makeModel(notComputed()); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar" })]); }); - it("computeDebounce respects maxDebounceMs ceiling", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - minDebounceMs: 200, - maxDebounceMs: 3000, - multiplier: 2, - }); - o.onStatGroupResult({ - type: "stat_group_result", - state_token: 0, - scope: "filt", - group: "aggregate", - elapsed_ms: 6000, // 2× = 12000, hits ceiling + it("asks for full when the server leaves requestable out (its default)", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 5, reason: "host" }); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "full" })]); + }); + + it("asks for the tier above the one reached once scalar is in", () => { + const model = makeModel(notComputed({ tier: "scalar" })); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "full" })]); + }); + + it("asks for scalar alone when only scalar is requestable", () => { + const model = makeModel(notComputed({ requestable: ["scalar"] })); + expect(forceStats(model)).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar" })]); + }); + + it("sends nothing when no tier is left to ask for", () => { + const model = makeModel(notComputed({ requestable: [], reason: "ceiling" })); + const before = model.get("df_meta"); + expect(forceStats(model)).toBe(false); + expect(model.sent).toEqual([]); + expect(model.get("df_meta")).toBe(before); + }); + + // The server cannot push a frame to a capable client, so the control marks + // the stats pending itself: the loading text and the placeholder rows show + // at once, and the control is gone, so a second click cannot start a second + // chain of requests. + it("marks the stats pending in a new df_meta, keeping the rest of it, so the control gives way to the loading state", () => { + const model = makeModel(notComputed()); + const before = model.get("df_meta"); + expect(forceStats(model)).toBe(true); + const after = model.get("df_meta"); + expect(after).not.toBe(before); + expect(after.stats).toEqual({ ...notComputed(), status: "pending" }); + expect(after.total_rows).toBe(before.total_rows); + }); + + it("sends nothing when df_meta carries no stats.gen", () => { + const model = makeModel(); + expect(forceStats(model)).toBe(false); + expect(model.sent).toEqual([]); + }); + + it("names no columns for the whole table, whatever the grid shows", () => { + const model = makeModel(notComputed()); + model.state.visible_columns = ["a", "b"]; + forceStats(model); + expect(model.sent[0]).not.toHaveProperty("columns"); + }); + + it("has a per-column form: the columns it is given, with the same force and tier", () => { + const model = makeModel(notComputed()); + model.state.visible_columns = ["a", "b"]; + expect(forceStats(model, { columns: ["c"] })).toBe(true); + expect(model.sent).toEqual([request(5, { force: true, tier: "scalar", columns: ["c"] })]); + }); +}); + +describe("a forced run is continued by the scheduler (rows-first c5)", () => { + const notComputed = (over: Record = {}) => policy({ gen: 5, ...over }); + const forced = (extra: Record = {}) => request(5, { force: true, tier: "scalar", ...extra }); + const startForced = async (opts?: { columns?: string[] }, stats = notComputed()) => { + const model = makeModel(stats); + start(model); + forceStats(model, opts); + await tick(); + return model; + }; + + it("asks again, with the same force, tier and columns, for each reply that is not final", async () => { + const model = await startForced({ columns: ["c"] }); + expect(model.sent).toEqual([forced({ columns: ["c"] })]); + + // No reply yet: nothing more goes out, however long the server takes. + await tick(10_000); + expect(model.sent).toHaveLength(1); + + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([forced({ columns: ["c"] }), forced({ columns: ["c"] })]); + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + await tick(); + expect(model.sent).toHaveLength(3); + }); + + it("sends the whole-table form again as the whole-table form", async () => { + const model = await startForced(); + model.state.visible_columns = ["a"]; + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toEqual([forced(), forced()]); + }); + + it("stops at a final reply, which replaces df_meta", async () => { + const model = await startForced(); + model.set("df_data_dict", dict([statRow("min")])); + await tick(); + expect(model.sent).toHaveLength(2); + + model.set("df_data_dict", dict([statRow("min"), statRow("max")])); + model.set("df_meta", meta({ status: "complete", tier: "scalar", gen: 5 })); + await tick(10_000); + expect(model.sent).toHaveLength(2); + }); + + it("stops at a ceiling reply or a refusal, which change df_meta and nothing else", async () => { + const model = await startForced(); + model.set("df_meta", meta(notComputed({ reason: "ceiling" }))); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + + it("drops the run when the gen moves on, and does not continue a late reply", async () => { + const model = await startForced(); + // The next gen's frame, with the stats pending as the run left them: only + // the gen differs. + model.frame({ df_meta: meta(notComputed({ gen: 6, status: "pending" })), df_data_dict: dict() }); + await tick(); + // A reply for the old run arrives after the new frame. + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + + it("is not started for a run recorded against a gen that is not on screen", async () => { + const model = makeModel(notComputed()); + start(model); + model.set("stats_forced", { gen: 9, tier: "scalar" }); + await tick(); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("does not continue a run recorded before the scheduler started", async () => { + const model = makeModel(notComputed()); + model.state.stats_forced = { gen: 5, tier: "scalar" }; + start(model); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("a full frame for the same gen ends the run: the model shows what the server says, and a late reply is not continued", async () => { + const model = await startForced(); + // The server's own view of the session is still not computed. + model.frame({ df_meta: meta(notComputed()), df_data_dict: dict([statRow("dtype")]) }); + await tick(10_000); + model.set("df_data_dict", dict([statRow("dtype"), statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); + + it("does nothing for a model nobody started a scheduler on", async () => { + const model = makeModel(notComputed()); + forceStats(model); + model.set("df_data_dict", dict([statRow("min")])); + await tick(10_000); + expect(model.sent).toEqual([forced()]); + }); +}); + +describe("requestStats", () => { + it("sends a stats_request for the gen the model shows", () => { + const model = makeModel(pending(7)); + expect(requestStats(model)).toBe(true); + expect(model.sent).toEqual([request(7)]); + }); + + it("force sends force: true, which the Compute summary stats control uses", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + expect(requestStats(model, { force: true })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true })]); + }); + + // Plan 3 s3.4: the control sends {stats_gen, scope, tier, force}, with a + // `columns` form per column. On a forced request `columns` names the + // columns the request is for, so the grid's visible columns (an ordering + // hint on the scheduler's requests) are not added to it (rows-first c5). + it("adds no columns hint to a forced request, however many columns the grid shows", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + model.state.visible_columns = ["a", "b"]; + expect(requestStats(model, { force: true })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true })]); + expect(model.sent[0]).not.toHaveProperty("columns"); + }); + + it("carries the tier it is asked for", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + expect(requestStats(model, { force: true, tier: "scalar" })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true, tier: "scalar" })]); + }); + + it("names the columns it is given, in place of the visible-columns hint", () => { + const model = makeModel({ status: "not_computed", tier: "schema", gen: 7 }); + model.state.visible_columns = ["a", "b"]; + expect(requestStats(model, { force: true, tier: "scalar", columns: ["c"] })).toBe(true); + expect(requestStats(model, { columns: ["d"] })).toBe(true); + expect(model.sent).toEqual([request(7, { force: true, tier: "scalar", columns: ["c"] }), request(7, { columns: ["d"] })]); + }); + + it("sends no tier and no columns the caller did not name, and no hint for an empty list", () => { + const model = makeModel(pending(7)); + expect(requestStats(model, { columns: [] })).toBe(true); + expect(model.sent).toEqual([request(7)]); + expect(model.sent[0]).not.toHaveProperty("tier"); + }); + + it("sends nothing when df_meta carries no stats.gen", () => { + const model = makeModel(); + expect(requestStats(model)).toBe(false); + expect(requestStats(model, { force: true })).toBe(false); + expect(model.sent).toEqual([]); + }); +}); + +describe("touchesDataflow", () => { + it("is true for each field the server reruns the dataflow for", () => { + expect(touchesDataflow(bState(), bState({ post_processing: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ cleaning_method: "x" }))).toBe(true); + expect(touchesDataflow(bState(), bState({ quick_command_args: { search: ["x"] } }))).toBe(true); + }); + + it("is false for the others, and for an equal quick_command_args that is a new object", () => { + expect(touchesDataflow(bState(), bState({ search_string: "x" }))).toBe(false); + expect(touchesDataflow(bState(), bState({ df_display: "summary" }))).toBe(false); + expect(touchesDataflow(bState({ quick_command_args: { search: ["x"] } }), bState({ quick_command_args: { search: ["x"] } }))).toBe(false); + }); + + it("is false when there is no earlier state to compare with", () => { + expect(touchesDataflow(undefined, bState({ post_processing: "x" }))).toBe(false); + }); +}); + +describe("start and stop", () => { + it("stop removes every listener and cancels the pending request", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + expect(model.listenerCount()).toBeGreaterThan(0); + + rowsArrived(model); + orchestrator.stop(); + expect(model.listenerCount()).toBe(0); + await tick(10_000); + expect(model.sent).toEqual([]); + }); + + it("start is idempotent: a second call adds no listeners", () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + const listeners = model.listenerCount(); + orchestrator.start(); + expect(model.listenerCount()).toBe(listeners); + }); + + it("can start again after a stop", async () => { + const model = makeModel(pending(3)); + const orchestrator = start(model); + orchestrator.stop(); + orchestrator.start(); + rowsArrived(model); + await tick(); + expect(model.sent).toEqual([request(3)]); + }); +}); + +// The scheduler is started by WebSocketModel, so a session reached through +// BuckarooServerView or the standalone page gets it with no wiring of its own. +describe("wired into WebSocketModel", () => { + class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } + deliverBinary() { + this.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + } + } + + const update = (gen: number, stat: string, final: boolean, remaining?: number) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final, + ...(remaining === undefined ? {} : { remaining }), + payload: { format: "json", layout: "wide", data: [{ index: stat, level_0: stat, a: 1 }] }, + elapsed_ms: 5, + }); + + const makeSocketModel = (stats?: Record) => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: meta(stats), + df_data_dict: { all_stats: [{ index: "dtype", level_0: "dtype", a: "int64" }] }, + buckaroo_state: bState(), }); - expect(o.computeDebounce("filt")).toBe(3000); + return { ws, model }; + }; + const rowsFromServer = (ws: FakeSocket) => { + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 3 }, length: 3 }); + ws.deliverBinary(); + }; + + it("requests the stats after the first rows, once per reply, and stops at the final one", async () => { + const { ws, model } = makeSocketModel(pending(3)); + await tick(100); + expect(ws.sent).toEqual([]); + + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver(update(3, "mean", false)); + await tick(); + expect(ws.sent).toEqual([request(3), request(3)]); + + ws.deliver(update(3, "std", true)); + await tick(10_000); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_meta").stats.status).toBe("complete"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std"]); }); - it("initialAggregateMs seeds the baseline before any observed compute", () => { - const o = new StateOrchestrator({ - ws: new FakeWs(), - initialAggregateMs: { filt: 250 }, - minDebounceMs: 10, - maxDebounceMs: 5000, - multiplier: 2, + it("walks partial replies to the final one: one request per reply, the stats pending until the last", async () => { + const { ws, model } = makeSocketModel(pending(3)); + model.set("visible_columns", ["a"]); + const pendingMeta = model.get("df_meta"); + rowsFromServer(ws); + await tick(); + const asked = request(3, { columns: ["a"] }); + expect(ws.sent).toEqual([asked]); + + for (const [i, stat] of ["mean", "std", "max"].entries()) { + ws.deliver(update(3, stat, false, 3 - i)); + await tick(); + expect(ws.sent).toEqual(Array(i + 2).fill(asked)); + // The merge adds the rows, and the stats stay pending: the same + // df_meta, so the placeholders and the loading text stay. + expect(model.get("df_meta")).toBe(pendingMeta); + expect(model.get("df_meta").stats.status).toBe("pending"); + } + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std", "max"]); + + // The final reply carries the complete all_stats. + ws.deliver({ + ...update(3, "min", true, 0), + payload: { + format: "json", + layout: "wide", + data: ["dtype", "mean", "std", "max", "min"].map((stat) => ({ index: stat, level_0: stat, a: stat === "dtype" ? "int64" : 2 })), + }, }); - expect(o.computeDebounce("filt")).toBe(500); // 2× 250 - // Unrelated scope still uses fallback. - expect(o.computeDebounce("clean")).toBe(1000); // 2× 500 (default) + await tick(10_000); + expect(ws.sent).toHaveLength(4); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean", "std", "max", "min"]); + expect(model.get("df_data_dict").all_stats[1].a).toBe(2); }); - it("dispose cancels all pending aggregate timers", () => { - orch.onStateChange({ search_string: "x" }); - orch.dispose(); - jest.advanceTimersByTime(10_000); - expect(ws.byType("compute_stat_group")).toHaveLength(0); + it("ends the run at a whole-run final from a server that ignores the incremental field", async () => { + const { ws, model } = makeSocketModel(pending(3)); + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + // An older server runs every unit and answers final, with no `remaining`. + ws.deliver(update(3, "mean", true)); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + expect(model.get("df_meta").stats.status).toBe("complete"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "mean"]); + }); + + it("stops a run when the gen changes, drops its late replies, and starts one for the new gen", async () => { + const { ws, model } = makeSocketModel(pending(3)); + model.set("visible_columns", ["a"]); + rowsFromServer(ws); + await tick(); + ws.deliver(update(3, "mean", false, 2)); + await tick(); + expect(ws.sent).toEqual([request(3, { columns: ["a"] }), request(3, { columns: ["a"] })]); + + // A state change moved the server to gen 4 with the run for gen 3 in flight. + ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); + ws.deliver(update(3, "std", false, 1)); + await tick(100); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_data_dict").all_stats).toEqual([]); + + rowsFromServer(ws); + await tick(DEBOUNCE); + expect(ws.sent[2]).toEqual(request(4, { columns: ["a"] })); + + // The new run chains like the old one did. + ws.deliver(update(4, "mean", false, 1)); + await tick(); + expect(ws.sent).toHaveLength(4); + expect(ws.sent[3]).toEqual(request(4, { columns: ["a"] })); + ws.deliver(update(4, "std", true, 0)); + await tick(10_000); + expect(ws.sent).toHaveLength(4); + expect(model.get("df_meta").stats.status).toBe("complete"); }); - it("scopesForAggregate parameter overrides default ['filt']", () => { - orch.onStateChange({ search_string: "x" }, { scopesForAggregate: ["filt", "clean"] }); - jest.advanceTimersByTime(10_000); - const reqs = ws.byType("compute_stat_group"); - const scopes = reqs.map((r) => r.scope).sort(); - expect(scopes).toEqual(["clean", "filt"]); + it("moves on to the next gen when a state change frame arrives", async () => { + const { ws } = makeSocketModel(pending(3)); + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3)]); + + ws.deliver({ type: "initial_state", df_meta: meta(pending(4)), df_data_dict: dict() }); + await tick(); + rowsFromServer(ws); + await tick(DEBOUNCE); + expect(ws.sent).toEqual([request(3), request(4)]); + }); + + it("sends nothing for a model whose df_meta has no stats", async () => { + const { ws } = makeSocketModel(); + rowsFromServer(ws); + await tick(10_000); + expect(ws.sent).toEqual([]); + }); + + // Rows-first c5: a not computed session, the control, and the replies. + describe("a session whose stats are not computed", () => { + const forcedRequest = (extra: Record = {}) => request(3, { force: true, tier: "scalar", ...extra }); + + it("sends nothing on its own, and the control's forced run goes tier by tier to its final reply", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + rowsFromServer(ws); + await tick(10_000); + expect(ws.sent).toEqual([]); + + expect(forceStats(model)).toBe(true); + expect(ws.sent).toEqual([forcedRequest()]); + + // The control marked the stats pending, and they stay pending until the + // final reply says otherwise. + expect(model.get("df_meta").stats.status).toBe("pending"); + const pendingMeta = model.get("df_meta"); + ws.deliver({ ...update(3, "min", false, 2), tier: "scalar" }); + await tick(); + expect(ws.sent).toEqual([forcedRequest(), forcedRequest()]); + expect(model.get("df_meta")).toBe(pendingMeta); + + ws.deliver({ ...update(3, "max", true, 0), tier: "scalar" }); + await tick(10_000); + expect(ws.sent).toHaveLength(2); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "scalar", gen: 3 }); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "min", "max"]); + }); + + it("a request over the ceiling ends in the ceiling message, and nothing more is sent", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "full", final: true, status: "not_computed", reason: "ceiling" }); + await tick(10_000); + expect(ws.sent).toEqual([forcedRequest()]); + expect(model.get("df_meta").stats).toMatchObject({ status: "not_computed", reason: "ceiling", gen: 3 }); + }); + + it("a request the server refuses with not_requestable returns the session to not computed and ends the run", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model); + expect(model.get("df_meta").stats.status).toBe("pending"); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + await tick(10_000); + expect(ws.sent).toEqual([forcedRequest()]); + expect(model.get("df_meta").stats).toEqual(policy({ gen: 3 })); + }); + + it("a run for some columns ends not computed, and the control can ask again", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3 })); + forceStats(model, { columns: ["a"] }); + expect(ws.sent).toEqual([forcedRequest({ columns: ["a"] })]); + + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + // The session is not computed again, and says which column the run filled. + expect(model.get("df_meta").stats).toEqual({ ...policy({ gen: 3 }), computed_columns: ["a"] }); + + expect(forceStats(model)).toBe(true); + expect(ws.sent).toEqual([forcedRequest({ columns: ["a"] }), forcedRequest()]); + expect(ws.sent[1]).not.toHaveProperty("columns"); + }); + + it("asks for the demand columns after the first rows and nothing else", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3, demand_columns: ["a"] })); + model.set("visible_columns", ["a", "b"]); + await tick(100); + expect(ws.sent).toEqual([]); + + rowsFromServer(ws); + await tick(); + expect(ws.sent).toEqual([request(3, { columns: ["a"], tier: "scalar" })]); + + // The reply carries stats for those columns and leaves the session not computed. + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(ws.sent).toHaveLength(1); + expect(model.get("df_meta").stats.status).toBe("not_computed"); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "min"]); + }); + + it("a demand run's final reply names the columns it filled", async () => { + const { ws, model } = makeSocketModel(policy({ gen: 3, demand_columns: ["a"] })); + rowsFromServer(ws); + await tick(); + ws.deliver({ ...update(3, "min", true, 0), tier: "scalar", status: "not_computed" }); + await tick(10_000); + expect(model.get("df_meta").stats).toEqual({ + ...policy({ gen: 3, demand_columns: ["a"] }), + computed_columns: ["a"], + }); + }); }); }); diff --git a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts index e096ac606..72a782eba 100644 --- a/packages/buckaroo-js-core/src/server/StateOrchestrator.ts +++ b/packages/buckaroo-js-core/src/server/StateOrchestrator.ts @@ -1,153 +1,481 @@ /** - * Client-side scheduler for the JS-driven progressive-stats protocol. + * Client-side scheduler for the stats wire (rows-first c4). * - * On a state_change, ships the cheap (scalar) request immediately so - * df_meta + scalar pinned rows update fast. Schedules the expensive - * (aggregate) compute per scope behind an adaptive debounce — by - * default 2× the last observed aggregate compute time for that scope, - * clamped to ``[minDebounceMs, maxDebounceMs]``. + * A server that defers a session's summary stats sends a first frame whose + * `df_meta.stats.status` is "pending", then waits to be asked. This scheduler + * asks: it sends `stats_request {stats_gen, scope: "raw", incremental: true}` + * once the first rows have arrived, sends another for each reply that leaves + * the stats pending (a `stats_update` that is not final), and stops when the + * status leaves "pending". Nothing is requested unless `df_meta.stats` says + * pending, so a session whose server reports no stats is never touched. * - * Token-based cancellation: every state_change bumps an internal - * token; results carrying a stale token are dropped on arrival. + * Every request is a time-boxed step: the server runs units for a short + * budget (about 75 ms) and replies with a partial `stats_update` + * (`final: false`, the fragments that are new), which StatsChannel merges into + * `all_stats` without leaving "pending". Each such reply is answered with the + * next request, and the final reply, which carries the complete `all_stats`, + * ends the run. A request also names the columns the grid shows (`columns`, the + * grid's own column names), so the server runs the units that cover them first. + * A server that does not know either field ignores them, runs the whole run and + * answers with a final reply, which ends the run the same way. * - * See plans/js-driven-stat-debounce.md for the full protocol design. + * A session the server's policy left without stats says so with status + * "not_computed" and the fields of its policy (see DFMetaStats). The scheduler + * asks for nothing on such a session, with two exceptions. With `auto_request` + * false it asks for the columns in `demand_columns`, whose styling needs stats + * now, as one scoped request per gen (see demand runs below). And it continues + * a run the user started with the "Compute summary stats" control (see + * forceStats): each reply that is not final is answered with the same request. * - * This module is transport-agnostic — pass any object with a - * ``send(s: string)`` method. Existing buckaroo WS connections fit. + * It watches the model and sends through it, so any IModel works: + * + * change:df_meta the status and stats_gen of the state on screen + * change:df_data_dict a reply merged into all_stats (StatsChannel's job) + * change:buckaroo_state a state change made on this client + * change:stats_forced a run the user started (forceStats) + * msg:custom an infinite_resp, the first rows + * + * The columns the grid shows are read from the model's `visible_columns` key + * when a request goes out (see setVisibleColumns). It is a client-side key: the + * server never sends it. + * + * The merge is not here. A reply reaches this class only as a change on the + * model: a new df_data_dict under the same df_meta is a partial update, and a + * status other than "pending" is the end. + * + * A state change is a change to a dataflow field of buckaroo_state (the fields + * the server reruns the dataflow for) and the frame that answers it carries the + * next stats_gen. Either one starts the wait again with a delay, so a burst of + * changes (typing in the search box) asks once, for the state it ends on. A + * change to anything else, a search_string for one, leaves the schedule alone. + * + * `stats_gen` is the server's token for the state a reply describes. A reply for + * another gen is dropped by StatsChannel, so no client-side token is kept here. + */ +import { + BuckarooState, + DFMeta, + DFMetaStats, + StatsTier, + demandTier, + nextRequestTier, + statsAutoRequest, +} from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +/** What the scheduler needs of a model. */ +export type StatsModel = Pick; + +/** The fields of buckaroo_state the server reruns the dataflow for, and so + * bumps stats_gen on. Mirrors _DATAFLOW_FIELDS in + * buckaroo/server/websocket_handler.py. */ +export const DATAFLOW_STATE_FIELDS = ["post_processing", "cleaning_method", "quick_command_args"] as const; + +/** Whether `next` differs from `prev` in a dataflow field. False when there is + * no earlier state to compare with. */ +export function touchesDataflow(prev: BuckarooState | undefined, next: BuckarooState | undefined): boolean { + if (prev === undefined || next === undefined) return false; + return DATAFLOW_STATE_FIELDS.some((field) => JSON.stringify(prev[field]) !== JSON.stringify(next[field])); +} + +/** The model key that holds the grid's column names currently on screen. */ +export const VISIBLE_COLUMNS_KEY = "visible_columns"; + +/** Record the columns the grid shows, for the next `stats_request` to carry as + * its `columns` hint. An empty list means none are known. */ +export function setVisibleColumns(model: Pick, columns: string[]): void { + model.set(VISIBLE_COLUMNS_KEY, columns); +} + +export interface StatsRequestOptions { + /** Ask for stats the server did not plan to compute. The "Compute summary + * stats" control sends this. */ + force?: boolean; + /** The tier asked for. */ + tier?: string; + /** The columns the request is for. A forced request names the columns it + * is for or none, so the grid's columns are not added to it. A request + * that is not forced carries them as a hint when it names none. */ + columns?: string[]; +} + +/** + * Send a time-boxed `stats_request` for the stats_gen of the state the model + * shows. The scheduler's requests carry the columns the grid shows as the hint, + * when they are known; a request that names its columns carries those instead, + * and a forced one carries only those. Returns false, and sends nothing, when + * the model's df_meta carries no stats.gen. */ +export function requestStats(model: Pick, opts: StatsRequestOptions = {}): boolean { + const gen = (model.get("df_meta") as DFMeta | undefined)?.stats?.gen; + if (typeof gen !== "number") return false; + const columns = opts.columns ?? (opts.force ? undefined : (model.get(VISIBLE_COLUMNS_KEY) as string[] | undefined)); + model.send({ + type: "stats_request", + stats_gen: gen, + scope: "raw", + incremental: true, + ...(opts.tier === undefined ? {} : { tier: opts.tier }), + ...(Array.isArray(columns) && columns.length > 0 ? { columns } : {}), + ...(opts.force ? { force: true } : {}), + }); + return true; +} -export type ScopeName = "raw" | "clean" | "filt"; -export type CostGroup = "scalar" | "aggregate"; +/** The model key that records a run the user started, for the scheduler to + * continue. */ +export const FORCED_RUN_KEY = "stats_forced"; -export interface WsLike { - send(message: string): void; +/** What forceStats records under FORCED_RUN_KEY. */ +export interface ForcedRun { + gen: number; + tier: string; + columns?: string[]; +} + +/** + * The "Compute summary stats" control: ask for stats the server did not plan to + * compute, `stats_request {force: true, tier}`. The tier is the smallest the + * server allows above the one reached (scalar before full), and `opts.columns` + * is the per-column form. Sends nothing, and returns false, when no tier is + * left to ask for or df_meta carries no stats.gen. + * + * The stats are marked pending in a new df_meta. The server sends no frame to a + * capable client, so without this the loading text and the placeholder rows + * would wait for the first reply, and the control would stay on screen to be + * clicked again. A final reply, a refusal or a frame from the server puts the + * status it names in place of it. + * + * The run is recorded on the model, where a scheduler started on it picks it up + * and answers each reply that is not final with the same request. + */ +export function forceStats(model: Pick, opts: { columns?: string[] } = {}): boolean { + const meta = model.get("df_meta") as DFMeta | undefined; + const stats = meta?.stats; + const tier = nextRequestTier(stats); + if (meta === undefined || stats === undefined || tier === undefined || typeof stats.gen !== "number") return false; + const columns = opts.columns !== undefined && opts.columns.length > 0 ? opts.columns : undefined; + if (!requestStats(model, { force: true, tier, columns })) return false; + const run: ForcedRun = { gen: stats.gen, tier, ...(columns === undefined ? {} : { columns }) }; + model.set("df_meta", { ...meta, stats: { ...stats, status: "pending" } }); + model.set(FORCED_RUN_KEY, run); + return true; } export interface OrchestratorOptions { - ws: WsLike; - /** Lower bound on the per-scope debounce. Default 200 ms. */ + model: StatsModel; + /** Lower bound on the delay before asking for a new state's stats. Default 200 ms. */ minDebounceMs?: number; - /** Upper bound on the per-scope debounce. Default 3000 ms. */ + /** Upper bound on that delay. Default 3000 ms. */ maxDebounceMs?: number; - /** Multiplier on the last observed aggregate compute time. Default 2. */ + /** Multiplier on the last observed request time. Default 2. */ multiplier?: number; - /** Initial aggregate baseline per scope (used until we observe a real one). */ - initialAggregateMs?: Partial>; + /** Request time assumed until one has been observed. Default 250 ms. */ + initialRequestMs?: number; + /** How long to wait for the first rows before asking anyway (an empty + * frame, the summary view, and a grid that never fetches send none). + * Default 1500 ms. */ + firstPaintTimeoutMs?: number; } -export interface StatGroupResult { - type: "stat_group_result"; - state_token: number; - scope: ScopeName; - group: CostGroup; - elapsed_ms: number; - stats?: unknown; +type Timer = ReturnType; + +// What the scheduler is driving. "auto" is the run of a pending session. A +// "demand" run asks for the columns whose styling needs stats, on a session +// that does not auto-request: a scoped request, at the tier that carries min and +// max. A "forced" run is one the user started; its first request was sent by +// forceStats and the scheduler sends the rest. +type Run = + | { kind: "auto" } + | { kind: "demand"; columns: string[]; tier: StatsTier } + | { kind: "forced"; gen: number; tier: string; columns?: string[] }; + +// The parts of df_meta.stats a demand or forced run is watching: a change in +// any of them means the reply to the run's request has arrived and ended it. +// A new df_meta with the same three is a full frame for the same state. +interface StatsBasis { + status: string; + reason?: string; + tier?: string; } +const basisOf = (stats: DFMetaStats | undefined): StatsBasis => ({ + status: stats?.status ?? "", + reason: stats?.reason, + tier: stats?.tier, +}); + +const sameBasis = (a: StatsBasis, b: StatsBasis): boolean => + a.status === b.status && a.reason === b.reason && a.tier === b.tier; + export class StateOrchestrator { - private token = 0; - private aggregateTimers = new Map>(); - private lastAggregateMs = new Map(); - private readonly ws: WsLike; + private readonly model: StatsModel; private readonly minDebounceMs: number; private readonly maxDebounceMs: number; private readonly multiplier: number; - /** Fallback baseline before any real aggregate compute has been observed. */ - private readonly defaultBaselineMs = 500; + private readonly initialRequestMs: number; + private readonly firstPaintTimeoutMs: number; + + private started = false; + // The run being driven and the key that identifies it (the gen, and for a + // demand run the columns and tier); undefined while nothing is. + private run: Run | undefined; + private runKey: string | undefined; + // The stats a demand or forced run started from. + private basis: StatsBasis | undefined; + // A demand run that has ended, which is not started again. + private doneKey: string | undefined; + // A request is out and its reply has not been seen. + private inFlight = false; + // Delay before the next request: 0 for the first state and for each reply + // in a chain, the debounce after a state change. + private delayMs = 0; + // The state the model held at start has been read, so a pending state after + // it is a state change, whatever the first one was. + private began = false; + private sentAt = 0; + private lastRequestMs: number | undefined; + private requestTimer: Timer | undefined; + private paintTimer: Timer | undefined; + private syncQueued = false; + private seenMeta: unknown; + private seenDict: unknown; + private seenState: BuckarooState | undefined; + private seenForced: unknown; constructor(opts: OrchestratorOptions) { - this.ws = opts.ws; + this.model = opts.model; this.minDebounceMs = opts.minDebounceMs ?? 200; this.maxDebounceMs = opts.maxDebounceMs ?? 3000; this.multiplier = opts.multiplier ?? 2; - if (opts.initialAggregateMs) { - for (const [scope, ms] of Object.entries(opts.initialAggregateMs)) { - if (ms != null) this.lastAggregateMs.set(scope as ScopeName, ms); - } - } + this.initialRequestMs = opts.initialRequestMs ?? 250; + this.firstPaintTimeoutMs = opts.firstPaintTimeoutMs ?? 1500; } - /** - * Current state-change token. Tests inspect this; production - * code doesn't usually need it. - */ - get currentToken(): number { - return this.token; + /** Start watching the model, and adopt the state it already holds. */ + start(): void { + if (this.started) return; + this.started = true; + this.seenMeta = this.model.get("df_meta"); + this.seenDict = this.model.get("df_data_dict"); + this.seenState = this.model.get("buckaroo_state"); + // A run recorded before this scheduler started is not its to continue. + this.seenForced = this.model.get(FORCED_RUN_KEY); + this.model.on("change:df_meta", this.onModelChange); + this.model.on("change:df_data_dict", this.onModelChange); + this.model.on("change:buckaroo_state", this.onState); + this.model.on(`change:${FORCED_RUN_KEY}`, this.onModelChange); + this.model.on("msg:custom", this.onMessage); + this.sync(); + this.began = true; + } + + /** Stop watching and cancel anything scheduled. Call on unmount. */ + stop(): void { + if (!this.started) return; + this.started = false; + this.model.off("change:df_meta", this.onModelChange); + this.model.off("change:df_data_dict", this.onModelChange); + this.model.off("change:buckaroo_state", this.onState); + this.model.off(`change:${FORCED_RUN_KEY}`, this.onModelChange); + this.model.off("msg:custom", this.onMessage); + this.standDown(); + this.doneKey = undefined; + this.began = false; } /** - * Compute the debounce delay (ms) for a given scope based on - * the last observed aggregate compute time, clamped to - * ``[minDebounceMs, maxDebounceMs]``. + * The delay (ms) before asking for a new state's stats: the last observed + * request time times the multiplier, clamped to + * `[minDebounceMs, maxDebounceMs]`. A request that took longer means the + * server was busy, so the next one waits longer. */ - computeDebounce(scope: ScopeName): number { - const last = this.lastAggregateMs.get(scope) ?? this.defaultBaselineMs; - const raw = last * this.multiplier; + computeDebounce(): number { + const raw = (this.lastRequestMs ?? this.initialRequestMs) * this.multiplier; return Math.max(this.minDebounceMs, Math.min(this.maxDebounceMs, raw)); } - /** - * Drive a single user-initiated state change. - * - * 1. Bump the state token. - * 2. Cancel any pending aggregate timers from the previous change. - * 3. Ship the ``state_change`` message (server will reply with - * the scalar stats fast). - * 4. For each scope expected to have aggregate work, schedule a - * debounced ``compute_stat_group`` request. Timer fires the - * request only if no further state_change has bumped the - * token meanwhile. - */ - onStateChange( - newState: Record, - opts?: { scopesForAggregate?: ScopeName[] }, - ): void { - const token = ++this.token; - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); - - this.ws.send(JSON.stringify({ - type: "state_change", - state_token: token, - new_state: newState, - })); - - const scopes = opts?.scopesForAggregate ?? ["filt"]; - for (const scope of scopes) { - const delay = this.computeDebounce(scope); - const tid = setTimeout(() => { - // If a newer state_change bumped the token while we - // were waiting, drop the request — sending it would - // produce a stat_group_aborted from the server anyway. - if (token !== this.token) return; - this.ws.send(JSON.stringify({ - type: "compute_stat_group", - state_token: token, - scope, - group: "aggregate", - })); - }, delay); - this.aggregateTimers.set(scope, tid); + // A frame fires one change event per key, in the order the server wrote + // them (df_data_dict before df_meta), so the model is read once they have + // all landed, in a microtask, not at the first event. + private readonly onModelChange = (): void => { + if (this.syncQueued) return; + this.syncQueued = true; + void Promise.resolve().then(() => { + this.syncQueued = false; + if (this.started) this.sync(); + }); + }; + + private readonly onState = (next?: BuckarooState): void => { + const prev = this.seenState; + this.seenState = next ?? this.model.get("buckaroo_state"); + if (this.run === undefined || this.run.kind === "forced" || !touchesDataflow(prev, this.seenState)) return; + this.begin(this.runKey!, this.run, this.basis); + }; + + private readonly onMessage = (msg?: { type?: string }): void => { + if (msg?.type === "infinite_resp") this.markPainted(); + }; + + private sync(): void { + const meta = this.model.get("df_meta") as DFMeta | undefined; + const dict = this.model.get("df_data_dict"); + const metaChanged = meta !== this.seenMeta; + const dictChanged = dict !== this.seenDict; + this.seenMeta = meta; + this.seenDict = dict; + + const stats = meta?.stats; + if (this.adoptForced(stats)) return; + if (this.run?.kind === "forced") { + this.syncForced(stats, metaChanged, dictChanged); + if (this.run !== undefined) return; + // The run is over; what it left is handled like any other state. + } + + const desired = this.desiredRun(stats); + if (desired === undefined) { + // Complete, an error, a session that asks for nothing, or a server + // that reports no stats. + this.standDown(); + } else if (desired.key === this.runKey) { + if (this.inFlight && dictChanged && !metaChanged) { + // A new df_data_dict under the same df_meta is a stats_update that + // is not final (a full frame for this state replaces both). One + // request per reply: ask again. + this.inFlight = false; + this.noteRequestTime(); + this.arm(); + } else if (this.inFlight && metaChanged && desired.run.kind === "demand" && this.basisChanged(stats)) { + // A refusal, or a final reply that left the session in another + // state: the run is over and is not asked for again. + this.doneKey = this.runKey; + this.standDown(); + } + } else if (desired.key !== this.doneKey) { + this.begin(desired.key, desired.run, basisOf(stats)); } } - /** - * Process a ``stat_group_result`` from the server. Stale results - * (mismatched token) are silently ignored. Successful results - * update the per-scope aggregate baseline used by the next - * debounce. - * - * Returns true if the result was applied, false if it was stale. - */ - onStatGroupResult(msg: StatGroupResult): boolean { - if (msg.state_token !== this.token) return false; - this.lastAggregateMs.set(msg.scope, msg.elapsed_ms); + // The run the stats ask for, if any. A pending session is run whole, unless + // the server says not to auto-request, and then only the columns whose + // styling needs stats are asked for. A not computed session is asked for + // nothing, on the same condition. + private desiredRun(stats: DFMetaStats | undefined): { key: string; run: Run } | undefined { + if (stats === undefined || typeof stats.gen !== "number") return undefined; + if (stats.status !== "pending" && stats.status !== "not_computed") return undefined; + if (statsAutoRequest(stats)) { + return stats.status === "pending" ? { key: JSON.stringify(["auto", stats.gen]), run: { kind: "auto" } } : undefined; + } + const columns = stats.demand_columns; + const tier = demandTier(stats); + if (!Array.isArray(columns) || columns.length === 0 || tier === undefined) return undefined; + return { key: JSON.stringify(["demand", stats.gen, tier, columns]), run: { kind: "demand", columns, tier } }; + } + + // A run the user started has been recorded on the model, and its first + // request has gone out: continue it, and give way to what the scheduler was + // driving. Returns whether it took one over. + private adoptForced(stats: DFMetaStats | undefined): boolean { + const record = this.model.get(FORCED_RUN_KEY) as ForcedRun | undefined; + if (record === this.seenForced) return false; + this.seenForced = record; + if (record === undefined || record.gen !== stats?.gen) return false; + this.standDown(); + this.runKey = JSON.stringify(["forced", record.gen]); + this.run = { kind: "forced", ...record }; + this.basis = basisOf(stats); + this.inFlight = true; + this.sentAt = Date.now(); return true; } - /** Cancel all pending aggregate timers. Call on widget unmount. */ - dispose(): void { - for (const t of this.aggregateTimers.values()) clearTimeout(t); - this.aggregateTimers.clear(); + private syncForced(stats: DFMetaStats | undefined, metaChanged: boolean, dictChanged: boolean): void { + const run = this.run; + if (run?.kind !== "forced") return; + if (stats?.gen !== run.gen || (metaChanged && this.basisChanged(stats))) { + // A final reply, a refusal or a new state ends the run. + this.standDown(); + } else if (this.inFlight && dictChanged && !metaChanged) { + // A reply that is not final: ask again. + this.inFlight = false; + this.noteRequestTime(); + this.arm(); + } + } + + private basisChanged(stats: DFMetaStats | undefined): boolean { + return this.basis === undefined || !sameBasis(this.basis, basisOf(stats)); + } + + // Wait for rows, then ask, for the run `key`. The state the model starts + // with asks as soon as rows are up; every later one is a state change and + // waits out the debounce. + private begin(key: string, run: Run, basis: StatsBasis | undefined): void { + this.clearTimers(); + this.runKey = key; + this.run = run; + this.basis = basis; + this.doneKey = undefined; + this.inFlight = false; + this.delayMs = this.began ? this.computeDebounce() : 0; + this.paintTimer = setTimeout(() => { + this.paintTimer = undefined; + this.markPainted(); + }, this.firstPaintTimeoutMs); + } + + private standDown(): void { + if (this.inFlight) this.noteRequestTime(); + this.clearTimers(); + this.run = undefined; + this.runKey = undefined; + this.basis = undefined; + this.inFlight = false; + } + + // The first rows are up (or the wait for them timed out): ask. + private markPainted(): void { + if (this.paintTimer !== undefined) { + clearTimeout(this.paintTimer); + this.paintTimer = undefined; + } + this.arm(); + } + + private arm(): void { + if (this.inFlight || this.run === undefined || this.requestTimer !== undefined) return; + this.requestTimer = setTimeout(() => { + this.requestTimer = undefined; + this.fire(); + }, this.delayMs); + } + + private fire(): void { + const run = this.run; + if (run === undefined) return; + const opts: StatsRequestOptions = + run.kind === "demand" + ? { columns: run.columns, tier: run.tier } + : run.kind === "forced" + ? { force: true, tier: run.tier, columns: run.columns } + : {}; + if (requestStats(this.model, opts)) { + this.inFlight = true; + this.sentAt = Date.now(); + this.delayMs = 0; + } + } + + private noteRequestTime(): void { + this.lastRequestMs = Date.now() - this.sentAt; + } + + private clearTimers(): void { + if (this.requestTimer !== undefined) clearTimeout(this.requestTimer); + if (this.paintTimer !== undefined) clearTimeout(this.paintTimer); + this.requestTimer = undefined; + this.paintTimer = undefined; } } diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.test.ts b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts new file mode 100644 index 000000000..a8c4ece8b --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.test.ts @@ -0,0 +1,566 @@ +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A capable client receives a stats-free `initial_state` (df_meta.stats.status + * "pending"), asks for the stats and merges the `stats_update` that answers, + * keyed by `stats_gen`. These tests drive a WebSocketModel with a fake socket, + * the way the server's frames reach it. + */ +import { WebSocketModel } from "./WebSocketModel"; +import { withStatsCapability } from "./StatsChannel"; +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; + +// A wide summary-stats envelope (parquet_b64, layout "wide") as the server +// sends one; the decoder tests use the same fixture. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const wideFixture = require("../components/DFViewerParts/test-fixtures/summary_stats_parquet_b64.json"); + +// The real decoder, except that an envelope carrying `hold` waits on `gate`, +// so a test can deliver a frame while a decode is in flight. +let gate: Promise = Promise.resolve(); +jest.mock("../components/DFViewerParts/resolveDFData", () => { + const actual = jest.requireActual("../components/DFViewerParts/resolveDFData"); + return { + ...actual, + decodeDFData: jest.fn(async (env: any, buffers?: DataView[]) => { + if (env && env.hold) await gate; + return actual.decodeDFData(env, buffers); + }), + }; +}); + +class FakeSocket { + readyState = 1; // WebSocket.OPEN + onmessage: ((e: MessageEvent) => void) | null = null; + sent: any[] = []; + send(data: string) { + this.sent.push(JSON.parse(data)); + } + deliver(msg: object) { + this.onmessage?.({ data: JSON.stringify(msg) } as MessageEvent); + } +} + +// Lets every promise continuation (the payload decode) run. +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +const row = (stat: string, cells: Record) => ({ index: stat, level_0: stat, ...cells }); + +// What the schema tier ships: identity rows only. +const schemaStats = () => [ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 3, c: 3 }), +]; + +const metaFor = (stats?: Record) => ({ + total_rows: 3, columns: 3, filtered_rows: 3, rows_shown: 3, + ...(stats === undefined ? {} : { stats }), +}); + +const pending = (gen: number) => ({ status: "pending", tier: "schema", gen }); + +const frame = (gen: number | undefined, allStats: any = schemaStats()) => ({ + type: "initial_state", + df_meta: metaFor(gen === undefined ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, +}); + +// The server's payload is a wide DFEnvelope; the json format decodes to the +// same row shape without a parquet fixture. +const update = (gen: number, rows: any[], extra: object = {}) => ({ + type: "stats_update", + stats_gen: gen, + scope: "raw", + tier: "full", + final: true, + payload: { format: "json", layout: "wide", data: rows }, + elapsed_ms: 12.5, + ...extra, +}); + +// `gen` null builds a model whose df_meta carries no stats, as an old server's does. +function makeModel(gen: number | null = 3, allStats: any = schemaStats()) { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(gen === null ? undefined : pending(gen)), + df_data_dict: { all_stats: allStats }, + }); + const events: { event: string; value: any }[] = []; + for (const key of ["df_data_dict", "df_meta"]) { + model.on(`change:${key}`, (value: any) => events.push({ event: `change:${key}`, value })); + } + return { ws, model, events }; +} + +// The capabilities a client advertises on the WebSocket URL. stats_update +// means it merges stats_update messages; stats_ondemand (rows-first c5) means it +// can show a session whose stats are not computed and send tiered requests, so +// the server may apply its stats policy to it. +describe("withStatsCapability", () => { + it("adds ?caps=stats_update,stats_ondemand to a bare URL", () => { + expect(withStatsCapability("ws://localhost:8700/ws/sales")).toBe( + "ws://localhost:8700/ws/sales?caps=stats_update,stats_ondemand", + ); + }); + + it("appends to an existing query string", () => { + expect(withStatsCapability("ws://h/ws/s?token=abc")).toBe("ws://h/ws/s?token=abc&caps=stats_update,stats_ondemand"); + }); + + it("extends an existing caps value", () => { + expect(withStatsCapability("ws://h/ws/s?caps=other")).toBe("ws://h/ws/s?caps=other,stats_update,stats_ondemand"); + }); + + it("leaves a URL that already advertises both capabilities alone", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update,stats_ondemand")).toBe( + "ws://h/ws/s?caps=stats_update,stats_ondemand", + ); + expect(withStatsCapability("ws://h/ws/s?caps=a,stats_ondemand,stats_update")).toBe( + "ws://h/ws/s?caps=a,stats_ondemand,stats_update", + ); + }); + + it("adds only the capability a host's caps value lacks", () => { + expect(withStatsCapability("ws://h/ws/s?caps=stats_update")).toBe("ws://h/ws/s?caps=stats_update,stats_ondemand"); + expect(withStatsCapability("ws://h/ws/s?caps=stats_ondemand")).toBe("ws://h/ws/s?caps=stats_ondemand,stats_update"); + }); + + it("keeps the fragment last", () => { + expect(withStatsCapability("ws://h/ws/s#frag")).toBe("ws://h/ws/s?caps=stats_update,stats_ondemand#frag"); + }); +}); + +describe("stats_update merge", () => { + it("key-merges the payload's columns into all_stats and keeps the other columns", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [ + row("length", { a: 3, b: 5 }), + row("mean", { a: 2, b: 4.5 }), + ])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int64", b: "float64", c: "object" }), + row("length", { a: 3, b: 5, c: 3 }), + row("mean", { a: 2, b: 4.5 }), + ]); + }); + + it("does not let a null in the payload erase a value already merged", async () => { + const { ws, model } = makeModel(3); + // The wide pivot pads a stat a column did not carry with null. + ws.deliver(update(3, [row("dtype", { a: null, b: "float32" })])); + await settle(); + const dtype = model.get("df_data_dict").all_stats.find((r: any) => r.index === "dtype"); + expect(dtype).toEqual(row("dtype", { a: "int64", b: "float32", c: "object" })); + }); + + it("assigns a new df_data_dict and leaves the previous objects untouched", async () => { + const { ws, model, events } = makeModel(3); + const before = model.get("df_data_dict"); + const beforeStats = before.all_stats; + const snapshot = JSON.parse(JSON.stringify(beforeStats)); + ws.deliver(update(3, [row("length", { a: 99 }), row("mean", { a: 2 })])); + await settle(); + const after = model.get("df_data_dict"); + expect(after).not.toBe(before); + expect(after.all_stats).not.toBe(beforeStats); + expect(beforeStats).toEqual(snapshot); + const dictEvents = events.filter((e) => e.event === "change:df_data_dict"); + expect(dictEvents).toHaveLength(1); + expect(dictEvents[0].value).toBe(after); + }); + + it("merges onto an all_stats that arrived as an undecoded envelope", async () => { + const { ws, model } = makeModel(3); + // A later initial_state hands the model the dict as the server sent it. + ws.deliver(frame(4, { format: "json", layout: "wide", data: schemaStats() })); + ws.deliver(update(4, [row("mean", { a: 2, b: 4.5, c: null })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(Array.isArray(stats)).toBe(true); + expect(stats.map((r: any) => r.index)).toEqual(["dtype", "length", "mean"]); + }); + + it("keeps the other df_data_dict keys", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { all_stats: schemaStats(), empty: [], main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const dict = model.get("df_data_dict"); + expect(dict.empty).toEqual([]); + expect(dict.main).toEqual([{ a: 1 }]); + expect(dict.all_stats).toHaveLength(3); + }); + + it("merges a wide parquet_b64 payload as the server sends it", async () => { + const { ws, model } = makeModel(3, [row("orig_col_name", { a: "first" })]); + const decoded: any[] = await decodeDFData(wideFixture); + expect(decoded.length).toBeGreaterThan(1); + ws.deliver(update(3, [], { payload: wideFixture })); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats[0]).toEqual(row("orig_col_name", { a: "first" })); + for (const decodedRow of decoded) { + expect(stats).toContainEqual(decodedRow); + } + }); + + it("builds all_stats when the model holds no dict yet", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { df_meta: metaFor(pending(3)) }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ all_stats: [row("mean", { a: 2 })] }); + }); + + it("adds all_stats to a dict that has none", async () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(pending(3)), + df_data_dict: { main: [{ a: 1 }] }, + }); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict")).toEqual({ main: [{ a: 1 }], all_stats: [row("mean", { a: 2 })] }); + }); + + it("applies updates that arrive back to back, in order", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + ws.deliver(update(3, [row("mean", { b: 4.5 }), row("max", { a: 9 })])); + await settle(); + const stats = model.get("df_data_dict").all_stats; + expect(stats.find((r: any) => r.index === "mean")).toEqual(row("mean", { a: 2, b: 4.5 })); + expect(stats.find((r: any) => r.index === "max")).toEqual(row("max", { a: 9 })); + }); +}); + +describe("stats_update order", () => { + it("merges updates in arrival order when an earlier payload decodes slower", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const { ws, model, events } = makeModel(3); + const held = { format: "json", layout: "wide", data: [row("mean", { a: 1 })], hold: true }; + ws.deliver(update(3, [], { final: false, payload: held })); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the second payload has decoded; the first is held + release(); + await settle(); + // The final update goes last: its value stands and the status + // completes only after both merges. + const mean = model.get("df_data_dict").all_stats.find((r: any) => r.index === "mean"); + expect(mean.a).toBe(2); + expect(events.map((e) => e.event)).toEqual(["change:df_data_dict", "change:df_data_dict", "change:df_meta"]); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + +describe("stats_update and df_meta.stats", () => { + it("a final update marks the stats complete at the update's tier", async () => { + const { ws, model, events } = makeModel(3); + const metaBefore = model.get("df_meta"); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + const meta = model.get("df_meta"); + expect(meta).not.toBe(metaBefore); + expect(meta.stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + expect(meta.total_rows).toBe(3); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(1); + }); + + it("a non-final update merges but leaves the status pending", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })], { final: false })); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + expect(model.get("df_meta").stats.status).toBe("pending"); + expect(events.filter((e) => e.event === "change:df_meta")).toHaveLength(0); + }); +}); + +// A reply that did not run, or ran for some columns only, names the status the +// session is left in (rows-first c5). The server answers a request over its +// ceiling with `stats_update {final: true, status: "not_computed", reason: +// "ceiling"}` and no payload. +describe("a final stats_update with a status", () => { + const policyStats = { + status: "not_computed", + tier: "schema", + gen: 3, + reason: "size", + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 44 }, + auto_request: false, + requestable: ["scalar", "full"], + }; + const makePolicyModel = () => { + const ws = new FakeSocket(); + const model = new WebSocketModel(ws as unknown as WebSocket, { + df_meta: metaFor(policyStats), + df_data_dict: { all_stats: schemaStats() }, + }); + const events: string[] = []; + for (const key of ["df_data_dict", "df_meta"]) model.on(`change:${key}`, () => events.push(key)); + return { ws, model, events }; + }; + + it("not_computed with reason ceiling ends in the ceiling message: the reason changes, the policy fields stay", async () => { + const { ws, model, events } = makePolicyModel(); + // The reply names the tier that was asked for; the session stays at the + // tier it had reached. + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "full", final: true, status: "not_computed", reason: "ceiling" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, reason: "ceiling" }); + // No payload: the stats the client has are left as they were. + expect(events).toEqual(["df_meta"]); + expect(model.get("df_data_dict").all_stats).toEqual(schemaStats()); + }); + + it("takes the reply's status and reason whatever they are", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", final: true, status: "not_computed", reason: "cost" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, reason: "cost" }); + }); + + it("merges a payload it carries, and leaves the session not computed, naming the columns it filled (a reply for some columns only)", async () => { + const { ws, model, events } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 }), row("max", { a: 9 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "min", "max"]); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); + expect(events).toEqual(["df_data_dict", "df_meta"]); + }); + + it("counts a column only where a cell has a value: a null fills nothing", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1, b: null })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["a"]); + }); + + it("names the columns every reply of a run filled, and a reply that is not final leaves df_meta alone", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1, b: null })], { tier: "scalar", final: false })); + await settle(); + expect(model.get("df_meta").stats).toEqual(policyStats); + + ws.deliver(update(3, [row("max", { b: 9, a: null })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a", "b"] }); + }); + + it("a final reply with no payload names the columns the run's earlier replies filled", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_update", stats_gen: 3, scope: "raw", tier: "scalar", final: true, status: "not_computed" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); + }); + + it("keeps the columns of an earlier run when a later one ends not computed", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", status: "not_computed" })); + await settle(); + ws.deliver(update(3, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["a", "b"]); + }); + + it("a new gen starts over: the replies for the gen the client left are not counted", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ + type: "initial_state", + df_meta: metaFor({ ...policyStats, gen: 4 }), + df_data_dict: { all_stats: schemaStats() }, + }); + ws.deliver(update(4, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, gen: 4, computed_columns: ["b"] }); + }); + + it("a frame that replaced the stats under the same gen takes the earlier replies' columns with it", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ + type: "initial_state", + df_meta: metaFor(policyStats), + df_data_dict: { all_stats: schemaStats() }, + }); + ws.deliver(update(3, [row("min", { b: 2 })], { tier: "scalar", status: "not_computed" })); + await settle(); + expect(model.get("df_meta").stats.computed_columns).toEqual(["b"]); + }); + + it("a refusal after replies that filled columns names them too", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, computed_columns: ["a"] }); + }); + + it("an error does not name columns: the session is not a not computed one", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("min", { a: 1 })], { tier: "scalar", final: false })); + await settle(); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "error" }); + expect(model.get("df_meta").stats).toEqual({ ...policyStats, status: "error", reason: "stats_failed" }); + }); + + it("a final update with no status still completes the session and drops the policy fields", async () => { + const { ws, model } = makePolicyModel(); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + }); + + it("is ignored for a gen the client has left", async () => { + const { ws, model, events } = makePolicyModel(); + ws.deliver({ type: "stats_update", stats_gen: 2, scope: "raw", final: true, status: "not_computed", reason: "ceiling" }); + await settle(); + expect(model.get("df_meta").stats.reason).toBe("size"); + expect(events).toEqual([]); + }); +}); + +describe("stats_gen", () => { + it("drops a stats_update whose stats_gen is not the expected one", async () => { + const { ws, model, events } = makeModel(3); + ws.deliver(update(2, [row("mean", { a: 2 })])); + ws.deliver(update(3, [row("max", { a: 9 })])); + await settle(); + expect(model.get("df_data_dict").all_stats.map((r: any) => r.index)).toEqual(["dtype", "length", "max"]); + expect(events.filter((e) => e.event === "change:df_data_dict")).toHaveLength(1); + }); + + it("drops every stats_update when the server reported no stats", async () => { + const { ws, model, events } = makeModel(null); + ws.deliver(update(0, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + expect(events).toHaveLength(0); + }); + + it("clears the expectation when an initial_state carries no df_meta.stats", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(undefined)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + }); + + it("advances the expected gen on a broadcast initial_state with no reply_seq", async () => { + const { ws, model } = makeModel(3); + ws.deliver(frame(4)); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(2); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toHaveLength(3); + }); + + it("discards a merge when an initial_state for a newer gen arrives while it decodes", async () => { + const { ws, model } = makeModel(3); + ws.deliver(update(3, [row("mean", { a: 2 })])); + // Still decoding: the model has not merged anything yet. + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + ws.deliver(update(4, [row("mean", { a: 2 })])); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" }), row("mean", { a: 2 })]); + }); + + it("merges onto the new dict when a same-gen initial_state replaces it while the old one decodes", async () => { + let release: () => void = () => {}; + gate = new Promise((resolve) => { release = resolve; }); + try { + const held = { format: "json", layout: "wide", data: schemaStats(), hold: true }; + const { ws, model } = makeModel(3, held); + ws.deliver(update(3, [row("mean", { a: 2 })])); + await settle(); // the update is now waiting on the held decode + expect((decodeDFData as jest.Mock).mock.calls.some(([env]) => env === held)).toBe(true); + ws.deliver(frame(3, [row("dtype", { a: "int32" })])); + release(); + await settle(); + expect(model.get("df_data_dict").all_stats).toEqual([ + row("dtype", { a: "int32" }), + row("mean", { a: 2 }), + ]); + expect(model.get("df_meta").stats).toEqual({ status: "complete", tier: "full", gen: 3 }); + } finally { + release(); + gate = Promise.resolve(); + } + }); +}); + +describe("stats_aborted", () => { + it("marks the stats failed when the run for the expected gen failed", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats).toEqual({ status: "error", tier: "schema", gen: 3, reason: "stats_failed" }); + }); + + it("marks the stats not computed when the server says they cannot be requested", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 3, scope: "raw", reason: "not_requestable" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("not_computed"); + }); + + it("changes nothing for a stale reply, an unsupported scope or a session with no data", async () => { + const { ws, events } = makeModel(3); + for (const reason of ["stale", "unsupported_scope", "no_data"]) { + ws.deliver({ type: "stats_aborted", stats_gen: 3, current_gen: 4, scope: "raw", reason }); + } + await settle(); + expect(events).toHaveLength(0); + }); + + it("ignores a reply for a gen the client has left", async () => { + const { ws, model } = makeModel(3); + ws.deliver({ type: "stats_aborted", stats_gen: 2, current_gen: 3, scope: "raw", reason: "error" }); + await settle(); + expect(model.get("df_meta").stats.status).toBe("pending"); + }); +}); + +describe("other messages", () => { + it("ignores a type it does not know and keeps applying frames", async () => { + const { ws, model, events } = makeModel(3); + expect(() => ws.deliver({ type: "something_new", stats_gen: 3 })).not.toThrow(); + ws.deliver({ type: "error", message: "boom" }); + await settle(); + expect(events).toHaveLength(0); + ws.deliver(frame(4, [row("dtype", { a: "int32" })])); + expect(model.get("df_data_dict").all_stats).toEqual([row("dtype", { a: "int32" })]); + expect(model.get("df_meta").stats).toEqual(pending(4)); + }); + + it("still pairs an infinite_resp with the binary frame that follows it", () => { + const { ws, model } = makeModel(3); + const seen: [any, DataView[]][] = []; + model.on("msg:custom", (msg: any, buffers: DataView[]) => seen.push([msg, buffers])); + ws.deliver({ type: "infinite_resp", key: { start: 0, end: 5 }, length: 5 }); + ws.onmessage?.({ data: new ArrayBuffer(8) } as MessageEvent); + expect(seen).toHaveLength(1); + expect(seen[0][0].type).toBe("infinite_resp"); + expect(seen[0][1][0].byteLength).toBe(8); + }); +}); diff --git a/packages/buckaroo-js-core/src/server/StatsChannel.ts b/packages/buckaroo-js-core/src/server/StatsChannel.ts new file mode 100644 index 000000000..cca73245e --- /dev/null +++ b/packages/buckaroo-js-core/src/server/StatsChannel.ts @@ -0,0 +1,309 @@ +/** + * StatsChannel — the client half of the stats wire (rows-first c2). + * + * A client that advertises `?caps=stats_update` gets a first `initial_state` + * whose `df_meta.stats` says the stats are pending, asks for them with + * `stats_request {stats_gen, scope}`, and receives either + * + * stats_update {stats_gen, scope, tier, final, payload, elapsed_ms} + * stats_aborted {stats_gen, current_gen?, scope, reason} + * + * A final `stats_update` may also carry `status` and `reason` (rows-first c5): + * `{final: true, status: "not_computed", reason: "ceiling"}`, with no payload, + * answers a request the server refused, and the session is left in that state. + * Without a `status` a final update completes the session. A run for some + * columns ends the same way, with a payload, and the columns its replies filled + * go in `df_meta.stats.computed_columns`, which the summary view reads to show + * what the run computed. + * + * `payload` is an inline wide DFEnvelope holding `all_stats`. `stats_gen` is the + * server's counter for the state the stats describe; it rides on every + * `initial_state` as `df_meta.stats.gen`, and a reply for any other gen is for + * a state the client has left. Sending the request is the scheduler's job, not + * this module's. + * + * Merge semantics are WebSocket-only: `WebSocketModel` hands every frame to + * `handle()` first, and Jupyter's widget sets `df_data_dict` whole. + */ +import { decodeDFData } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; +import { DFMeta, DFMetaStats, StatsStatus } from "../components/WidgetTypes"; +import { IModel } from "./IModel"; + +/** A capability this client advertises, as one value of `?caps=` on the + * WebSocket URL: it merges `stats_update` messages. The server records it per + * connection when the socket opens, since it sends the first message before + * the client can say anything. */ +export const STATS_UPDATE_CAP = "stats_update"; + +/** The second capability: this client can show a session whose stats are not + * computed, with the reason, and send tiered requests (see forceStats). A + * server applies its stats policy only to a client that has it. */ +export const STATS_ONDEMAND_CAP = "stats_ondemand"; + +const CLIENT_CAPS = [STATS_UPDATE_CAP, STATS_ONDEMAND_CAP]; + +const decodeQueryValue = (value: string): string => { + try { + return decodeURIComponent(value); + } catch { + return value; + } +}; + +/** `wsUrl` with `caps=stats_update,stats_ondemand` added: a new query on a bare + * URL, a new parameter after an existing query, or a comma-joined value when + * the host already passes `caps`, to which only the capabilities it lacks are + * added. The fragment stays last and other parameters are left as the host + * wrote them. */ +export function withStatsCapability(wsUrl: string): string { + const hashAt = wsUrl.indexOf("#"); + const fragment = hashAt === -1 ? "" : wsUrl.slice(hashAt); + const beforeFragment = hashAt === -1 ? wsUrl : wsUrl.slice(0, hashAt); + const queryAt = beforeFragment.indexOf("?"); + const path = queryAt === -1 ? beforeFragment : beforeFragment.slice(0, queryAt); + const params = queryAt === -1 ? [] : beforeFragment.slice(queryAt + 1).split("&").filter((p) => p !== ""); + + const capsAt = params.findIndex((p) => p === "caps" || p.startsWith("caps=")); + if (capsAt === -1) { + params.push(`caps=${CLIENT_CAPS.join(",")}`); + } else { + const caps = decodeQueryValue(params[capsAt].slice("caps=".length)) + .split(",") + .map((cap) => cap.trim()) + .filter((cap) => cap !== ""); + for (const cap of CLIENT_CAPS) { + if (!caps.includes(cap)) caps.push(cap); + } + params[capsAt] = `caps=${caps.join(",")}`; + } + return `${path}?${params.join("&")}${fragment}`; +} + +export interface StatsUpdateMessage { + type: "stats_update"; + stats_gen: number; + scope?: string; + tier?: string; + final?: boolean; + // A final reply that did not run, or ran for some columns only, says where + // the stats stand. Absent means "complete". + status?: StatsStatus; + reason?: string; + payload?: DFDataOrPayload; + elapsed_ms?: number; +} + +export interface StatsAbortedMessage { + type: "stats_aborted"; + /** The request's gen. */ + stats_gen?: number; + /** The session's gen, omitted when the session has no data. */ + current_gen?: number; + scope?: string; + reason?: "stale" | "unsupported_scope" | "not_requestable" | "error" | "no_data"; +} + +/** + * Key-merge a stats payload into `all_stats`. Both are row-per-stat tables + * (`{index: , : , ...}`). For each stat row the payload + * names, every column it carries replaces that cell, columns it does not carry + * keep theirs, and a stat the table lacks is appended. + * + * A `null` in the payload never replaces a value already merged: the wide + * pivot fills a stat a column did not carry in this message with `null`. + * + * Returns new row objects and a new array. `base` may be the decoder's cached + * array, so nothing in it is touched. + */ +export function mergeStatRows(base: DFData, update: DFData): DFData { + const merged = base.slice(); + const at = new Map(); + merged.forEach((row, i) => at.set(row.index, i)); + + for (const updateRow of update) { + const i = at.get(updateRow.index); + if (i === undefined) { + at.set(updateRow.index, merged.length); + merged.push({ ...updateRow }); + continue; + } + const row = { ...merged[i] }; + for (const [column, value] of Object.entries(updateRow)) { + if (column === "index" || column === "level_0") continue; + if (value === null && row[column] != null) continue; + row[column] = value; + } + merged[i] = row; + } + return merged; +} + +const genOf = (meta: DFMeta | undefined): number | undefined => { + const gen = meta?.stats?.gen; + return typeof gen === "number" ? gen : undefined; +}; + +/** The columns of `rows` that hold a value in some stat row. The wide pivot + * fills a stat a column did not carry in a message with null, so a null says + * nothing about the column. */ +function columnsWithValues(rows: DFData): string[] { + const columns = new Set(); + for (const row of rows) { + for (const [column, value] of Object.entries(row)) { + if (column !== "index" && column !== "level_0" && value != null) columns.add(column); + } + } + return Array.from(columns); +} + +/** `stats` with `columns` added to its `computed_columns`, or `stats` itself + * when there is nothing to add. */ +function withComputedColumns(stats: DFMetaStats, columns: string[]): DFMetaStats { + const all = Array.from(new Set([...(stats.computed_columns ?? []), ...columns])); + return all.length === 0 ? stats : { ...stats, computed_columns: all }; +} + +/** + * `df_meta.stats` after a final update. With no status it is complete at the + * update's tier, and the policy fields, which only describe what is left to + * ask for, go. With a status (a refusal, or a run for some columns only) the + * session keeps its stats and takes the status and the reason, if the update + * names one; the update's tier is what was asked for, not what was reached. + * `filled` lists the columns the run's replies merged, and a session left not + * computed adds them to `computed_columns`. + */ +function finalStats(stats: DFMetaStats, msg: StatsUpdateMessage, filled: string[]): DFMetaStats { + const status = msg.status ?? "complete"; + if (status === "complete") return { status, tier: msg.tier ?? stats.tier, gen: stats.gen }; + return withComputedColumns( + { ...stats, status, ...(msg.reason === undefined ? {} : { reason: msg.reason }) }, + filled, + ); +} + +/** What the channel needs of a model. */ +type StatsModel = Pick; + +export class StatsChannel { + // Updates are applied one at a time: each reads the dict the previous one + // wrote, so a later update cannot overwrite an earlier one's merge. + private applying: Promise = Promise.resolve(); + + // The columns the replies of the run in progress filled, which its final + // reply puts in df_meta.stats.computed_columns when it leaves the session + // not computed. They are for one gen and for the df_data_dict this channel + // last wrote: a new gen, or a frame that replaced the dict, took what the + // run had merged, so the columns start over. + private filled = new Set(); + private filledGen: number | undefined; + private filledDict: unknown; + + constructor(private model: StatsModel) {} + + /** + * The stats_gen of the state the client is showing, or `undefined` when the + * server reports none; a `stats_request` carries it. It is read off the + * model's `df_meta` each time, so it starts from the frame the model was + * built from and every applied `initial_state` (a broadcast frame with no + * `reply_seq` included) moves it. A `df_meta` with no `stats.gen` leaves + * nothing expected, so a server that stops reporting stats cannot have a + * late `stats_update` merged. Reading the model, not the message, keeps this + * correct under any scheme that drops stale `initial_state` frames. + */ + get expectedGen(): number | undefined { + return genOf(this.model.get("df_meta")); + } + + /** Consume a stats message. Returns false for every other message type, + * which the model handles (or ignores) as before. */ + handle(msg: { type?: string }): boolean { + if (msg.type === "stats_update") { + this.receiveUpdate(msg as StatsUpdateMessage); + return true; + } + if (msg.type === "stats_aborted") { + this.receiveAborted(msg as StatsAbortedMessage); + return true; + } + return false; + } + + private receiveUpdate(msg: StatsUpdateMessage): void { + if (msg.stats_gen !== this.expectedGen) return; + this.applying = this.applying + .then(() => this.applyUpdate(msg)) + .catch((e) => console.error("[StatsChannel] stats_update failed:", e)); + } + + private async applyUpdate(msg: StatsUpdateMessage): Promise { + // A reply that did not run (a request over the ceiling) has no payload + // and nothing to merge. + const update = msg.payload === undefined ? undefined : await decodeDFData(msg.payload); + for (;;) { + // A frame may have moved the gen on while something decoded. + if (msg.stats_gen !== this.expectedGen) return; + if (update !== undefined) { + const dict: Record | null | undefined = this.model.get("df_data_dict"); + // The dict is decoded when the seed built it and raw when a later + // initial_state did. + const base = await decodeDFData(dict?.all_stats); + // A frame replaced the dict (or moved the gen) while it decoded: + // start over from the new one. + if (dict !== this.model.get("df_data_dict") || msg.stats_gen !== this.expectedGen) continue; + this.model.set("df_data_dict", { ...dict, all_stats: mergeStatRows(base, update) }); + this.noteFilled(msg.stats_gen, dict, update); + } + if (msg.final) { + const filled = this.takeFilled(msg.stats_gen); + this.replaceStats((stats) => finalStats(stats, msg, filled)); + } + return; + } + } + + private receiveAborted(msg: StatsAbortedMessage): void { + // Only a reply to a request for the state on screen says anything about + // it. A `stale` reply is answered by the initial_state that carries the + // new gen; taking `current_gen` without that frame would merge stats + // for a state the client has not seen into the one it shows. + if (msg.stats_gen !== this.expectedGen) return; + const filled = this.takeFilled(msg.stats_gen); + if (msg.reason === "error") { + this.replaceStats((stats) => ({ ...stats, status: "error", reason: "stats_failed" })); + } else if (msg.reason === "not_requestable") { + this.replaceStats((stats) => withComputedColumns({ ...stats, status: "not_computed" }, filled)); + } + } + + /** Note the columns `update` filled in the dict this channel has just + * written over `before`. */ + private noteFilled(gen: number, before: unknown, update: DFData): void { + if (this.filledGen !== gen || (this.filledDict !== undefined && this.filledDict !== before)) { + this.filled = new Set(); + } + this.filledGen = gen; + for (const column of columnsWithValues(update)) this.filled.add(column); + this.filledDict = this.model.get("df_data_dict"); + } + + /** The columns the run's replies filled, as of the reply that ends it: + * none when the gen or the dict has changed since. The next run starts + * from nothing. */ + private takeFilled(gen: number | undefined): string[] { + const current = this.filledGen === gen && this.filledDict === this.model.get("df_data_dict"); + const columns = current ? Array.from(this.filled) : []; + this.filled = new Set(); + this.filledGen = undefined; + this.filledDict = undefined; + return columns; + } + + /** Replace `df_meta.stats` in a new `df_meta`, the reference that c0a's + * `inFlight` rule and the pinned rows' placeholders key on. */ + private replaceStats(change: (stats: DFMetaStats) => DFMetaStats): void { + const meta: DFMeta | undefined = this.model.get("df_meta"); + if (!meta?.stats) return; + this.model.set("df_meta", { ...meta, stats: change(meta.stats) }); + } +} diff --git a/packages/buckaroo-js-core/src/server/WebSocketModel.ts b/packages/buckaroo-js-core/src/server/WebSocketModel.ts index 1e1fd4a2d..0d249081d 100644 --- a/packages/buckaroo-js-core/src/server/WebSocketModel.ts +++ b/packages/buckaroo-js-core/src/server/WebSocketModel.ts @@ -13,21 +13,32 @@ * Binary protocol (matching anywidget's msg + buffers pattern): * Server sends a JSON text frame (infinite_resp), then a binary frame (Parquet). * This class pairs them and emits "msg:custom" with (msg, [DataView]). + * + * stats_update and stats_aborted frames go to `stats` (see StatsChannel), and + * `scheduler` asks for the stats a session defers (see StateOrchestrator). */ +import { StatsChannel } from "./StatsChannel"; +import { StateOrchestrator } from "./StateOrchestrator"; + export class WebSocketModel { private ws: WebSocket; private pendingMsg: any = null; private handlers: Map> = new Map(); private state: Record; private pendingChanges: Set = new Set(); + readonly stats: StatsChannel; + readonly scheduler: StateOrchestrator; constructor(ws: WebSocket, initialState: Record) { this.state = { ...initialState }; this.ws = ws; + this.stats = new StatsChannel(this); + this.scheduler = new StateOrchestrator({ model: this }); this.ws.onmessage = (event: MessageEvent) => { if (typeof event.data === "string") { const msg = JSON.parse(event.data); + if (this.stats.handle(msg)) return; if (msg.type === "infinite_resp") { // Expect a following binary frame — stash this JSON @@ -60,6 +71,9 @@ export class WebSocketModel { } } }; + + // Idle unless df_meta.stats says pending, which no default session does. + this.scheduler.start(); } send(msg: any): void { diff --git a/packages/buckaroo-js-core/src/server/latestDictDecoder.ts b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts new file mode 100644 index 000000000..16c4c3ef9 --- /dev/null +++ b/packages/buckaroo-js-core/src/server/latestDictDecoder.ts @@ -0,0 +1,34 @@ +import { decodeDFDataDict } from "../components/DFViewerParts/resolveDFData"; +import { DFData, DFDataOrPayload } from "../components/DFViewerParts/DFWhole"; + +export type RawDFDataDict = Record | undefined | null; + +/** + * Decode df_data_dict values arriving from a model and hand `apply` only the + * newest result. + * + * A full `initial_state` reaches a view twice, as `change:df_data_dict` and + * then as `metadata`, and both handlers read the same dict off the model. The + * returned function ignores a dict it has already seen (same reference), so + * that frame decodes once. Decoding is async, so a slow decode of an earlier + * dict can finish after a later one; each call takes a token, and a result is + * applied only if no later call has started. + * + * `seen` is a dict the caller has already applied (the view's seed), so the + * first read of that same dict off the model does not decode it again. + */ +export function makeLatestDictDecoder( + apply: (decoded: Record) => void, + seen?: RawDFDataDict, +): (raw: RawDFDataDict) => void { + let latest = 0; + let lastRaw: RawDFDataDict = seen; + return (raw) => { + if (raw === lastRaw) return; + lastRaw = raw; + const mine = ++latest; + decodeDFDataDict(raw).then((decoded) => { + if (mine === latest) apply(decoded); + }); + }; +} diff --git a/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx new file mode 100644 index 000000000..22e3e0c44 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsPendingPinnedRows.stories.tsx @@ -0,0 +1,128 @@ +/** + * Story for the two-message protocol's client states (rows-first c0a). + * + * The server sends the first message once the schema and row count are known, + * and the summary stats later or never. `df_meta.stats.status` says which: + * + * - "pending" pinned keys with no value show a placeholder row + * - "not_computed" pinned keys with no value are omitted + * - "complete" stats are present; also the meaning of a missing + * `df_meta.stats`, as servers without the field send + * + * Column `a` is color-mapped, so the story also shows that its cells restyle + * when the histogram bins arrive. Used by stats-pending-pinned-rows.spec.ts. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import { DFViewerInfiniteDS } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { DFMeta } from "../components/WidgetTypes"; + +type Status = "pending" | "not_computed" | "complete"; +const STATUSES: Status[] = ["pending", "not_computed", "complete"]; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, + { index: 3, a: 4, b: "w" }, + { index: 4, a: 5, b: "v" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 3, b: "N/A" }, + { index: "histogram_bins", a: [0, 1, 2, 3, 4, 5], b: [] }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { + col_name: "a", + header_name: "a", + displayer_args: { displayer: "obj" }, + color_map_config: { color_rule: "color_map", map_name: "BLUE_TO_YELLOW", val_column: "a" }, + }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const StatsPendingPinnedRowsInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { status }, + }) as DFMeta, + [status], + ); + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : ([] as DFData), + empty: [] as DFData, + }), + [status], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + df_meta.stats.status = {status} +
+
+ +
+
+ ); +}; + +const meta = { + title: "Buckaroo/DFViewer/StatsPendingPinnedRows", + component: StatsPendingPinnedRowsInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; diff --git a/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx new file mode 100644 index 000000000..4f63c9ca3 --- /dev/null +++ b/packages/buckaroo-js-core/src/stories/StatsSchedulerStates.stories.tsx @@ -0,0 +1,386 @@ +/** + * Story for the client states of the stats wire (rows-first c4). + * + * df_meta.stats.status says where the summary stats stand, and the status bar + * shows it in a column of its own: + * + * - "pending" "Computing summary stats", pinned keys show placeholders + * - "not_computed" a "Compute summary stats" button, pinned keys omitted + * - "error" "Stats error: ", pinned keys omitted + * - "complete" "Summary stats ready", the values in place + * + * The buttons switch the status. The "partial" button is "pending" with the + * stats of one column in, as a partial stats_update leaves them. The button in + * the status bar sends `stats_request {force: true, incremental: true, tier}` + * through a fake model, which logs what it was asked to send and answers + * `visible_columns` with the columns the grid reports. Used by + * stats-scheduler-states.spec.ts. + * + * The NotComputed story (rows-first c5) is a session the server's policy left + * without stats: typed columns over schema-tier stats, a main and a summary + * view, and the policy in df_meta.stats. The buttons switch the view and the + * reason the stats were not computed (which also puts the session back as the + * server described it). The controls send through `forceStats`, which marks the + * stats pending on the model, as it does for a real session. + */ +import type { Meta, StoryObj } from "@storybook/react"; +import React, { useMemo, useState } from "react"; +import "../style/dcf-npm.css"; +import { BuckarooInfiniteWidget } from "../components/BuckarooWidgetInfinite"; +import { DFData, DFViewerConfig } from "../components/DFViewerParts/DFWhole"; +import { IDisplayArgs } from "../components/DFViewerParts/gridUtils"; +import { KeyAwareSmartRowCache, PayloadResponse } from "../components/DFViewerParts/SmartRowCache"; +import { BuckarooOptions, BuckarooState, DFMeta, StatsStatus } from "../components/WidgetTypes"; +import { CommandConfigT } from "../components/CommandUtils"; +import { Operation } from "../components/OperationUtils"; +import { baseOperationResults } from "../components/DependentTabs"; +import { forceStats } from "../server/StateOrchestrator"; + +const STATUSES: StatsStatus[] = ["pending", "not_computed", "error", "complete"]; +const GEN = 7; + +const mainData: DFData = [ + { index: 0, a: 1, b: "x" }, + { index: 1, a: 2, b: "y" }, + { index: 2, a: 3, b: "z" }, +]; + +const completeStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 2, b: "N/A" }, +]; + +// dtype comes with the schema; mean has arrived for column a only. +const partialStats: DFData = [ + { index: "dtype", a: "int64", b: "object" }, + { index: "mean", a: 2 }, +]; + +const viewerConfig: DFViewerConfig = { + column_config: [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "obj" } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "obj" } }, + ], + left_col_configs: [{ col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }], + pinned_rows: [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "mean", displayer_args: { displayer: "obj" } }, + ], +}; + +const displayArgs: Record = { + main: { data_key: "main", df_viewer_config: viewerConfig, summary_stats_key: "all_stats" }, +}; + +const buckarooOptions: BuckarooOptions = { + sampled: [], + cleaning_method: [], + post_processing: [], + df_display: ["main"], + show_commands: [], +}; +const commandConfig: CommandConfigT = { argspecs: {}, defaultArgs: {} }; + +const StatsSchedulerStatesInner: React.FC = () => { + const [status, setStatus] = useState("pending"); + const [partial, setPartial] = useState(false); + const [visibleColumns, setVisibleColumns] = useState([]); + const [sent, setSent] = useState([]); + const [buckarooState, setBuckarooState] = useState({ + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, + }); + const [operations, setOperations] = useState([]); + + const df_meta = useMemo( + () => + ({ + total_rows: mainData.length, + columns: 2, + filtered_rows: mainData.length, + rows_shown: mainData.length, + stats: { + status, + tier: status === "complete" ? "full" : "schema", + gen: GEN, + ...(status === "error" ? { reason: "stats_failed" } : {}), + }, + }) as DFMeta, + [status], + ); + + // The model the control sends through: it answers get("df_meta") and + // get("visible_columns") from the story's state and logs what it is asked to + // send. + const model = useMemo( + () => ({ + get: (key: string) => (key === "df_meta" ? df_meta : key === "visible_columns" ? visibleColumns : undefined), + set: () => {}, + send: (msg: unknown) => setSent((log) => [...log, msg]), + }), + [df_meta, visibleColumns], + ); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: mainData.slice(pa.start, Math.min(pa.end, mainData.length)), + length: mainData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_data_dict = useMemo( + () => ({ + main: [] as DFData, + all_stats: status === "complete" ? completeStats : partial ? partialStats : ([] as DFData), + empty: [] as DFData, + }), + [status, partial], + ); + + return ( +
+
+ {STATUSES.map((s) => ( + + ))} + + df_meta.stats.status = {status} +
+
+ forceStats(model, opts)} + on_visible_columns={setVisibleColumns} + /> +
+ 
+        {JSON.stringify(sent)}
+
+
+ ); +}; + +const meta = { + title: "Buckaroo/StatsSchedulerStates", + component: StatsSchedulerStatesInner, + parameters: { layout: "centered" }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Primary: Story = {}; + + +// ---- NotComputed: a session whose stats the server's policy did not compute ---- + +type Reason = "size" | "ceiling" | "cost"; +const REASONS: Record = { + size: { reason: "size", requestable: ["scalar", "full"] }, + ceiling: { reason: "ceiling", requestable: [] }, + cost: { reason: "cost", requestable: ["full"] }, +}; + +const typedData: DFData = [ + { index: 0, a: 1, b: "x", c: 1.5 }, + { index: 1, a: 2, b: "y", c: 2.25 }, + { index: 2, a: 3, b: "z", c: 3 }, +]; + +// What the schema tier sends: dtype and identity keys, nothing computed. +const schemaStats: DFData = [ + { index: "dtype", a: "int64", b: "object", c: "float64" }, + { index: "length", a: 3, b: 3, c: 3 }, +]; + +const typedColumns: DFViewerConfig["column_config"] = [ + { col_name: "a", header_name: "a", displayer_args: { displayer: "float", min_fraction_digits: 0, max_fraction_digits: 0 } }, + { col_name: "b", header_name: "b", displayer_args: { displayer: "string", max_length: 35 } }, + { col_name: "c", header_name: "c", displayer_args: { displayer: "float", min_fraction_digits: 3, max_fraction_digits: 3 } }, +]; +const indexColumn: DFViewerConfig["left_col_configs"] = [ + { col_name: "index", header_name: "index", displayer_args: { displayer: "obj" } }, +]; +// dtype is in the schema stats; histogram is not, and no value is coming. +const typedPinned: DFViewerConfig["pinned_rows"] = [ + { primary_key_val: "dtype", displayer_args: { displayer: "obj" } }, + { primary_key_val: "histogram", displayer_args: { displayer: "histogram" } }, +]; + +const notComputedDisplayArgs: Record = { + main: { + data_key: "main", + df_viewer_config: { column_config: typedColumns, left_col_configs: indexColumn, pinned_rows: typedPinned }, + summary_stats_key: "all_stats", + }, + summary: { + data_key: "empty", + df_viewer_config: { + column_config: typedColumns, + left_col_configs: indexColumn, + pinned_rows: [{ primary_key_val: "dtype", displayer_args: { displayer: "obj" } }], + }, + summary_stats_key: "all_stats", + }, +}; + +const notComputedOptions: BuckarooOptions = { ...buckarooOptions, df_display: ["main", "summary"] }; + +const NotComputedInner: React.FC = () => { + const [reason, setReason] = useState("size"); + const [sent, setSent] = useState([]); + const [buckarooState, setBuckarooState] = useState({ + sampled: false, + cleaning_method: false, + quick_command_args: {}, + post_processing: false, + df_display: "main", + show_commands: false, + }); + const [operations, setOperations] = useState([]); + + // What the server said for the reason picked, and what the page made of it + // since: forceStats marks the stats pending on the model, as a real session + // does, and the story shows that. + const [override, setOverride] = useState(undefined); + const serverMeta = useMemo( + () => + ({ + total_rows: typedData.length, + columns: 3, + filtered_rows: typedData.length, + rows_shown: typedData.length, + stats: { + status: "not_computed", + tier: "schema", + gen: GEN, + reason: REASONS[reason].reason, + tier_target: "schema", + estimate: { rows: 12_400_000, cols: 3 }, + auto_request: false, + requestable: REASONS[reason].requestable, + }, + }) as DFMeta, + [reason], + ); + const df_meta = override ?? serverMeta; + + // The model the controls send through: it answers get("df_meta") from the + // story's state, keeps what is set on it and logs what it is asked to send. + const model = useMemo( + () => ({ + get: (key: string) => (key === "df_meta" ? df_meta : undefined), + set: (key: string, value: unknown) => { + if (key === "df_meta") setOverride(value as DFMeta); + }, + send: (msg: unknown) => setSent((log) => [...log, msg]), + }), + [df_meta], + ); + + const src = useMemo(() => { + const cache = new KeyAwareSmartRowCache((pa) => { + const resp: PayloadResponse = { + key: pa, + data: typedData.slice(pa.start, Math.min(pa.end, typedData.length)), + length: typedData.length, + }; + setTimeout(() => cache.addPayloadResponse(resp), 10); + }); + return cache; + }, []); + + const df_data_dict = useMemo( + () => ({ main: [] as DFData, all_stats: schemaStats, empty: [] as DFData }), + [], + ); + + const setView = (view: string) => setBuckarooState((state) => ({ ...state, df_display: view })); + + return ( +
+
+ {["main", "summary"].map((view) => ( + + ))} + {(Object.keys(REASONS) as Reason[]).map((r) => ( + + ))} + reason = {reason} +
+
+ forceStats(model, opts)} + /> +
+ 
+        {JSON.stringify(sent)}
+
+
+ ); +}; + +export const NotComputed: Story = { + render: () => , +}; diff --git a/packages/buckaroo-js-core/src/style/dcf-npm.css b/packages/buckaroo-js-core/src/style/dcf-npm.css index d64c928f5..93994c79b 100644 --- a/packages/buckaroo-js-core/src/style/dcf-npm.css +++ b/packages/buckaroo-js-core/src/style/dcf-npm.css @@ -435,9 +435,67 @@ div.dependent-tabs ul.tabs li.active { @media (prefers-reduced-motion: reduce) { .bk-status-inflight-dot { animation: none; opacity: 1; } } +/* Summary stats status in the status bar (rows-first c4). One line in a + * fixed-width column, whatever the status, so a status change moves nothing. + * The status bar's row is about 20px high, so the control has to fit inside it + * with room to spare: a button that overflows the row makes the browser scroll + * the grid to reveal it, and the grid's scrollbar overlay then covers the + * button and takes its clicks. */ +.bk-stats-status { + display: flex; + align-items: center; + height: 100%; + gap: 6px; + max-width: 100%; + font-size: 12px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.bk-stats-status button { + font-size: 11px; + line-height: 12px; + padding: 0 6px; + cursor: pointer; +} +/* The summary view while the stats are not computed (rows-first c5): the reason, + * and the control that asks for them, in place of the grid. */ +.bk-stats-empty { + box-sizing: border-box; + min-height: 120px; + padding: 16px; + font-size: 13px; + border: 1px solid rgba(128, 128, 128, 0.4); + border-top: none; +} +.bk-stats-empty-message { + margin: 0 0 12px; +} +.bk-stats-empty-control { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} +.bk-stats-empty-control select, +.bk-stats-empty-control button { + font-size: 12px; + padding: 2px 8px; +} +.bk-stats-empty-control button { + cursor: pointer; +} .status-bar .ag-center-cols-viewport { min-height: unset !important; } +/* The status bar is a single row, so the grid's overlay scrollbar (positioned + * over the last 16px of the grid, and not drawn) covers most of it. It sits + * above the cells while the grid is scrolling and takes the clicks meant for + * the control. The strip still scrolls with the wheel or a trackpad, and a + * scrollbar the browser draws in its own space (not an overlay) is left alone. */ +.status-bar .ag-body-horizontal-scroll.ag-scrollbar-invisible { + pointer-events: none; +} .status-bar .ag-root-wrapper { /* the status-bar is always displayed with the df-viewer */ diff --git a/packages/js/standalone.tsx b/packages/js/standalone.tsx index 715312832..29f982316 100644 --- a/packages/js/standalone.tsx +++ b/packages/js/standalone.tsx @@ -75,28 +75,57 @@ function updateFilenameDisplay(metadata: any, prompt?: string) { } } +/** What the model held for `keys` when a component first rendered. */ +function snapshotModel(model: WebSocketModel, keys: string[]): Record { + return Object.fromEntries(keys.map((k) => [k, model.get(k)])); +} + +/** A change:* emitted between the first render and the effect that registers + * the listeners had no listener. The model holds the latest value of every + * key, so apply each one that differs from what the first render read. */ +function catchUpOnModel( + model: WebSocketModel, + seed: Record, + appliers: Record void>, +) { + for (const [key, apply] of Object.entries(appliers)) { + const v = model.get(key); + if (v !== undefined && v !== seed[key]) apply(v); + } +} + function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { const [dfMeta, setDfMeta] = React.useState(model.get("df_meta") || { total_rows: 0 }); const [dfDataDict, setDfDataDict] = React.useState(model.get("df_data_dict") || {}); const [dfDisplayArgs, setDfDisplayArgs] = React.useState(patchDisplayArgsHeight(model.get("df_display_args") || {})); + const seed = React.useRef(snapshotModel(model, ["df_meta", "df_data_dict", "df_display_args"])); + // A full frame reaches the view as change:df_data_dict and again as + // metadata; this decodes each dict once and applies only the newest decode. + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: metadata.rows || 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); }; model.on("metadata", onMeta); const onDfMeta = (v: any) => setDfMeta(v); - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); model.on("change:df_meta", onDfMeta); model.on("change:df_data_dict", onDfDataDict); model.on("change:df_display_args", onDfDisplayArgs); + catchUpOnModel(model, seed.current, { + df_meta: onDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + }); // Catch up on metadata that arrived before useEffect registered listeners const existingMeta = model.get("metadata"); @@ -110,7 +139,7 @@ function ViewerApp({ model, src }: { model: WebSocketModel; src: any }) { model.off("change:df_data_dict", onDfDataDict); model.off("change:df_display_args", onDfDisplayArgs); }; - }, [model]); + }, [model, loadDict]); if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
@@ -140,12 +169,20 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { const [commandConfig, setCommandConfig] = React.useState(model.get("command_config") || {}); const [operationResults, setOperationResults] = React.useState(model.get("operation_results") || {}); const [operations, setOperations] = React.useState(model.get("operations") || []); + const seed = React.useRef(snapshotModel(model, [ + "df_meta", "df_data_dict", "df_display_args", "buckaroo_state", "buckaroo_options", + "command_config", "operation_results", "operations", + ])); + const loadDict = React.useMemo( + () => srt.makeLatestDictDecoder(setDfDataDict, model.get("df_data_dict")), + [model], + ); React.useEffect(() => { const onMeta = (metadata: any, prompt?: string) => { updateFilenameDisplay(metadata, prompt); setDfMeta(model.get("df_meta") || { total_rows: 0 }); - srt.decodeDFDataDict(model.get("df_data_dict") || {}).then(setDfDataDict); + loadDict(model.get("df_data_dict")); setDfDisplayArgs(patchDisplayArgsHeight(model.get("df_display_args") || {})); setBuckarooState(model.get("buckaroo_state") || {}); setBuckarooOptions(model.get("buckaroo_options") || {}); @@ -168,9 +205,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { }; // df_data_dict needs async pre-resolution of parquet_b64 values - const onDfDataDict = (v: any) => { - srt.decodeDFDataDict(v).then(setDfDataDict); - }; + const onDfDataDict = (v: any) => loadDict(v); model.on("change:df_data_dict", onDfDataDict); const onDfDisplayArgs = (v: any) => setDfDisplayArgs(patchDisplayArgsHeight(v)); @@ -185,6 +220,16 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { ["operation_results", onChange("operation_results", setOperationResults)], ["operations", onChange("operations", setOperations)], ]; + catchUpOnModel(model, seed.current, { + df_meta: setDfMeta, + df_data_dict: onDfDataDict, + df_display_args: onDfDisplayArgs, + buckaroo_state: setBuckarooState, + buckaroo_options: setBuckarooOptions, + command_config: setCommandConfig, + operation_results: setOperationResults, + operations: setOperations, + }); return () => { model.off("metadata", onMeta); @@ -193,7 +238,7 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.off(`change:${key}`, handler); } }; - }, [model]); + }, [model, loadDict]); const onBuckarooState = React.useCallback((newState: any) => { // newState may be a value or a setter function @@ -207,6 +252,19 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { model.save_changes(); }, [model]); + // The Compute summary stats control, shown while the server reports the + // stats as not computed: the status bar's button, and the summary view's + // button and per-column form (which pass their columns). + const onComputeStats = React.useCallback((opts?: { columns?: string[] }) => { + srt.forceStats(model, opts); + }, [model]); + + // The columns the grid shows, kept on the model for stats requests to carry + // as their hint. + const onVisibleColumns = React.useCallback((columns: string[]) => { + srt.setVisibleColumns(model, columns); + }, [model]); + if (!dfDisplayArgs || !dfDisplayArgs["main"]) { return
Waiting for data... @@ -227,6 +285,8 @@ function BuckarooApp({ model, src }: { model: WebSocketModel; src: any }) { on_buckaroo_state={onBuckarooState} buckaroo_options={buckarooOptions} src={src} + on_compute_stats={onComputeStats} + on_visible_columns={onVisibleColumns} />
); @@ -242,7 +302,8 @@ async function main() { // Show connecting state rootEl.textContent = "Connecting..."; - const ws = new WebSocket(wsUrl); + // The server reads capabilities from the URL at open. + const ws = new WebSocket(srt.withStatsCapability(wsUrl)); ws.binaryType = "arraybuffer"; // Wait for connection diff --git a/scripts/test_playwright_storybook.sh b/scripts/test_playwright_storybook.sh index c0064c16d..9445be842 100755 --- a/scripts/test_playwright_storybook.sh +++ b/scripts/test_playwright_storybook.sh @@ -95,6 +95,8 @@ STORYBOOK_TESTS=( "pw-tests/transcript-replayer.spec.ts" "pw-tests/outside-params.spec.ts" "pw-tests/fitcontent-height.spec.ts" + "pw-tests/stats-pending-pinned-rows.spec.ts" + "pw-tests/stats-scheduler-states.spec.ts" # "pw-tests/example.spec.ts" # Has pre-existing failures, excluded for now )