diff --git a/.changeset/summary-sanitization-diagnostics.md b/.changeset/summary-sanitization-diagnostics.md new file mode 100644 index 000000000..5fafe6f91 --- /dev/null +++ b/.changeset/summary-sanitization-diagnostics.md @@ -0,0 +1,5 @@ +--- +"@martian-engineering/lossless-claw": patch +--- + +Log metadata when reasoning-shaped content is stripped from an otherwise nonempty summary, including retry responses. Report the response source and before/after character counts without logging reasoning or summary text. diff --git a/src/summarize.ts b/src/summarize.ts index 075f969e9..bf08561a5 100644 --- a/src/summarize.ts +++ b/src/summarize.ts @@ -492,7 +492,9 @@ function sanitizeReasoningSummaryText(value: string): { const strippedLeading = stripLeadingClosedReasoningBlocks(trimmed); const strippedStandalone = stripStandaloneReasoningBlocks(strippedLeading.text); const remainder = strippedStandalone.text; - if (!remainder) { + const droppedReasoning = strippedLeading.dropped || strippedStandalone.dropped; + // Punctuation alone is empty output, even when the host already separated reasoning. + if (!remainder || /^[\p{P}\p{Z}\p{C}]*$/u.test(remainder)) { // Either the entire body was a closed reasoning block (no summary // text after) or the input was empty — both are reasoning-only. return { summary: "", droppedReasoning: true, reasoningOnly: true }; @@ -507,7 +509,7 @@ function sanitizeReasoningSummaryText(value: string): { return { summary: remainder, - droppedReasoning: strippedLeading.dropped || strippedStandalone.dropped, + droppedReasoning, reasoningOnly: false, }; } @@ -1775,6 +1777,18 @@ export async function createLcmSummarizeFromLegacyParams(params: { break; } + const sanitizeSummaryWithDiagnostics = (value: string, source: string) => { + const sanitized = sanitizeReasoningSummaryText(value); + if (sanitized.droppedReasoning && !sanitized.reasoningOnly) { + params.deps.log.warn( + `[lcm] stripped reasoning-shaped content from summary; provider=${provider}; ` + + `model=${model}; source=${source}; ` + + `before_chars=${value.length}; after_chars=${sanitized.summary.length}`, + ); + } + return sanitized; + }; + const normalized = normalizeCompletionSummary(result.content); let summary = normalized.summary; let summarySource: "content" | "envelope" | "retry" | "fallback" = "content"; @@ -1803,7 +1817,7 @@ export async function createLcmSummarizeFromLegacyParams(params: { // output is treated as empty so the retry/fallback chain runs. if (summary) { const rawSummaryChars = summary.length; - const sanitizedSummary = sanitizeReasoningSummaryText(summary); + const sanitizedSummary = sanitizeSummaryWithDiagnostics(summary, summarySource); if (sanitizedSummary.droppedReasoning) { summary = sanitizedSummary.summary; } @@ -1823,7 +1837,7 @@ export async function createLcmSummarizeFromLegacyParams(params: { `summary_shape=reasoning-only; summary_chars=${rawSummaryChars}`, ); const recoveryNormalized = envelopeNormalized ?? normalizeCompletionEnvelope(result); - const recoveredSummary = sanitizeReasoningSummaryText(recoveryNormalized.summary); + const recoveredSummary = sanitizeSummaryWithDiagnostics(recoveryNormalized.summary, "envelope"); if (recoveredSummary.summary && !recoveredSummary.reasoningOnly) { summary = recoveredSummary.summary; envelopeNormalized = { @@ -1888,7 +1902,7 @@ export async function createLcmSummarizeFromLegacyParams(params: { // reasoning-only retry output drops to deterministic fallback. if (summary) { const rawRetrySummaryChars = summary.length; - const sanitizedRetrySummary = sanitizeReasoningSummaryText(summary); + const sanitizedRetrySummary = sanitizeSummaryWithDiagnostics(summary, "retry"); if (sanitizedRetrySummary.droppedReasoning) { summary = sanitizedRetrySummary.summary; } @@ -1899,8 +1913,9 @@ export async function createLcmSummarizeFromLegacyParams(params: { `summary_shape=reasoning-only; summary_chars=${rawRetrySummaryChars}`, ); const retryRecoveryNormalized = normalizeCompletionEnvelope(retryResult); - const retryRecoveredSummary = sanitizeReasoningSummaryText( + const retryRecoveredSummary = sanitizeSummaryWithDiagnostics( retryRecoveryNormalized.summary, + "retry", ); summary = retryRecoveredSummary.summary && !retryRecoveredSummary.reasoningOnly diff --git a/test/compaction-work-accounting.test.ts b/test/compaction-work-accounting.test.ts index cba92613a..8d940b091 100644 --- a/test/compaction-work-accounting.test.ts +++ b/test/compaction-work-accounting.test.ts @@ -5,11 +5,13 @@ import type { CompactionGuards } from "../src/compaction-guards.js"; import type { CompactionTelemetryRecorder } from "../src/compaction-telemetry.js"; import type { LcmContextEngine } from "../src/engine.js"; import type { CompactResult, ContextEngineRuntimeSettings } from "../src/openclaw-bridge.js"; -import { LcmProviderAuthError, LcmRuntimeLifecycleError, LcmRuntimeLlmPolicyError, LcmRuntimeLlmUnavailableError, LcmSummarySpendLimitError } from "../src/summarize.js"; +import { createLcmSummarizeFromLegacyParams, LcmProviderAuthError, LcmRuntimeLifecycleError, LcmRuntimeLlmPolicyError, LcmRuntimeLlmUnavailableError, LcmSummarySpendLimitError } from "../src/summarize.js"; import { cleanupEngineTestState, createEngineWithConfig, createEngineWithDeps, + createTestDeps, + getEngineConfig, seedBacklogContext, } from "./helpers.js"; @@ -62,6 +64,32 @@ function authError() { } describe("compactUntilUnder committed work", () => { + it.each([ + ["after reasoning removal", "PRIVATE_REASONING_REMNANT."], + ["bare punctuation-only", "..."], + ])("stores the recovered summary rather than punctuation: %s", async (_label, text) => { + const complete = vi.fn() + .mockResolvedValueOnce({ content: [{ type: "text", text }] }) + .mockResolvedValueOnce({ content: [{ type: "text", text: "Recovered conversation details." }] }); + const engine = createEngineWithDeps({ freshTailCount: 0, leafMinFanout: 2, leafChunkTokens: 1_000 }, { complete }); + await seedBacklogContext(engine, "punctuation-summary", [150, 150, 150, 150]); + const conversation = await engine.getConversationStore().getConversationBySessionId("punctuation-summary"); + const summarize = await createLcmSummarizeFromLegacyParams({ + deps: createTestDeps(getEngineConfig(engine), { complete }), + legacyParams: {}, + }); + const result = await accountingInternals(engine).compaction.compact({ + conversationId: conversation!.conversationId, + tokenBudget: 10_000, + summarize: summarize!.fn, + force: true, + }); + expect(result.actionTaken).toBe(true); + const stored = await engine.getSummaryStore().getSummary(result.createdSummaryId!); + expect(stored?.content).toBe("Recovered conversation details."); + expect(complete).toHaveBeenCalledTimes(2); + }); + it.each([10, 30_000])("does not infer work from observed/stored mismatch (stored %i)", async (storedTokens) => { const engine = createEngineWithConfig({ freshTailCount: 12 }); await seedBacklogContext(engine, "protected", [storedTokens]); diff --git a/test/summarize.test.ts b/test/summarize.test.ts index 005574b6a..06dd2ec5e 100644 --- a/test/summarize.test.ts +++ b/test/summarize.test.ts @@ -93,6 +93,76 @@ describe("createLcmSummarizeFromLegacyParams", () => { vi.unstubAllEnvs(); }); + describe("punctuation-only reasoning remnants", () => { + const bareResponse = (source: "content" | "envelope", text: string) => + source === "content" + ? { content: [{ type: "text", text }] } + : { content: [], output_text: text }; + + it.each(["content", "envelope"] as const)("retries bare punctuation-only %s summaries", async (source) => { + const complete = vi.fn() + .mockResolvedValueOnce(bareResponse(source, "...")) + .mockResolvedValueOnce({ content: [{ type: "text", text: "Recovered conversation summary." }] }); + const summarize = await createSummarizeFn({ deps: makeDeps({ complete }), legacyParams: {} }); + expect(await summarize!("Conversation source details", false)).toBe("Recovered conversation summary."); + expect(complete).toHaveBeenCalledTimes(2); + }); + + it.each(["content", "envelope"] as const)("falls back after bare punctuation-only retry %s summaries", async (source) => { + const complete = vi.fn() + .mockResolvedValueOnce({ content: [] }) + .mockResolvedValueOnce(bareResponse(source, "。… \u200b")); + const summarize = await createSummarizeFn({ deps: makeDeps({ complete }), legacyParams: {} }); + expect(await summarize!("Conversation source details", false)).toContain("Conversation source details"); + expect(complete).toHaveBeenCalledTimes(2); + }); + + it.each(["短摘要", "I", "42", "✅"])("preserves substantive output without reasoning alongside bare punctuation-only checks: %s", async (text) => { + const complete = vi.fn().mockResolvedValue(bareResponse("content", text)); + const summarize = await createSummarizeFn({ deps: makeDeps({ complete }), legacyParams: {} }); + expect(await summarize!("Conversation source details", false)).toBe(text); + expect(complete).toHaveBeenCalledTimes(1); + }); + + const responseFor = (source: "content" | "envelope", suffix: string) => { + const body = `PRIVATE_REASONING_REMNANT${suffix}`; + return source === "content" + ? { content: [{ type: "text", text: body }] } + : { content: [{ type: "text", text: "PRIVATE_CONTENT_REASONING" }], output_text: body }; + }; + + it.each(["content", "envelope"] as const)("retries a punctuation-only %s remnant", async (source) => { + const complete = vi.fn() + .mockResolvedValueOnce(responseFor(source, ".")) + .mockResolvedValueOnce({ content: [{ type: "text", text: "Recovered conversation summary." }] }); + const deps = makeDeps({ complete }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + expect(await summarize!("Conversation source details", false)).toBe("Recovered conversation summary."); + expect(complete).toHaveBeenCalledTimes(2); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_REASONING_REMNANT"); + }); + + it.each(["content", "envelope"] as const)("falls back when the retry %s remnant contains only Unicode punctuation and formatting", async (source) => { + const complete = vi.fn() + .mockResolvedValueOnce({ content: [] }) + .mockResolvedValueOnce(responseFor(source, "。… \u200b")); + const deps = makeDeps({ complete }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + const summary = await summarize!("Conversation source details", false); + expect(summary).toContain("Conversation source details"); + expect(summary).not.toContain("PRIVATE_REASONING_REMNANT"); + expect(complete).toHaveBeenCalledTimes(2); + }); + + it.each(["短摘要", "I", "42", "✅"])("preserves a substantive short remnant: %s", async (suffix) => { + const complete = vi.fn().mockResolvedValue(responseFor("content", suffix)); + const deps = makeDeps({ complete }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + expect(await summarize!("Conversation source details", false)).toBe(suffix); + expect(complete).toHaveBeenCalledTimes(1); + }); + }); + it.each([false, true])("does not retry providers or synthesize fallback after host scope closure (retry=%s)", async (retry) => { const complete = vi.fn().mockResolvedValue({ content: [], error: { kind: "runtime_lifecycle", message: "Async work scope is closed" }, @@ -2174,6 +2244,86 @@ describe("createLcmSummarizeFromLegacyParams", () => { }); }); + describe("summary sanitization diagnostics", () => { + it.each(["content", "envelope"] as const)( + "reports a mixed %s response rewrite without disclosing its content", + async (source) => { + const body = "PRIVATE_REASONING_PAYLOADPRIVATE_VISIBLE_SUMMARY"; + const deps = makeDeps({ + complete: vi.fn(async () => source === "content" + ? { content: [{ type: "text", text: body }] } + : { content: [], output_text: body }), + }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + + expect(await summarize!("Conversation to summarize", false)).toBe("PRIVATE_VISIBLE_SUMMARY"); + expect(deps.complete).toHaveBeenCalledTimes(1); + expect(deps.log.warn).toHaveBeenCalledExactlyOnceWith( + `[lcm] stripped reasoning-shaped content from summary; provider=anthropic; ` + + `model=claude-opus-4-5; source=${source}; ` + + `before_chars=${body.length}; after_chars=${"PRIVATE_VISIBLE_SUMMARY".length}`, + ); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_REASONING_PAYLOAD"); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_VISIBLE_SUMMARY"); + }, + ); + + it("reports a retry rewrite without disclosing the visible summary", async () => { + const body = "PRIVATE_RETRY_REASONINGVisible summary."; + const complete = vi.fn() + .mockResolvedValueOnce({ content: [] }) + .mockResolvedValueOnce({ content: [{ type: "text", text: body }] }); + const deps = makeDeps({ complete }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + + expect(await summarize!("Conversation to summarize", false)).toBe("Visible summary."); + expect(complete).toHaveBeenCalledTimes(2); + expect(deps.log.warn).toHaveBeenCalledWith( + `[lcm] stripped reasoning-shaped content from summary; provider=anthropic; ` + + `model=claude-opus-4-5; source=retry; before_chars=${body.length}; after_chars=${"Visible summary.".length}`, + ); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_RETRY_REASONING"); + }); + + it.each([false, true])("reports envelope sanitization after reasoning-only content (retry=%s)", async (retry) => { + const content = "PRIVATE_CONTENT_REASONING"; + const envelope = "PRIVATE_ENVELOPE_REASONINGVisible envelope summary."; + const response = { content: [{ type: "text", text: content }], output_text: envelope }; + const complete = vi.fn().mockResolvedValue(response); + if (retry) complete.mockResolvedValueOnce({ content: [] }); + const deps = makeDeps({ complete }); + const summarize = await createSummarizeFn({ deps, legacyParams: {} }); + + expect(await summarize!("Conversation to summarize", false)).toBe("Visible envelope summary."); + expect(complete).toHaveBeenCalledTimes(retry ? 2 : 1); + expect(deps.log.warn).toHaveBeenCalledWith( + `[lcm] stripped reasoning-shaped content from summary; provider=anthropic; ` + + `model=claude-opus-4-5; source=${retry ? "retry" : "envelope"}; ` + + `before_chars=${envelope.length}; after_chars=${"Visible envelope summary.".length}`, + ); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_CONTENT_REASONING"); + expect(getDepsLogText(deps)).not.toContain("PRIVATE_ENVELOPE_REASONING"); + }); + + it("keeps plain summaries quiet and retains the reasoning-only warning", async () => { + const plainDeps = makeDeps(); + const plainSummarize = await createSummarizeFn({ deps: plainDeps, legacyParams: {} }); + expect(await plainSummarize!("Conversation to summarize", false)).toBe("summary output"); + expect(plainDeps.log.warn).not.toHaveBeenCalled(); + + const reasoningDeps = makeDeps({ + complete: vi.fn() + .mockResolvedValueOnce({ content: [{ type: "text", text: "PRIVATE_ONLY" }] }) + .mockResolvedValueOnce({ content: [{ type: "text", text: "Recovered summary" }] }), + }); + const reasoningSummarize = await createSummarizeFn({ deps: reasoningDeps, legacyParams: {} }); + expect(await reasoningSummarize!("Conversation to summarize", false)).toBe("Recovered summary"); + expect(getDepsLogText(reasoningDeps, ["warn"])).toContain("dropped reasoning-shaped summary on first attempt"); + expect(getDepsLogText(reasoningDeps, ["warn"])).not.toContain("stripped reasoning-shaped content from summary"); + expect(getDepsLogText(reasoningDeps)).not.toContain("PRIVATE_ONLY"); + }); + }); + // --- Envelope-aware extraction tests --- describe("envelope-aware summary extraction", () => {