From 5fccd99bf9c108bace081bda3669d2a153f40c5d Mon Sep 17 00:00:00 2001 From: Tomaz Zinko Date: Sat, 22 Aug 2026 16:32:19 +0200 Subject: [PATCH] feat: build PDF chapters from the document outline PDFs were ingested as a single "Document" chapter, so a 539-page book arrived as one unbroken chapter whose reading sections could only be labelled by their opening words. pdfjs already exposes the embedded outline, so read it and give each entry its own chapter. Entries at any nesting depth become chapters running to the next entry's page. Entries that cannot describe a contiguous range are dropped: backwards or out-of-range destinations, blank titles, and entries sharing a start page with the previous one. An outline with fewer than two usable entries keeps the existing single-chapter fallback, as does a PDF with no outline at all. --- src/core/ingest/readers/pdfReader.test.ts | 87 ++++++++++++++ src/core/ingest/readers/pdfReader.ts | 132 +++++++++++++++++++--- src/core/ingest/readers/pdfjsAdapter.ts | 68 ++++++++++- 3 files changed, 272 insertions(+), 15 deletions(-) diff --git a/src/core/ingest/readers/pdfReader.test.ts b/src/core/ingest/readers/pdfReader.test.ts index e47bec2..df00be4 100644 --- a/src/core/ingest/readers/pdfReader.test.ts +++ b/src/core/ingest/readers/pdfReader.test.ts @@ -44,6 +44,93 @@ describe('PdfIngestReader', () => { expect(chapters[0].slices[2].html).toContain('data-pdf-page="3"'); }); + it('builds a chapter per PDF outline entry at any nesting depth', async () => { + const pages = Array.from({ length: 8 }, (_, index) => ({ + pageNumber: index + 1, + text: `Page ${index + 1} text.`, + })); + const parsePdf = vi.fn().mockResolvedValue({ + pages, + outline: [ + { title: 'Cover', pageNumber: 1 }, + { title: 'A. Consciousness', pageNumber: 3 }, + { title: 'I. Sensuous-Certainty', pageNumber: 4 }, + { title: 'II. Perceiving', pageNumber: 6 }, + { title: 'B. Self-Consciousness', pageNumber: 7 }, + ], + }); + const reader = new PdfIngestReader({ parsePdf }); + + const prepared = await reader.prepareInitial(new File(['%PDF-1.7'], 'outlined.pdf')); + const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7')); + + const titles = [ + 'Cover', + 'A. Consciousness', + 'I. Sensuous-Certainty', + 'II. Perceiving', + 'B. Self-Consciousness', + ]; + expect(prepared.chapters.map((chapter) => chapter.title)).toEqual(titles); + expect(prepared.structureMode).toBe('authored'); + expect(prepared.chapters[1].boundaryEvidence).toEqual(['publisher-toc']); + expect(chapters.map((chapter) => chapter.title)).toEqual(titles); + + // The first chapter absorbs the pages ahead of it; each later chapter runs to the next entry. + expect(chapters.map((chapter) => chapter.slices.map((slice) => slice.text))).toEqual([ + ['Page 1 text.', 'Page 2 text.'], + ['Page 3 text.'], + ['Page 4 text.', 'Page 5 text.'], + ['Page 6 text.'], + ['Page 7 text.', 'Page 8 text.'], + ]); + }); + + it('drops outline entries that cannot describe a page range', async () => { + const reader = new PdfIngestReader({ + parsePdf: vi.fn().mockResolvedValue({ + pages: [ + { pageNumber: 1, text: 'One.' }, + { pageNumber: 2, text: 'Two.' }, + { pageNumber: 3, text: 'Three.' }, + ], + outline: [ + { title: 'Part One', pageNumber: 1 }, + { title: 'Chapter I', pageNumber: 1 }, + { title: 'Backwards', pageNumber: 1 }, + { title: 'Off The End', pageNumber: 99 }, + { title: ' ', pageNumber: 2 }, + { title: 'Chapter II', pageNumber: 3 }, + ], + }), + }); + + const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7')); + + expect(chapters.map((chapter) => chapter.title)).toEqual(['Part One', 'Chapter II']); + expect(chapters[0].slices).toHaveLength(2); + }); + + it('falls back to a single document chapter when the outline is too thin', async () => { + const reader = new PdfIngestReader({ + parsePdf: vi.fn().mockResolvedValue({ + pages: [ + { pageNumber: 1, text: 'One.' }, + { pageNumber: 2, text: 'Two.' }, + ], + outline: [{ title: 'Only Entry', pageNumber: 2 }], + }), + }); + + const prepared = await reader.prepareInitial(new File(['%PDF-1.7'], 'thin.pdf')); + const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7')); + + expect(prepared.chapters.map((chapter) => chapter.title)).toEqual(['Document']); + expect(prepared.structureMode).toBeUndefined(); + expect(chapters.map((chapter) => chapter.title)).toEqual(['Document']); + expect(chapters[0].slices).toHaveLength(2); + }); + it('recognizes PDF MIME, extension, and raw signatures', () => { const reader = new PdfIngestReader({ parsePdf: vi.fn() }); diff --git a/src/core/ingest/readers/pdfReader.ts b/src/core/ingest/readers/pdfReader.ts index fb23f9f..cd99f76 100644 --- a/src/core/ingest/readers/pdfReader.ts +++ b/src/core/ingest/readers/pdfReader.ts @@ -23,10 +23,22 @@ export interface ParsedPdfPage { noteAnchors?: PdfNoteAnchor[]; } +export interface PdfOutlineEntry { + title: string; + pageNumber: number; +} + export interface ParsedPdfDocument { title?: string; author?: string; pages: ParsedPdfPage[]; + outline?: PdfOutlineEntry[]; +} + +export interface PdfPlannedChapter { + title: string; + startPage: number; + endPage: number; } export interface PdfParseOptions { @@ -59,6 +71,8 @@ const normalizePdfText = (value: string): string => value .replace(/\n{3,}/g, '\n\n') .trim(); +const normalizeWhitespace = (value: string): string => value.replaceAll('\0', '').replace(/\s+/g, ' ').trim(); + const validatePdfSize = (size: number): void => { if (size > MAX_PDF_BYTES) { throw new Error(`PDF is too large. The maximum supported size is ${MAX_PDF_BYTES / 1024 / 1024} MB.`); @@ -136,6 +150,66 @@ const applyPdfLineWraps = (document: ParsedPdfDocument): ParsedPdfDocument => { }; }; +const toChapterSlice = (page: ParsedPdfPage) => ({ + text: page.text, + html: `
${escapeHtml(page.text)}
`, +}); + +const MIN_OUTLINE_CHAPTERS = 2; + +const sanitizeOutline = ( + outline: PdfOutlineEntry[] | undefined, + pageCount: number, +): PdfOutlineEntry[] => { + if (!outline || outline.length === 0) return []; + + const entries: PdfOutlineEntry[] = []; + let lastPage = 0; + + for (const entry of outline) { + const title = normalizeWhitespace(entry.title); + if (!title) continue; + if (!Number.isInteger(entry.pageNumber) || entry.pageNumber < 1 || entry.pageNumber > pageCount) continue; + // A destination that jumps backwards means the outline is not in reading order; it cannot + // describe contiguous page ranges, so drop the stray entry rather than the whole outline. + if (entry.pageNumber < lastPage) continue; + // Entries sharing a start page (a part and its first chapter) would leave an empty range. + if (entries.length > 0 && entry.pageNumber === lastPage) continue; + + lastPage = entry.pageNumber; + entries.push({ title, pageNumber: entry.pageNumber }); + } + + return entries; +}; + +/** + * Turns a PDF's outline into contiguous chapters, one per entry at any nesting depth. Returns null + * when the outline is missing or too thin to describe the document, leaving the single-chapter + * fallback in place. + */ +export const planPdfChapters = ( + outline: PdfOutlineEntry[] | undefined, + pageCount: number, +): PdfPlannedChapter[] | null => { + if (pageCount < 1) return null; + + const entries = sanitizeOutline(outline, pageCount); + if (entries.length < MIN_OUTLINE_CHAPTERS) return null; + + return entries.map((entry, index) => { + const next = entries[index + 1]; + // The first chapter absorbs any pages the outline leaves in front of it. + const startPage = index === 0 ? 1 : entry.pageNumber; + + return { + title: entry.title, + startPage, + endPage: next ? Math.max(startPage, next.pageNumber - 1) : pageCount, + }; + }); +}; + export class PdfIngestReader implements IngestReaderPlugin { public readonly id = 'pdf'; @@ -167,14 +241,25 @@ export class PdfIngestReader implements IngestReaderPlugin { const document = normalizeParsedDocument(await this.dependencies.parsePdf(rawData, onProgress), false); const fallbackTitle = stripFileExtension(file.name).trim() || file.name; + const plan = planPdfChapters(document.outline, document.pages.length); + return { title: document.title?.trim() || fallbackTitle, author: document.author?.trim() || 'Unknown', images: [], - chapters: [{ - title: 'Document', - source: 'spine', - }], + chapters: plan + ? plan.map((chapter) => ({ + title: chapter.title, + source: 'toc' as const, + structureOwnership: 'authored' as const, + reformationReason: 'authored-boundary' as const, + boundaryEvidence: ['publisher-toc' as const], + })) + : [{ + title: 'Document', + source: 'spine', + }], + ...(plan ? { structureVersion: 1 as const, structureMode: 'authored' as const } : {}), }; } @@ -186,17 +271,36 @@ export class PdfIngestReader implements IngestReaderPlugin { ))); const notes = document.pages.flatMap((page) => page.notes || []); const noteAnchors = document.pages.flatMap((page) => page.noteAnchors || []); + const plan = planPdfChapters(document.outline, document.pages.length); + + if (!plan) { + return [{ + title: 'Document', + source: 'spine', + slices: document.pages.map(toChapterSlice), + ...(notes.length > 0 ? { notes } : {}), + ...(noteAnchors.length > 0 ? { noteAnchors } : {}), + }]; + } - return [{ - title: 'Document', - source: 'spine', - slices: document.pages.map((page) => ({ - text: page.text, - html: `
${escapeHtml(page.text)}
`, - })), - ...(notes.length > 0 ? { notes } : {}), - ...(noteAnchors.length > 0 ? { noteAnchors } : {}), - }]; + return plan.map((chapter) => { + const pages = document.pages.filter((page) => ( + page.pageNumber >= chapter.startPage && page.pageNumber <= chapter.endPage + )); + const chapterNotes = pages.flatMap((page) => page.notes || []); + const chapterNoteAnchors = pages.flatMap((page) => page.noteAnchors || []); + + return { + title: chapter.title, + source: 'toc' as const, + structureOwnership: 'authored' as const, + reformationReason: 'authored-boundary' as const, + boundaryEvidence: ['publisher-toc' as const], + slices: pages.map(toChapterSlice), + ...(chapterNotes.length > 0 ? { notes: chapterNotes } : {}), + ...(chapterNoteAnchors.length > 0 ? { noteAnchors: chapterNoteAnchors } : {}), + }; + }); } } diff --git a/src/core/ingest/readers/pdfjsAdapter.ts b/src/core/ingest/readers/pdfjsAdapter.ts index 83da30a..596bfbf 100644 --- a/src/core/ingest/readers/pdfjsAdapter.ts +++ b/src/core/ingest/readers/pdfjsAdapter.ts @@ -1,4 +1,4 @@ -import { MAX_PDF_PAGES, type ParsedPdfDocument, type PdfParseOptions } from './pdfReader'; +import { MAX_PDF_PAGES, type ParsedPdfDocument, type PdfOutlineEntry, type PdfParseOptions } from './pdfReader'; import { TesseractPdfOcrEngine, type PdfOcrEngine, type PdfOcrWord } from './pdfOcrAdapter'; import { normalizePdfBox, type PdfLayoutWord, type PdfTextDirection } from './pdfLayout'; import { createOperationHandle } from '../../operations/progressReporter'; @@ -40,14 +40,78 @@ interface PdfLoadingTask { destroy: () => Promise; } +interface PdfOutlineNode { + title?: unknown; + dest?: unknown; + items?: unknown; +} + interface PdfDocument { numPages: number; getMetadata: () => Promise; getPageLabels: () => Promise; + getOutline: () => Promise; + getDestination: (id: string) => Promise; + getPageIndex: (ref: unknown) => Promise; getPage: (pageNumber: number) => Promise; destroy: () => Promise; } +const isOutlineNode = (value: unknown): value is PdfOutlineNode => ( + !!value && typeof value === 'object' +); + +const resolveOutlinePage = async ( + document: PdfDocument, + dest: unknown, +): Promise => { + try { + const resolved = typeof dest === 'string' ? await document.getDestination(dest) : dest; + if (!Array.isArray(resolved) || resolved.length === 0) return undefined; + const pageIndex = await document.getPageIndex(resolved[0]); + if (!Number.isInteger(pageIndex) || pageIndex < 0) return undefined; + return pageIndex + 1; + } catch { + return undefined; + } +}; + +const collectOutlineEntries = async ( + document: PdfDocument, + nodes: unknown, + depth: number, + entries: PdfOutlineEntry[], +): Promise => { + if (!Array.isArray(nodes) || depth > MAX_OUTLINE_DEPTH) return; + + for (const node of nodes) { + if (!isOutlineNode(node)) continue; + if (entries.length >= MAX_OUTLINE_ENTRIES) return; + + const title = typeof node.title === 'string' ? node.title.replaceAll('\0', '').trim() : ''; + const pageNumber = await resolveOutlinePage(document, node.dest); + if (title && pageNumber !== undefined) { + entries.push({ title, pageNumber }); + } + + await collectOutlineEntries(document, node.items, depth + 1, entries); + } +}; + +const extractOutline = async (document: PdfDocument): Promise => { + try { + const nodes = await document.getOutline(); + const entries: PdfOutlineEntry[] = []; + await collectOutlineEntries(document, nodes, 0, entries); + return entries; + } catch { + return []; + } +}; + +const MAX_OUTLINE_DEPTH = 6; +const MAX_OUTLINE_ENTRIES = 5_000; + export const MAX_OCR_LONG_EDGE = 3_500; export const MAX_OCR_PIXELS = 16_000_000; const OCR_SCALE = 250 / 72; @@ -286,6 +350,7 @@ export const parsePdfWithPdfJs = async ( const metadata = await loadedDocument.getMetadata() as PdfMetadata; const pageLabels = await loadedDocument.getPageLabels(); + const outline = await extractOutline(loadedDocument); if (options.useOcr) { ocrEngine = options.ocrEngine || new TesseractPdfOcrEngine(); ownsOcrEngine = !options.ocrEngine; @@ -337,6 +402,7 @@ export const parsePdfWithPdfJs = async ( title: readMetadataValue(metadata, 'Title', 'dc:title'), author: readMetadataValue(metadata, 'Author', 'dc:creator'), pages, + ...(outline.length > 0 ? { outline } : {}), }; operation.complete('PDF parsed'); return result;