Build PDF chapters from the document outline

src/core/ingest/readers/pdfReader.test.ts

@@ -44,6 +44,93 @@ describe('PdfIngestReader', () => {
         expect(chapters[0].slices[2].html).toContain('data-pdf-page="3"');
     });
 
+    it('builds a chapter per PDF outline entry at any nesting depth', async () => {
+        const pages = Array.from({ length: 8 }, (_, index) => ({
+            pageNumber: index + 1,
+            text: `Page ${index + 1} text.`,
+        }));
+        const parsePdf = vi.fn().mockResolvedValue({
+            pages,
+            outline: [
+                { title: 'Cover', pageNumber: 1 },
+                { title: 'A. Consciousness', pageNumber: 3 },
+                { title: 'I. Sensuous-Certainty', pageNumber: 4 },
+                { title: 'II. Perceiving', pageNumber: 6 },
+                { title: 'B. Self-Consciousness', pageNumber: 7 },
+            ],
+        });
+        const reader = new PdfIngestReader({ parsePdf });
+
+        const prepared = await reader.prepareInitial(new File(['%PDF-1.7'], 'outlined.pdf'));
+        const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7'));
+
+        const titles = [
+            'Cover',
+            'A. Consciousness',
+            'I. Sensuous-Certainty',
+            'II. Perceiving',
+            'B. Self-Consciousness',
+        ];
+        expect(prepared.chapters.map((chapter) => chapter.title)).toEqual(titles);
+        expect(prepared.structureMode).toBe('authored');
+        expect(prepared.chapters[1].boundaryEvidence).toEqual(['publisher-toc']);
+        expect(chapters.map((chapter) => chapter.title)).toEqual(titles);
+
+        // The first chapter absorbs the pages ahead of it; each later chapter runs to the next entry.
+        expect(chapters.map((chapter) => chapter.slices.map((slice) => slice.text))).toEqual([
+            ['Page 1 text.', 'Page 2 text.'],
+            ['Page 3 text.'],
+            ['Page 4 text.', 'Page 5 text.'],
+            ['Page 6 text.'],
+            ['Page 7 text.', 'Page 8 text.'],
+        ]);
+    });
+
+    it('drops outline entries that cannot describe a page range', async () => {
+        const reader = new PdfIngestReader({
+            parsePdf: vi.fn().mockResolvedValue({
+                pages: [
+                    { pageNumber: 1, text: 'One.' },
+                    { pageNumber: 2, text: 'Two.' },
+                    { pageNumber: 3, text: 'Three.' },
+                ],
+                outline: [
+                    { title: 'Part One', pageNumber: 1 },
+                    { title: 'Chapter I', pageNumber: 1 },
+                    { title: 'Backwards', pageNumber: 1 },
+                    { title: 'Off The End', pageNumber: 99 },
+                    { title: '   ', pageNumber: 2 },
+                    { title: 'Chapter II', pageNumber: 3 },
+                ],
+            }),
+        });
+
+        const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7'));
+
+        expect(chapters.map((chapter) => chapter.title)).toEqual(['Part One', 'Chapter II']);
+        expect(chapters[0].slices).toHaveLength(2);
+    });
+
+    it('falls back to a single document chapter when the outline is too thin', async () => {
+        const reader = new PdfIngestReader({
+            parsePdf: vi.fn().mockResolvedValue({
+                pages: [
+                    { pageNumber: 1, text: 'One.' },
+                    { pageNumber: 2, text: 'Two.' },
+                ],
+                outline: [{ title: 'Only Entry', pageNumber: 2 }],
+            }),
+        });
+
+        const prepared = await reader.prepareInitial(new File(['%PDF-1.7'], 'thin.pdf'));
+        const chapters = await reader.loadChapters(new TextEncoder().encode('%PDF-1.7'));
+
+        expect(prepared.chapters.map((chapter) => chapter.title)).toEqual(['Document']);
+        expect(prepared.structureMode).toBeUndefined();
+        expect(chapters.map((chapter) => chapter.title)).toEqual(['Document']);
+        expect(chapters[0].slices).toHaveLength(2);
+    });
+
     it('recognizes PDF MIME, extension, and raw signatures', () => {
         const reader = new PdfIngestReader({ parsePdf: vi.fn() });
 

src/core/ingest/readers/pdfReader.ts

@@ -23,10 +23,22 @@ export interface ParsedPdfPage {
     noteAnchors?: PdfNoteAnchor[];
 }
 
+export interface PdfOutlineEntry {
+    title: string;
+    pageNumber: number;
+}
+
 export interface ParsedPdfDocument {
     title?: string;
     author?: string;
     pages: ParsedPdfPage[];
+    outline?: PdfOutlineEntry[];
+}
+
+export interface PdfPlannedChapter {
+    title: string;
+    startPage: number;
+    endPage: number;
 }
 
 export interface PdfParseOptions {
@@ -59,6 +71,8 @@ const normalizePdfText = (value: string): string => value
     .replace(/\n{3,}/g, '\n\n')
     .trim();
 
+const normalizeWhitespace = (value: string): string => value.replaceAll('\0', '').replace(/\s+/g, ' ').trim();
+
 const validatePdfSize = (size: number): void => {
     if (size > MAX_PDF_BYTES) {
         throw new Error(`PDF is too large. The maximum supported size is ${MAX_PDF_BYTES / 1024 / 1024} MB.`);
@@ -136,6 +150,66 @@ const applyPdfLineWraps = (document: ParsedPdfDocument): ParsedPdfDocument => {
     };
 };
 
+const toChapterSlice = (page: ParsedPdfPage) => ({
+    text: page.text,
+    html: `<div data-pdf-page="${page.pageNumber}">${escapeHtml(page.text)}</div>`,
+});
+
+const MIN_OUTLINE_CHAPTERS = 2;
+
+const sanitizeOutline = (
+    outline: PdfOutlineEntry[] | undefined,
+    pageCount: number,
+): PdfOutlineEntry[] => {
+    if (!outline || outline.length === 0) return [];
+
+    const entries: PdfOutlineEntry[] = [];
+    let lastPage = 0;
+
+    for (const entry of outline) {
+        const title = normalizeWhitespace(entry.title);
+        if (!title) continue;
+        if (!Number.isInteger(entry.pageNumber) || entry.pageNumber < 1 || entry.pageNumber > pageCount) continue;
+        // A destination that jumps backwards means the outline is not in reading order; it cannot
+        // describe contiguous page ranges, so drop the stray entry rather than the whole outline.
+        if (entry.pageNumber < lastPage) continue;
+        // Entries sharing a start page (a part and its first chapter) would leave an empty range.
+        if (entries.length > 0 && entry.pageNumber === lastPage) continue;
+
+        lastPage = entry.pageNumber;
+        entries.push({ title, pageNumber: entry.pageNumber });
+    }
+
+    return entries;
+};
+
+/**
+ * Turns a PDF's outline into contiguous chapters, one per entry at any nesting depth. Returns null
+ * when the outline is missing or too thin to describe the document, leaving the single-chapter
+ * fallback in place.
+ */
+export const planPdfChapters = (
+    outline: PdfOutlineEntry[] | undefined,
+    pageCount: number,
+): PdfPlannedChapter[] | null => {
+    if (pageCount < 1) return null;
+
+    const entries = sanitizeOutline(outline, pageCount);
+    if (entries.length < MIN_OUTLINE_CHAPTERS) return null;
+
+    return entries.map((entry, index) => {
+        const next = entries[index + 1];
+        // The first chapter absorbs any pages the outline leaves in front of it.
+        const startPage = index === 0 ? 1 : entry.pageNumber;
+
+        return {
+            title: entry.title,
+            startPage,
+            endPage: next ? Math.max(startPage, next.pageNumber - 1) : pageCount,
+        };
+    });
+};
+
 export class PdfIngestReader implements IngestReaderPlugin {
     public readonly id = 'pdf';
 
@@ -167,14 +241,25 @@ export class PdfIngestReader implements IngestReaderPlugin {
         const document = normalizeParsedDocument(await this.dependencies.parsePdf(rawData, onProgress), false);
         const fallbackTitle = stripFileExtension(file.name).trim() || file.name;
 
+        const plan = planPdfChapters(document.outline, document.pages.length);
+
         return {
             title: document.title?.trim() || fallbackTitle,
             author: document.author?.trim() || 'Unknown',
             images: [],
-            chapters: [{
-                title: 'Document',
-                source: 'spine',
-            }],
+            chapters: plan
+                ? plan.map((chapter) => ({
+                    title: chapter.title,
+                    source: 'toc' as const,
+                    structureOwnership: 'authored' as const,
+                    reformationReason: 'authored-boundary' as const,
+                    boundaryEvidence: ['publisher-toc' as const],
+                }))
+                : [{
+                    title: 'Document',
+                    source: 'spine',
+                }],
+            ...(plan ? { structureVersion: 1 as const, structureMode: 'authored' as const } : {}),
         };
     }
 
@@ -186,17 +271,36 @@ export class PdfIngestReader implements IngestReaderPlugin {
         )));
         const notes = document.pages.flatMap((page) => page.notes || []);
         const noteAnchors = document.pages.flatMap((page) => page.noteAnchors || []);
+        const plan = planPdfChapters(document.outline, document.pages.length);
+
+        if (!plan) {
+            return [{
+                title: 'Document',
+                source: 'spine',
+                slices: document.pages.map(toChapterSlice),
+                ...(notes.length > 0 ? { notes } : {}),
+                ...(noteAnchors.length > 0 ? { noteAnchors } : {}),
+            }];
+        }
 
-        return [{
-            title: 'Document',
-            source: 'spine',
-            slices: document.pages.map((page) => ({
-                text: page.text,
-                html: `<div data-pdf-page="${page.pageNumber}">${escapeHtml(page.text)}</div>`,
-            })),
-            ...(notes.length > 0 ? { notes } : {}),
-            ...(noteAnchors.length > 0 ? { noteAnchors } : {}),
-        }];
+        return plan.map((chapter) => {
+            const pages = document.pages.filter((page) => (
+                page.pageNumber >= chapter.startPage && page.pageNumber <= chapter.endPage
+            ));
+            const chapterNotes = pages.flatMap((page) => page.notes || []);
+            const chapterNoteAnchors = pages.flatMap((page) => page.noteAnchors || []);
+
+            return {
+                title: chapter.title,
+                source: 'toc' as const,
+                structureOwnership: 'authored' as const,
+                reformationReason: 'authored-boundary' as const,
+                boundaryEvidence: ['publisher-toc' as const],
+                slices: pages.map(toChapterSlice),
+                ...(chapterNotes.length > 0 ? { notes: chapterNotes } : {}),
+                ...(chapterNoteAnchors.length > 0 ? { noteAnchors: chapterNoteAnchors } : {}),
+            };
+        });
     }
 }
 

src/core/ingest/readers/pdfjsAdapter.ts

@@ -1,4 +1,4 @@
-import { MAX_PDF_PAGES, type ParsedPdfDocument, type PdfParseOptions } from './pdfReader';
+import { MAX_PDF_PAGES, type ParsedPdfDocument, type PdfOutlineEntry, type PdfParseOptions } from './pdfReader';
 import { TesseractPdfOcrEngine, type PdfOcrEngine, type PdfOcrWord } from './pdfOcrAdapter';
 import { normalizePdfBox, type PdfLayoutWord, type PdfTextDirection } from './pdfLayout';
 import { createOperationHandle } from '../../operations/progressReporter';
@@ -40,14 +40,78 @@ interface PdfLoadingTask {
     destroy: () => Promise<void>;
 }
 
+interface PdfOutlineNode {
+    title?: unknown;
+    dest?: unknown;
+    items?: unknown;
+}
+
 interface PdfDocument {
     numPages: number;
     getMetadata: () => Promise<unknown>;
     getPageLabels: () => Promise<string[] | null>;
+    getOutline: () => Promise<unknown>;
+    getDestination: (id: string) => Promise<unknown>;
+    getPageIndex: (ref: unknown) => Promise<number>;
     getPage: (pageNumber: number) => Promise<PdfPage>;
     destroy: () => Promise<void>;
 }
 
+const isOutlineNode = (value: unknown): value is PdfOutlineNode => (
+    !!value && typeof value === 'object'
+);
+
+const resolveOutlinePage = async (
+    document: PdfDocument,
+    dest: unknown,
+): Promise<number | undefined> => {
+    try {
+        const resolved = typeof dest === 'string' ? await document.getDestination(dest) : dest;
+        if (!Array.isArray(resolved) || resolved.length === 0) return undefined;
+        const pageIndex = await document.getPageIndex(resolved[0]);
+        if (!Number.isInteger(pageIndex) || pageIndex < 0) return undefined;
+        return pageIndex + 1;
+    } catch {
+        return undefined;
+    }
+};
+
+const collectOutlineEntries = async (
+    document: PdfDocument,
+    nodes: unknown,
+    depth: number,
+    entries: PdfOutlineEntry[],
+): Promise<void> => {
+    if (!Array.isArray(nodes) || depth > MAX_OUTLINE_DEPTH) return;
+
+    for (const node of nodes) {
+        if (!isOutlineNode(node)) continue;
+        if (entries.length >= MAX_OUTLINE_ENTRIES) return;
+
+        const title = typeof node.title === 'string' ? node.title.replaceAll('\0', '').trim() : '';
+        const pageNumber = await resolveOutlinePage(document, node.dest);
+        if (title && pageNumber !== undefined) {
+            entries.push({ title, pageNumber });
+        }
+
+        await collectOutlineEntries(document, node.items, depth + 1, entries);
+    }
+};
+
+const extractOutline = async (document: PdfDocument): Promise<PdfOutlineEntry[]> => {
+    try {
+        const nodes = await document.getOutline();
+        const entries: PdfOutlineEntry[] = [];
+        await collectOutlineEntries(document, nodes, 0, entries);
+        return entries;
+    } catch {
+        return [];
+    }
+};
+
+const MAX_OUTLINE_DEPTH = 6;
+const MAX_OUTLINE_ENTRIES = 5_000;
+
 export const MAX_OCR_LONG_EDGE = 3_500;
 export const MAX_OCR_PIXELS = 16_000_000;
 const OCR_SCALE = 250 / 72;
@@ -286,6 +350,7 @@ export const parsePdfWithPdfJs = async (
 
         const metadata = await loadedDocument.getMetadata() as PdfMetadata;
         const pageLabels = await loadedDocument.getPageLabels();
+        const outline = await extractOutline(loadedDocument);
         if (options.useOcr) {
             ocrEngine = options.ocrEngine || new TesseractPdfOcrEngine();
             ownsOcrEngine = !options.ocrEngine;
@@ -337,6 +402,7 @@ export const parsePdfWithPdfJs = async (
             title: readMetadataValue(metadata, 'Title', 'dc:title'),
             author: readMetadataValue(metadata, 'Author', 'dc:creator'),
             pages,
+            ...(outline.length > 0 ? { outline } : {}),
         };
         operation.complete('PDF parsed');
         return result;