diff --git a/CHANGES.txt b/CHANGES.txt index 4d46e869628..64baa29f356 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,17 @@ Release 3.3.3 - ??? * MagicDetector now compiles its regular expression once, in the constructor, instead of recompiling it on every match (TIKA-4796). + * OneNote extraction now follows document order, omits superseded page + revisions, sorts author metadata, extracts embedded object BLOBs, and + bounds malformed-input recursion and file-derived allocations. Parse + warnings and embedded relationship IDs are exposed in metadata. Malformed + or truncated files that cannot be fully parsed, and files whose walk + yields no content, now fall back to the legacy string dump instead of + failing or returning empty output. The legacy MS-ONESTORE walker bounds + its recursion (depth caps plus file-node-list and fragment-chain cycle + guards) and now honors shouldParseEmbedded for embedded file data + (TIKA-4814). + * PDFs whose %PDF- header is preceded by a print-composition job ticket are no longer detected as text/x-matlab. Up to 50 %% comment or blank lines of up to 150 characters, and nothing else, may now precede the header; the diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/FileNodePtr.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/FileNodePtr.java index 2646def38b6..e82706a7add 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/FileNodePtr.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/FileNodePtr.java @@ -56,12 +56,21 @@ public FileNode dereference(OneNoteDocument document) throws TikaException { if (nodeListPositions.isEmpty()) { return null; } - if (nodeListPositions.get(0) >= document.root.children.size()) { + int rootPosition = nodeListPositions.get(0); + if (rootPosition < 0 || rootPosition >= document.root.children.size()) { throw new TikaException("Exceeded root child size"); } - FileNode cur = document.root.children.get(nodeListPositions.get(0)); + FileNode cur = document.root.children.get(rootPosition); for (int i = 1, ie = nodeListPositions.size(); i < ie; ++i) { - cur = cur.childFileNodeList.children.get(nodeListPositions.get(i)); + if (cur == null || cur.childFileNodeList == null || + cur.childFileNodeList.children == null) { + throw new TikaException("Missing child file node list"); + } + int childPosition = nodeListPositions.get(i); + if (childPosition < 0 || childPosition >= cur.childFileNodeList.children.size()) { + throw new TikaException("Exceeded child file node size"); + } + cur = cur.childFileNodeList.children.get(childPosition); } return cur; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java index 670824eec7b..55e788bb4a5 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Locale; +import org.apache.tika.exception.TikaException; import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.BitConverter; import org.apache.tika.utils.StringUtils; @@ -41,17 +42,44 @@ public GUID(int[] guid) { * -EEEEEEEEEEEE} * @return GUID object parsed from guid bytes. */ - public static GUID fromCurlyBraceUTF16Bytes(byte[] guid) { + public static GUID fromCurlyBraceUTF16Bytes(byte[] guid) throws TikaException { int[] intGuid = new int[16]; - String utf16Str = new String(guid, StandardCharsets.UTF_16LE).replaceAll("\\{", "") - .replaceAll("-", "").replaceAll("}", ""); - for (int i = 0; i < utf16Str.length(); i += 2) { - intGuid[i / 2] = - Integer.parseUnsignedInt("" + utf16Str.charAt(i) + utf16Str.charAt(i + 1), 16); + String utf16Str = new String(guid, StandardCharsets.UTF_16LE); + if (utf16Str.length() != 38 || utf16Str.charAt(0) != '{' || + utf16Str.charAt(9) != '-' || utf16Str.charAt(14) != '-' || + utf16Str.charAt(19) != '-' || utf16Str.charAt(24) != '-' || + utf16Str.charAt(37) != '}') { + throw new TikaException("Invalid GUID string"); + } + String hex = utf16Str.substring(1, 37).replace("-", ""); + if (hex.length() != 32) { + throw new TikaException("Invalid GUID string"); + } + for (int i = 0; i < hex.length(); i += 2) { + int high = asciiHexDigit(hex.charAt(i)); + int low = asciiHexDigit(hex.charAt(i + 1)); + if (high < 0 || low < 0) { + throw new TikaException("Invalid GUID string"); + } + intGuid[i / 2] = (high << 4) | low; } return new GUID(intGuid); } + // Character.digit accepts non-ASCII Unicode digits; GUIDs are ASCII hex only + private static int asciiHexDigit(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + public static int memcmp(int[] b1, int[] b2, int sz) { for (int i = 0; i < sz; i++) { if (b1[i] != b2[i]) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java index 936228f25c8..cd76a58a558 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java @@ -34,6 +34,9 @@ class OneNoteDocument { Map> revisionRoleMap = new HashMap<>(); ExtendedGUID currentRevision = ExtendedGUID.nil(); FileNodeList root = new FileNodeList(); + // set when the root file node list could not be fully parsed (e.g. a truncated file); + // the header and any structure parsed so far remain usable + Exception structureParseException; public OneNoteDocument() { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java index d4a265caeef..f7e03378c02 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -27,10 +28,13 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.WriteLimitReachedException; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -52,6 +56,7 @@ public class OneNoteParser implements Parser { public static final String ONE_NOTE_PREFIX = "onenote:"; + private static final Logger LOG = LoggerFactory.getLogger(OneNoteParser.class); private static final Map> TYPES_MAP = new HashMap<>(); /** * Serial version UID @@ -126,53 +131,76 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, metadata.set(ONE_NOTE_PREFIX + "rgbPlaceholder", "0x" + Long.toHexString(oneNoteDocument.header.rgbPlaceholder)); - Pair roleAndContext = Pair.of(1L, ExtendedGUID.nil()); - OneNoteTreeWalker oneNoteTreeWalker = - new OneNoteTreeWalker(options, oneNoteDocument, oneNoteDirectFileResource, - xhtml, metadata, context, roleAndContext); + Exception structureFailure = oneNoteDocument.structureParseException; + boolean walked = false; + if (structureFailure == null) { + try { + Pair roleAndContext = Pair.of(1L, ExtendedGUID.nil()); + OneNoteTreeWalker oneNoteTreeWalker = + new OneNoteTreeWalker(options, oneNoteDocument, + oneNoteDirectFileResource, xhtml, metadata, context, + roleAndContext); - oneNoteTreeWalker.walkTree(); + oneNoteTreeWalker.walkTree(); - if (!oneNoteTreeWalker.getAuthors().isEmpty()) { - metadata.set(TikaCoreProperties.CREATOR, - oneNoteTreeWalker.getAuthors().toArray(new String[]{})); + if (!oneNoteTreeWalker.getAuthors().isEmpty()) { + metadata.set(TikaCoreProperties.CREATOR, + sortedValues(oneNoteTreeWalker.getAuthors())); + } + if (!oneNoteTreeWalker.getMostRecentAuthors().isEmpty()) { + metadata.set( + Property.externalTextBag(ONE_NOTE_PREFIX + "mostRecentAuthors"), + sortedValues(oneNoteTreeWalker.getMostRecentAuthors())); + } + if (!oneNoteTreeWalker.getOriginalAuthors().isEmpty()) { + metadata.set( + Property.externalTextBag(ONE_NOTE_PREFIX + "originalAuthors"), + sortedValues(oneNoteTreeWalker.getOriginalAuthors())); + } + if (!Instant.MAX.equals( + Instant.ofEpochMilli(oneNoteTreeWalker.getCreationTimestamp()))) { + metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", + String.valueOf(oneNoteTreeWalker.getCreationTimestamp())); + } + if (!Instant.MIN.equals(oneNoteTreeWalker.getLastModifiedTimestamp())) { + metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp", String.valueOf( + oneNoteTreeWalker.getLastModifiedTimestamp().toEpochMilli())); + } + if (oneNoteTreeWalker.getLastModified() > Long.MIN_VALUE) { + metadata.set(TikaCoreProperties.MODIFIED, + String.valueOf(oneNoteTreeWalker.getLastModified())); + } + walked = true; + } catch (Exception e) { + rethrowIfLimitReached(e); + structureFailure = e; + } } - if (!oneNoteTreeWalker.getMostRecentAuthors().isEmpty()) { - metadata.set(Property.externalTextBag(ONE_NOTE_PREFIX + "mostRecentAuthors"), - oneNoteTreeWalker.getMostRecentAuthors().toArray(new String[]{})); - } - if (!oneNoteTreeWalker.getOriginalAuthors().isEmpty()) { - metadata.set(Property.externalTextBag(ONE_NOTE_PREFIX + "originalAuthors"), - oneNoteTreeWalker.getOriginalAuthors().toArray(new String[]{})); - } - if (!Instant.MAX.equals( - Instant.ofEpochMilli(oneNoteTreeWalker.getCreationTimestamp()))) { - metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", - String.valueOf(oneNoteTreeWalker.getCreationTimestamp())); - } - if (!Instant.MIN.equals(oneNoteTreeWalker.getLastModifiedTimestamp())) { - metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp", String.valueOf( - oneNoteTreeWalker.getLastModifiedTimestamp().toEpochMilli())); - } - if (oneNoteTreeWalker.getLastModified() > Long.MIN_VALUE) { - metadata.set(TikaCoreProperties.MODIFIED, - String.valueOf(oneNoteTreeWalker.getLastModified())); + if (!walked) { + legacyFallbackDump("OneNote parse failed; falling back to legacy text dump: " + + failureMessage(structureFailure), structureFailure, metadata, + xhtml, oneNoteDirectFileResource); } } else if (header.isLegacyOrAlternativePackaging()) { + MSOneStorePackage pkg = null; try { AlternativePackaging alternatePackageOneStoreFile = new AlternativePackaging(); + //enable streaming deserialization alternatePackageOneStoreFile.doDeserializeFromByteArray(oneStoreFileBytes, 0); MSOneStoreParser onenoteParser = new MSOneStoreParser(); - MSOneStorePackage pkg = - onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage); + pkg = onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage); - pkg.walkTree(options, metadata, xhtml); + pkg.walkTree(options, metadata, xhtml, context); } catch (Exception e) { - OneNoteLegacyDumpStrings dumpStrings = - new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml); - dumpStrings.dump(); + rethrowIfLimitReached(e); + legacyFallbackDump( + "OneNote FSSHTTPB parse failed; falling back to legacy text dump: " + + failureMessage(e), e, metadata, xhtml, + oneNoteDirectFileResource); + pkg = null; } + legacyFallbackIfNoContent(pkg, metadata, xhtml, oneNoteDirectFileResource); } else { throw new TikaException("Invalid OneStore document - could not parse headers"); } @@ -182,6 +210,45 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, } + private static String[] sortedValues(Set values) { + String[] sorted = values.toArray(new String[0]); + Arrays.sort(sorted); + return sorted; + } + + private static void rethrowIfLimitReached(Exception e) throws TikaException, SAXException { + WriteLimitReachedException.throwIfWriteLimitReached(e); + } + + private static String failureMessage(Exception e) { + return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + } + + // the walk completed but every page dangled - without this a degraded + // file would yield empty output where the dump still finds its text + static void legacyFallbackIfNoContent(MSOneStorePackage pkg, Metadata metadata, + XHTMLContentHandler xhtml, + OneNoteDirectFileResource oneNoteDirectFileResource) + throws TikaException, SAXException { + if (pkg != null && !pkg.hasEmittedContent()) { + legacyFallbackDump("OneNote FSSHTTPB parse produced no content; " + + "falling back to legacy text dump", null, metadata, xhtml, + oneNoteDirectFileResource); + } + } + + private static void legacyFallbackDump(String warning, Exception cause, Metadata metadata, + XHTMLContentHandler xhtml, + OneNoteDirectFileResource oneNoteDirectFileResource) + throws TikaException, SAXException { + LOG.warn(warning); + if (cause != null) { + LOG.debug("OneNote parse failure", cause); + } + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml).dump(); + } + /** * Create a OneNoteDocument object. *

@@ -215,7 +282,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, * content. * @return A parsed one note document. This document does not contain any of the binary data, * rather it just contains - * the data pointers and metadata. + * the data pointers and metadata. A failure while parsing the root file node list is not + * thrown; it is recorded in the returned document's {@code structureParseException}. * @throws IOException Will throw IOException in typical IO issue situations. */ public OneNoteDocument createOneNoteDocumentFromDirectFileResource( @@ -227,9 +295,15 @@ public OneNoteDocument createOneNoteDocumentFromDirectFileResource( if (oneNoteDocument.header.isMsOneStoreFormat()) { // Now that we parsed the header, the "root file node list" - oneNotePtr.reposition(oneNoteDocument.header.fcrFileNodeListRoot); - FileNodePtr curPath = new FileNodePtr(); - oneNotePtr.deserializeFileNodeList(oneNoteDocument.root, curPath); + try { + oneNotePtr.reposition(oneNoteDocument.header.fcrFileNodeListRoot); + FileNodePtr curPath = new FileNodePtr(); + oneNotePtr.deserializeFileNodeList(oneNoteDocument.root, curPath); + } catch (TikaException | IOException | RuntimeException e) { + // a truncated or malformed root list is recorded, not thrown, so the + // caller can fall back to the legacy string dump + oneNoteDocument.structureParseException = e; + } } return oneNoteDocument; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java index 38a9f3d9f60..916b9bbb5ac 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java @@ -19,11 +19,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.Set; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.EndianUtils; @@ -54,8 +55,28 @@ class OneNotePtr { new byte[]{60, 0, 105, 0, 102, 0, 110, 0, 100, 0, 102, 0, 62, 0}; private static final String PACKAGE_STORAGE_FILE_FORMAT_GUID = "{638DE92F-A6D4-4BC1-9A36-B3FC2511A5B7}"; + private static final int MAX_PROPERTY_VALUES = 100_000; + private static final int MAX_PROPERTY_SET_DEPTH = 1000; + // the format nests file node lists only a handful of levels deep; 100 leaves wide + // margin while keeping the recursion far from any stack limit + private static final int MAX_FILE_NODE_LIST_DEPTH = 100; + + private static final class PropertyValueBudget { + private long remaining = MAX_PROPERTY_VALUES; + } + + /** + * Recursion state for the baseType-2 file-node-list nesting, shared by every pointer + * copied while parsing one document. + */ + private static final class FileNodeListRecursion { + private int depth; + private final Set activeListOffsets = new HashSet<>(); + } int indentLevel = 0; + private PropertyValueBudget propertyValueBudget = new PropertyValueBudget(); + private FileNodeListRecursion fileNodeListRecursion = new FileNodeListRecursion(); long offset; long end; @@ -77,6 +98,8 @@ public OneNotePtr(OneNotePtr oneNotePtr) { this.offset = oneNotePtr.offset; this.end = oneNotePtr.end; this.indentLevel = oneNotePtr.indentLevel; + this.propertyValueBudget = oneNotePtr.propertyValueBudget; + this.fileNodeListRecursion = oneNotePtr.fileNodeListRecursion; } public OneNoteHeader deserializeHeader() throws IOException, TikaException { @@ -260,8 +283,11 @@ private void reposition(long offset) throws IOException { public OneNotePtr internalDeserializeFileNodeList(OneNotePtr ptr, FileNodeList fileNodeList, FileNodePtr curPath) throws IOException, TikaException { - OneNotePtr localPtr = new OneNotePtr(document, dif); + OneNotePtr localPtr = new OneNotePtr(ptr); FileNodePtrBackPush bp = new FileNodePtrBackPush(curPath); + // a next-fragment reference pointing at an already-seen fragment would loop forever + Set seenFragmentOffsets = new HashSet<>(); + seenFragmentOffsets.add(ptr.offset); try { while (true) { FileChunkReference next = FileChunkReference.nil(); @@ -269,6 +295,11 @@ public OneNotePtr internalDeserializeFileNodeList(OneNotePtr ptr, FileNodeList f if (FileChunkReference.nil().equals(next)) { break; } + if (!seenFragmentOffsets.add(next.stp)) { + throw new TikaException( + "OneNote file node list fragment cycle detected at offset " + + next.stp); + } localPtr.reposition(next); ptr = localPtr; } @@ -279,9 +310,30 @@ public OneNotePtr internalDeserializeFileNodeList(OneNotePtr ptr, FileNodeList f } + /** + * Deserializes one file-node list and its fragments. The property-value budget is reset for + * each list; pointers copied while parsing that list share the budget. + */ public OneNotePtr deserializeFileNodeList(FileNodeList fileNodeList, FileNodePtr curPath) throws IOException, TikaException { - return internalDeserializeFileNodeList(this, fileNodeList, curPath); + if (fileNodeListRecursion.depth >= MAX_FILE_NODE_LIST_DEPTH) { + throw new TikaMemoryLimitException( + "OneNote file node list nesting exceeds depth limit " + + MAX_FILE_NODE_LIST_DEPTH); + } + long listOffset = offset; + if (!fileNodeListRecursion.activeListOffsets.add(listOffset)) { + throw new TikaException( + "OneNote file node list cycle detected at offset " + listOffset); + } + fileNodeListRecursion.depth++; + try { + propertyValueBudget = new PropertyValueBudget(); + return internalDeserializeFileNodeList(this, fileNodeList, curPath); + } finally { + fileNodeListRecursion.depth--; + fileNodeListRecursion.activeListOffsets.remove(listOffset); + } } /** @@ -453,23 +505,22 @@ private FileNode deserializeFileNode(FileNode data, FileNodePtr curPath) data.subType.globalIdTableEntryFNDX.guid = deserializeGUID(); - document.revisionMap.get(document.currentRevision).globalId.put( - data.subType.globalIdTableEntryFNDX.index, + Revision currentRevision = getCurrentRevision(); + currentRevision.globalId.put(data.subType.globalIdTableEntryFNDX.index, data.subType.globalIdTableEntryFNDX.guid); } else if (data.id == FndStructureConstants.GlobalIdTableEntry2FNDX) { data.subType.globalIdTableEntry2FNDX.indexMapFrom = deserializeLittleEndianInt(); data.subType.globalIdTableEntry2FNDX.indexMapTo = deserializeLittleEndianInt(); - ExtendedGUID dependentRevision = - document.revisionMap.get(document.currentRevision).dependent; - // Get the compactId from the revisionMap's globalId map. - GUID compactId = document.revisionMap.get(dependentRevision).globalId.get( + Revision currentRevision = getCurrentRevision(); + Revision dependentRevision = getRevision(currentRevision.dependent); + GUID compactId = dependentRevision.globalId.get( data.subType.globalIdTableEntry2FNDX.indexMapFrom); if (compactId == null) { throw new TikaException("COMPACT_ID_MISSING"); } - document.revisionMap.get(document.currentRevision).globalId.put( - data.subType.globalIdTableEntry2FNDX.indexMapTo, compactId); + currentRevision.globalId.put(data.subType.globalIdTableEntry2FNDX.indexMapTo, + compactId); } else if (data.id == FndStructureConstants.GlobalIdTableEntry3FNDX) { data.subType.globalIdTableEntry3FNDX.indexCopyFromStart = deserializeLittleEndianInt(); @@ -477,16 +528,17 @@ private FileNode deserializeFileNode(FileNode data, FileNodePtr curPath) data.subType.globalIdTableEntry3FNDX.indexCopyToStart = deserializeLittleEndianInt(); - ExtendedGUID dependent_revision = - document.revisionMap.get(document.currentRevision).dependent; - for (int i = 0; i < data.subType.globalIdTableEntry3FNDX.entriesToCopy; ++i) { - Map globalIdMap = document.revisionMap.get(dependent_revision).globalId; - GUID compactId = globalIdMap.get( + Revision currentRevision = getCurrentRevision(); + Revision dependentRevision = getRevision(currentRevision.dependent); + int entriesToCopy = checkedCount( + data.subType.globalIdTableEntry3FNDX.entriesToCopy, "global ID entries"); + for (int i = 0; i < entriesToCopy; ++i) { + GUID compactId = dependentRevision.globalId.get( data.subType.globalIdTableEntry3FNDX.indexCopyFromStart + i); if (compactId == null) { throw new TikaException("COMPACT_ID_MISSING"); } - document.revisionMap.get(document.currentRevision).globalId.put( + currentRevision.globalId.put( data.subType.globalIdTableEntry3FNDX.indexCopyToStart + i, compactId); } } else if (data.id == FndStructureConstants.CanRevise.ObjectRevisionWithRefCountFNDX || @@ -623,9 +675,9 @@ private FileNode deserializeFileNode(FileNode data, FileNodePtr curPath) "Data out of bounds - cch " + cch + " is > room left = " + roomLeftLong); } - if (cch > dif.size()) { + if (cch > dif.size() || cch > Integer.MAX_VALUE / 2L) { throw new TikaMemoryLimitException( - "CCH=" + cch + " was found that was greater" + " than file size " + + "CCH=" + cch + " cannot be safely allocated for file size " + dif.size()); } ByteBuffer dataSpaceBuffer = ByteBuffer.allocate((int) cch * 2); @@ -816,7 +868,7 @@ private CompactID deserializeCompactID() throws IOException, TikaException { compactID.guid = ExtendedGUID.nil(); compactID.guid.n = compactID.n; long index = compactID.guidIndex; - Map globalIdMap = document.revisionMap.get(document.currentRevision).globalId; + Map globalIdMap = getCurrentRevision().globalId; GUID guid = globalIdMap.get(index); if (guid != null) { compactID.guid.guid = guid; @@ -976,22 +1028,69 @@ private void postprocessObjectDeclarationContents(FileNode data, FileNodePtr cur } } + private Revision getCurrentRevision() throws TikaException { + Revision revision = document.revisionMap.get(document.currentRevision); + if (revision == null) { + throw new TikaException("Missing current revision"); + } + return revision; + } + + private Revision getRevision(ExtendedGUID revisionId) throws TikaException { + Revision revision = document.revisionMap.get(revisionId); + if (revision == null) { + throw new TikaException("Missing dependent revision"); + } + return revision; + } + + private int checkedCount(long count, String description) throws TikaException { + if (count < 0 || count > MAX_PROPERTY_VALUES) { + throw new TikaMemoryLimitException( + description + " count " + count + " exceeds limit " + MAX_PROPERTY_VALUES); + } + return (int) count; + } + + private int reservePropertyCount(long count, String description) throws TikaException { + int checked = checkedCount(count, description); + if (count > propertyValueBudget.remaining) { + throw new TikaMemoryLimitException(description + " count " + count + + " exceeds remaining property value budget " + propertyValueBudget.remaining); + } + propertyValueBudget.remaining -= count; + return checked; + } + private PropertySet deserializePropertySet(ObjectStreamCounters counters, ObjectSpaceObjectPropSet streams) throws IOException, TikaException { + return deserializePropertySet(counters, streams, 0); + } + + private PropertySet deserializePropertySet(ObjectStreamCounters counters, + ObjectSpaceObjectPropSet streams, + int depth) + throws IOException, TikaException { + if (depth >= MAX_PROPERTY_SET_DEPTH) { + throw new TikaException("Property set nesting exceeds limit " + + MAX_PROPERTY_SET_DEPTH); + } PropertySet data = new PropertySet(); - long count = deserializeLittleEndianShort(); - data.rgPridsData = - Stream.generate(PropertyValue::new).limit((int) count).collect(Collectors.toList()); + // Reserve the count before allocating any PropertyValue instances. + int count = reservePropertyCount(deserializeLittleEndianShort(), "property"); + data.rgPridsData = new ArrayList<>(); for (int i = 0; i < count; ++i) { - data.rgPridsData.get(i).propertyId = deserializePropertyID(); - LOG.debug("{}Property {}", getIndent(), data.rgPridsData.get(i).propertyId); + PropertyValue propertyValue = new PropertyValue(); + propertyValue.propertyId = deserializePropertyID(); + data.rgPridsData.add(propertyValue); + LOG.debug("{}Property {}", getIndent(), propertyValue.propertyId); } LOG.debug("{}{} elements in property set:", getIndent(), count); for (int i = 0; i < count; ++i) { data.rgPridsData.set(i, deserializePropertyValueFromPropertyID(data.rgPridsData.get(i).propertyId, - streams, counters)); + streams, counters, depth)); } LOG.debug(""); return data; @@ -1000,8 +1099,13 @@ private PropertySet deserializePropertySet(ObjectStreamCounters counters, private PropertyValue deserializePropertyValueFromPropertyID(OneNotePropertyId propertyID, ObjectSpaceObjectPropSet streams, - ObjectStreamCounters counters) + ObjectStreamCounters counters, + int propertySetDepth) throws IOException, TikaException { + if (propertySetDepth >= MAX_PROPERTY_SET_DEPTH) { + throw new TikaException("Property value nesting exceeds limit " + + MAX_PROPERTY_SET_DEPTH); + } PropertyValue data = new PropertyValue(); data.propertyId = propertyID; char val8; @@ -1119,22 +1223,18 @@ private PropertyValue deserializePropertyValueFromPropertyID(OneNotePropertyId p val32 = deserializeLittleEndianInt(); OneNotePropertyId propId = deserializePropertyID(); LOG.debug(" UnifiedSubPropertySet {} {}", val32, propId); - data.propertySet.rgPridsData = - Stream.generate(PropertyValue::new).limit((int) val32) - .collect(Collectors.toList()); - for (int i = 0; i < val32; ++i) { - try { - data.propertySet.rgPridsData.set(i, - deserializePropertyValueFromPropertyID(propId, streams, - counters)); - } catch (IOException e) { - return data; - } + int propertyCount = reservePropertyCount(val32, "unified property"); + data.propertySet.rgPridsData = new ArrayList<>(); + for (int i = 0; i < propertyCount; ++i) { + data.propertySet.rgPridsData.add( + deserializePropertyValueFromPropertyID(propId, streams, counters, + propertySetDepth + 1)); } break; case 0x11: LOG.debug(" SubPropertySet"); - data.propertySet = deserializePropertySet(counters, streams); + data.propertySet = deserializePropertySet(counters, streams, + propertySetDepth + 1); break; default: throw new TikaException("Invalid type: " + type); @@ -1185,7 +1285,8 @@ private ObjectSpaceObjectStreamOfOIDsOSIDsOrContextIDs deserializeObjectSpaceObj getIndent(), data.count, data.osidsStreamNotPresent, data.extendedStreamsPresent); } - for (int i = 0; i < data.count; ++i) { + int count = checkedCount(data.count, "object stream"); + for (int i = 0; i < count; ++i) { CompactID cid; cid = deserializeCompactID(); data.data.add(cid); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java index 452b48089ec..939d36bbc09 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -35,6 +36,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; import org.xml.sax.helpers.AttributesImpl; @@ -44,6 +47,7 @@ import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.EmbeddedContentHandler; import org.apache.tika.sax.XHTMLContentHandler; @@ -54,6 +58,10 @@ */ class OneNoteTreeWalker { + private static final Logger LOG = LoggerFactory.getLogger(OneNoteTreeWalker.class); + private static final int MAX_CYCLE_WARNINGS = 100; + // low enough that the ~3 stack frames per level cannot overflow a default thread stack + private static final int MAX_WALK_DEPTH = 500; private static final String P = "p"; /** * See spec MS-ONE - 2.3.1 - TIME32 - epoch of jan 1 1980 UTC. @@ -102,6 +110,14 @@ class OneNoteTreeWalker { * Contains pairs of {Offset,Length} that we have added to the text stream already. */ private final Set> textAlreadyFetched = new HashSet<>(); + private final Set activeFileNodes = + Collections.newSetFromMap(new IdentityHashMap<>()); + private final Set reportedCycleNodes = + Collections.newSetFromMap(new IdentityHashMap<>()); + private int cycleWarningCount; + private boolean cycleWarningsSuppressed; + private int walkDepth; + private boolean depthWarningRecorded; /** * Create a one tree walker. @@ -290,6 +306,68 @@ public Map walkFileNodeList(FileNodeList fileNodeList, OneNotePr public Map walkFileNode(FileNode fileNode, OneNotePropertyId parentPropertyId) throws IOException, TikaException, SAXException { + if (fileNode == null) { + return Collections.emptyMap(); + } + if (walkDepth >= MAX_WALK_DEPTH) { + // the cycle guard cannot catch a long acyclic chain - cap the recursion depth + // so a crafted file cannot overflow the stack + recordDepthWarning(); + return Collections.emptyMap(); + } + if (!activeFileNodes.add(fileNode)) { + recordCycleWarning(fileNode); + return Collections.emptyMap(); + } + walkDepth++; + try { + return walkFileNodeInternal(fileNode, parentPropertyId); + } finally { + walkDepth--; + activeFileNodes.remove(fileNode); + } + } + + private void recordDepthWarning() { + if (depthWarningRecorded) { + return; + } + depthWarningRecorded = true; + String warning = "OneNote file-node traversal exceeded depth limit " + MAX_WALK_DEPTH; + LOG.warn(warning); + if (parentMetadata != null) { + parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + } + } + + private void recordCycleWarning(FileNode fileNode) { + if (reportedCycleNodes.contains(fileNode)) { + return; + } + if (cycleWarningCount >= MAX_CYCLE_WARNINGS) { + if (!cycleWarningsSuppressed) { + cycleWarningsSuppressed = true; + String warning = "Additional OneNote file-node cycle warnings were suppressed " + + "after " + MAX_CYCLE_WARNINGS + " distinct cycles"; + LOG.warn(warning); + if (parentMetadata != null) { + parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + } + } + return; + } + reportedCycleNodes.add(fileNode); + cycleWarningCount++; + String warning = "OneNote file-node cycle detected at " + fileNode.gosid; + LOG.warn(warning); + if (parentMetadata != null) { + parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + } + } + + private Map walkFileNodeInternal(FileNode fileNode, + OneNotePropertyId parentPropertyId) + throws IOException, TikaException, SAXException { Map structure = new HashMap<>(); structure.put("oneNoteType", "FileNode"); structure.put("gosid", fileNode.gosid.toString()); @@ -312,7 +390,7 @@ public Map walkFileNode(FileNode fileNode, if (fileNode.subType.fileDataStoreObjectReference.ref != null && !FileChunkReference.nil() .equals(fileNode.subType.fileDataStoreObjectReference.ref.fileData)) { structure.put("fileDataStoreObjectReference", walkFileDataStoreObjectReference( - fileNode.subType.fileDataStoreObjectReference)); + fileNode.subType.fileDataStoreObjectReference, fileNode.gosid)); } return structure; } @@ -325,7 +403,7 @@ public Map walkFileNode(FileNode fileNode, * @throws IOException Can throw these when manipulating the seekable byte channel. */ private Map walkFileDataStoreObjectReference( - FileDataStoreObjectReference fileDataStoreObjectReference) + FileDataStoreObjectReference fileDataStoreObjectReference, ExtendedGUID gosid) throws IOException, SAXException, TikaException { Map structure = new HashMap<>(); OneNotePtr content = new OneNotePtr(oneNoteDocument, dif); @@ -335,12 +413,13 @@ private Map walkFileDataStoreObjectReference( "File data store cb " + fileDataStoreObjectReference.ref.fileData.cb + " exceeds document size: " + dif.size()); } - handleEmbedded((int) fileDataStoreObjectReference.ref.fileData.cb); + handleEmbedded((int) fileDataStoreObjectReference.ref.fileData.cb, gosid); structure.put("fileDataStoreObjectMetadata", fileDataStoreObjectReference); return structure; } - private void handleEmbedded(int length) throws TikaException, IOException, SAXException { + private void handleEmbedded(int length, ExtendedGUID gosid) + throws TikaException, IOException, SAXException { TikaInputStream stream = null; ByteBuffer buf; try { @@ -352,14 +431,21 @@ private void handleEmbedded(int length) throws TikaException, IOException, SAXEx return; } Metadata embeddedMetadata = new Metadata(); + embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, + TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString()); + if (gosid != null && !ExtendedGUID.nil().equals(gosid)) { + embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, gosid.toString()); + } try { AttributesImpl attributes = new AttributesImpl(); attributes.addAttribute("", "class", "class", "CDATA", "embedded"); xhtml.startElement("div", attributes); xhtml.endElement("div"); - stream = TikaInputStream.get(buf.array()); - embeddedDocumentExtractor.parseEmbedded(stream, new EmbeddedContentHandler(xhtml), - embeddedMetadata, false); + if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) { + stream = TikaInputStream.get(buf.array()); + embeddedDocumentExtractor.parseEmbedded(stream, new EmbeddedContentHandler(xhtml), + embeddedMetadata, false); + } } finally { IOUtils.closeQuietly(stream); } @@ -471,8 +557,11 @@ private Map processPropertyValue(PropertyValue propertyValue, propMap.put("dataUnicode16LE", new String(buf.array(), StandardCharsets.UTF_16LE)); if (options.getUtf16PropertiesToPrint().contains(propertyValue.propertyId.propertyEnum)) { xhtml.startElement(P); - xhtml.characters((String) propMap.get("dataUnicode16LE")); - xhtml.endElement(P); + try { + xhtml.characters((String) propMap.get("dataUnicode16LE")); + } finally { + xhtml.endElement(P); + } } } else if (propertyValue.propertyId.propertyEnum == OneNotePropertyEnum.TextExtendedAscii) { @@ -485,8 +574,11 @@ private Map processPropertyValue(PropertyValue propertyValue, dif.read(buf); propMap.put("dataAscii", new String(buf.array(), StandardCharsets.US_ASCII)); xhtml.startElement(P); - xhtml.characters((String) propMap.get("dataAscii")); - xhtml.endElement(P); + try { + xhtml.characters((String) propMap.get("dataAscii")); + } finally { + xhtml.endElement(P); + } } else if (!isBinary) { if (content.size() > dif.size()) { throw new TikaMemoryLimitException( @@ -498,8 +590,11 @@ private Map processPropertyValue(PropertyValue propertyValue, propMap.put("dataUnicode16LE", new String(buf.array(), StandardCharsets.UTF_16LE)); if (options.getUtf16PropertiesToPrint().contains(propertyValue.propertyId.propertyEnum)) { xhtml.startElement(P); - xhtml.characters((String) propMap.get("dataUnicode16LE")); - xhtml.endElement(P); + try { + xhtml.characters((String) propMap.get("dataUnicode16LE")); + } finally { + xhtml.endElement(P); + } } } else { if (content.size() > dif.size()) { @@ -591,12 +686,18 @@ private void handleRichEditTextUnicode(int length) Matcher m = HYPERLINK_PATTERN.matcher(txt); if (m.find()) { xhtml.startElement("a", "href", m.group(1)); - xhtml.characters(m.group(2)); - xhtml.endElement("a"); + try { + xhtml.characters(m.group(2)); + } finally { + xhtml.endElement("a"); + } } else { xhtml.startElement(P); - xhtml.characters(txt); - xhtml.endElement(P); + try { + xhtml.characters(txt); + } finally { + xhtml.endElement(P); + } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java index 3b8ad38227d..f459522ba91 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java @@ -25,27 +25,44 @@ import java.time.Month; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum; import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.EightBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.FourBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestDataElementData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; @@ -59,6 +76,7 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; import org.apache.tika.parser.microsoft.onenote.fsshttpb.unsigned.Unsigned; import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.BitConverter; +import org.apache.tika.sax.EmbeddedContentHandler; import org.apache.tika.sax.XHTMLContentHandler; public class MSOneStorePackage { @@ -76,7 +94,11 @@ public class MSOneStorePackage { private static final long DATETIME_EPOCH_DIFF_1601; private static final Pattern HYPERLINK_PATTERN = Pattern.compile("\uFDDFHYPERLINK\\s+\"([^\"]+)\"([^\"]+)$"); + private static final Logger LOG = LoggerFactory.getLogger(MSOneStorePackage.class); private static final String P = "p"; + private static final int MAX_OBJECT_WALK_DEPTH = 1000; + private static final int MAX_REFERENCE_COUNT = 100000; + private static final int MAX_PARSE_WARNINGS = 100; static { LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY, 1, 0, 0); @@ -93,6 +115,10 @@ public class MSOneStorePackage { private final Set authors = new HashSet<>(); private final Set mostRecentAuthors = new HashSet<>(); private final Set originalAuthors = new HashSet<>(); + /** + * The fully populated storage index. Set this before performing storage-index lookups; its + * mapping lists are indexed once and must not be mutated afterward. + */ public StorageIndexDataElementData storageIndex; public StorageManifestDataElementData storageManifest; public CellManifestDataElementData headerCellCellManifest; @@ -100,52 +126,75 @@ public class MSOneStorePackage { public List revisionManifests; public List cellManifests; public HeaderCell headerCell; - public List dataRoot; + public RevisionStoreCell dataRootCell; public List OtherFileNodeList; - private boolean mostRecentAuthorProp = false; - private boolean originalAuthorProp = false; + /** + * The content cells (object spaces, e.g. pages), each with its object groups and the + * root object declarations of its current revision. + */ + public List cells; private Instant lastModifiedTimestamp = Instant.MIN; private long creationTimestamp = Long.MAX_VALUE; private long lastModified = Long.MIN_VALUE; + private ParseContext parseContext; + private EmbeddedDocumentExtractor embeddedDocumentExtractor; + private Metadata parentMetadata; + private final List parseWarnings = new ArrayList<>(); + // This state intentionally spans parser construction and tree walking for one package. + private final Set recordedParseWarningKeys = new HashSet<>(); + private boolean parseWarningsSuppressed; + private boolean storageMappingsIndexed; + private boolean contentEmitted; + // flattened property actions per object; objects can be re-flattened many times during + // picture/resource-name resolution, which is quadratic without this cache + private final Map> objectActionsCache = + new IdentityHashMap<>(); + private final Map storageIndexCellMappingsById = + new HashMap<>(); + private final Map storageIndexRevisionMappingsById = + new HashMap<>(); public MSOneStorePackage() { this.revisionManifests = new ArrayList<>(); this.cellManifests = new ArrayList<>(); this.OtherFileNodeList = new ArrayList<>(); + this.cells = new ArrayList<>(); } /** * This method is used to find the Storage Index Cell Mapping matches the Cell ID. * * @param cellID Specify the Cell ID. - * @return Return the specific Storage Index Cell Mapping. + * @return the specific Storage Index Cell Mapping, or {@code null} if it is absent. */ public StorageIndexCellMapping findStorageIndexCellMapping(CellID cellID) { - StorageIndexCellMapping storageIndexCellMapping = null; - if (this.storageIndex != null) { - storageIndexCellMapping = this.storageIndex.storageIndexCellMappingList.stream() - .filter(s -> s.cellID.equals(cellID)).findFirst() - .orElse(new StorageIndexCellMapping()); - } - return storageIndexCellMapping; + indexStorageMappings(); + return storageIndexCellMappingsById.get(cellID); } /** * This method is used to find the Storage Index Revision Mapping that matches the Revision Mapping Extended GUID. * * @param revisionExtendedGUID Specify the Revision Mapping Extended GUID. - * @return Return the instance of Storage Index Revision Mapping. + * @return the instance of Storage Index Revision Mapping, or {@code null} if it is absent. */ public StorageIndexRevisionMapping findStorageIndexRevisionMapping( ExGuid revisionExtendedGUID) { - StorageIndexRevisionMapping instance = null; - if (this.storageIndex != null) { - instance = this.storageIndex.storageIndexRevisionMappingList.stream() - .filter(r -> r.revisionExGuid.equals(revisionExtendedGUID)).findFirst() - .orElse(new StorageIndexRevisionMapping()); - } + indexStorageMappings(); + return storageIndexRevisionMappingsById.get(revisionExtendedGUID); + } - return instance; + private void indexStorageMappings() { + if (storageMappingsIndexed || storageIndex == null) { + return; + } + storageMappingsIndexed = true; + for (StorageIndexCellMapping mapping : storageIndex.storageIndexCellMappingList) { + storageIndexCellMappingsById.putIfAbsent(mapping.cellID, mapping); + } + for (StorageIndexRevisionMapping mapping : storageIndex.storageIndexRevisionMappingList) { + storageIndexRevisionMappingsById.putIfAbsent(mapping.revisionExGuid, mapping); + } } /** @@ -160,112 +209,810 @@ private boolean propertyIsBinary(OneNotePropertyEnum property) { property == OneNotePropertyEnum.RichEditTextUnicode; } + /** + * The attribution of an Author property, determined by the property through which the + * author object was referenced. + */ + private enum AuthorRole { + NONE, MOST_RECENT, ORIGINAL + } + + private enum ParseWarningKind { + DEFAULT, + OBJECT_REFERENCE_ARRAY_UNAVAILABLE, + OBJECT_REFERENCE_ARRAY_CAPPED, + OBJECT_SPACE_REFERENCE_ARRAY_UNAVAILABLE, + OBJECT_SPACE_REFERENCE_ARRAY_CAPPED + } + public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, - XHTMLContentHandler xhtml) + XHTMLContentHandler xhtml, ParseContext parseContext) throws SAXException, TikaException, IOException { - for (RevisionStoreObjectGroup revisionStoreObjectGroup : OtherFileNodeList) { - for (RevisionStoreObject revisionStoreObject : revisionStoreObjectGroup.objects) { - PropertySet propertySet = - revisionStoreObject.propertySet.objectSpaceObjectPropSet.body; - for (int i = 0; i < propertySet.rgData.size(); ++i) { - IProperty property = propertySet.rgData.get(i); - PropertyID propertyID = propertySet.rgPrids[i]; - PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); - OneNotePropertyEnum oneNotePropertyEnum = - OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); - if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { - long fullval = getScalar(property); - Instant instant = Instant.ofEpochSecond( - fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); - if (instant.isAfter(lastModifiedTimestamp)) { - lastModifiedTimestamp = instant; - } - metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp", - String.valueOf(lastModifiedTimestamp.toEpochMilli())); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long creationTs = scalar + TIME32_EPOCH_DIFF_1980; - if (creationTs < creationTimestamp) { - creationTimestamp = creationTs; - } - metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", String.valueOf(creationTimestamp)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long lastMod = scalar + TIME32_EPOCH_DIFF_1980; - if (lastMod > lastModified) { - lastModified = lastMod; - } - metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { - String author = - new String(((PrtFourBytesOfLengthFollowedByData) property).data, - StandardCharsets.UTF_8); - if (mostRecentAuthorProp) { - mostRecentAuthors.add(author); - } else if (originalAuthorProp) { - originalAuthors.add(author); - } else { - authors.add(author); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { - mostRecentAuthorProp = true; - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { - originalAuthorProp = true; - } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { - boolean isBinary = propertyIsBinary(oneNotePropertyEnum); - PrtFourBytesOfLengthFollowedByData dataProperty = - (PrtFourBytesOfLengthFollowedByData) property; - if ((dataProperty.data.length & 1) == 0 && - oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && - !isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.US_ASCII)); - xhtml.endElement(P); - } else if (!isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else { - if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { - handleRichEditTextUnicode(dataProperty.data, xhtml); - } else { - //TODO -- these seem to be somewhat broken font files and other - //odds and ends...what are they and how should we process them? - //handleEmbedded(content.size()); - } - } - } + this.parseContext = parseContext; + this.parentMetadata = metadata; + this.embeddedDocumentExtractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext); + for (String warning : parseWarnings) { + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + } + parseWarnings.clear(); + if (!cells.isEmpty()) { + // Walk each page cell (object space) as a tree, starting from the root objects of + // its current revision and following the object references in property order. This + // emits the text in document order. The pages are walked in the order in which the + // section object space references them; cells that hold older versions of a page + // (the same object space in a different revision context) are skipped. + List pageCells = new ArrayList<>(); + List otherCells = new ArrayList<>(); + splitCells(pageCells, otherCells); + for (RevisionStoreCell cell : pageCells) { + xhtml.startElement("div", "class", "page"); + try { + walkCell(cell, options, metadata, xhtml); + } finally { + xhtml.endElement("div"); + } + } + for (RevisionStoreCell cell : otherCells) { + walkCell(cell, options, metadata, xhtml); + } + } else { + // no cell information available - walk the object groups in revision order + Map objectsById = indexObjectsById(OtherFileNodeList); + Set visited = new HashSet<>(); + for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, + xhtml, 0); } } } if (!authors.isEmpty()) { - metadata.set(TikaCoreProperties.CREATOR, authors.toArray(new String[]{})); + metadata.set(TikaCoreProperties.CREATOR, sortedValues(authors)); } if (!mostRecentAuthors.isEmpty()) { metadata.set(Property.externalTextBag(ONE_NOTE_PREFIX + "mostRecentAuthors"), - mostRecentAuthors.toArray(new String[]{})); + sortedValues(mostRecentAuthors)); } if (!originalAuthors.isEmpty()) { metadata.set(Property.externalTextBag(ONE_NOTE_PREFIX + "originalAuthors"), - originalAuthors.toArray(new String[]{})); + sortedValues(originalAuthors)); + } + } + + /** + * Splits the cells into page cells, ordered as the section object space references them, + * and the remaining cells. A cell that holds an older version of a page - the same object + * space referenced by a page cell, but in a different revision context - is dropped, so + * content is not emitted once per version snapshot. + */ + private void splitCells(List pageCells, + List otherCells) { + if (dataRootCell == null) { + // Without a data root there is no reliable page ordering information. + otherCells.addAll(cells); + return; + } + List orderedCellIds = collectSectionReferencedCells(); + if (orderedCellIds.isEmpty()) { + // no page ordering information available - process the cells in storage order + otherCells.addAll(cells); + return; + } + Map remainingCells = new LinkedHashMap<>(); + for (RevisionStoreCell cell : cells) { + remainingCells.put(cell.cellID, cell); + } + Set coveredObjectSpaces = new HashSet<>(); + for (CellID cellId : orderedCellIds) { + RevisionStoreCell cell = remainingCells.remove(cellId); + if (cell != null) { + pageCells.add(cell); + if (cellId.extendGUID2 != null) { + coveredObjectSpaces.add(cellId.extendGUID2); + } + } + } + for (RevisionStoreCell cell : remainingCells.values()) { + if (!coveredObjectSpaces.contains(cell.cellID.extendGUID2)) { + // not an older version of one of the pages - keep it so no content is lost + otherCells.add(cell); + } + } + } + + /** + * Walks the section object space (the data root cell) and collects the object space (cell) + * references in document order - this is the order of the pages in the section. + */ + private List collectSectionReferencedCells() { + List orderedCellIds = new ArrayList<>(); + if (dataRootCell == null) { + return orderedCellIds; + } + Map objectsById = indexObjectsById(dataRootCell.objectGroups); + Set visited = new HashSet<>(); + for (RevisionManifestRootDeclare rootDeclare : dataRootCell.rootDeclares) { + RevisionStoreObject rootObject = objectsById.get(rootDeclare.objectExGuid); + if (rootObject == null) { + recordParseWarning("OneNote section root object " + rootDeclare.objectExGuid + + " could not be resolved"); + } + collectReferencedCells(rootObject, objectsById, visited, orderedCellIds, 0); + } + return orderedCellIds; + } + + private void collectReferencedCells(RevisionStoreObject object, + Map objectsById, + Set visited, List out, int depth) { + if (object == null || object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + if (depth >= MAX_OBJECT_WALK_DEPTH) { + recordParseWarning("OneNote section reference traversal exceeded depth limit " + + MAX_OBJECT_WALK_DEPTH); + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + return; + } + List actions = collectObjectActions(object); + for (PropertyAction action : actions) { + if (action.spaceReference != null) { + out.add(action.spaceReference); + } else if (action.isChildReference && action.childReference != null) { + RevisionStoreObject child = objectsById.get(action.childReference); + if (child == null && !warningsSaturated()) { + recordParseWarning("OneNote section object " + action.childReference + + " could not be resolved"); + } + collectReferencedCells(child, objectsById, visited, out, depth + 1); + } + } + } + + private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + Map objectsById = indexObjectsById(cell.objectGroups); + Set visited = new HashSet<>(); + // Only objects reachable from the root objects of the current revision are part of + // the current content. The object groups may also contain older, superseded versions + // of objects (under a different object ID); those are intentionally not walked. + // A blob-only root (no property set) cannot reach the page body, so it does not + // count as a resolved content root: if the content root declare dangles next to it, + // the walk-everything fallback must still fire or the page body is silently lost. + boolean resolvedContentRoot = false; + boolean unresolvedRoot = false; + for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) { + RevisionStoreObject rootObject = objectsById.get(rootDeclare.objectExGuid); + if (rootObject == null) { + recordParseWarning("OneNote cell root object " + rootDeclare.objectExGuid + + " could not be resolved"); + unresolvedRoot = true; + } else { + if (rootObject.propertySet != null) { + resolvedContentRoot = true; + } + walkObject(rootObject, objectsById, visited, AuthorRole.NONE, options, + metadata, xhtml, 0); + } + } + if (cell.rootDeclares.isEmpty() || (unresolvedRoot && !resolvedContentRoot)) { + if (cell.rootDeclares.isEmpty()) { + recordParseWarning("OneNote cell has no declared root objects; walking all objects"); + } else { + recordParseWarning("OneNote cell root objects could not be resolved; walking all objects"); + } + for (RevisionStoreObjectGroup objectGroup : cell.objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, + xhtml, 0); + } + } + } + } + + private String[] sortedValues(Set values) { + String[] sorted = values.toArray(new String[0]); + Arrays.sort(sorted); + return sorted; + } + + private boolean warningsSaturated() { + return parseWarningsSuppressed; + } + + void recordParseWarning(String warning) { + recordParseWarning(ParseWarningKind.DEFAULT, warning); + } + + private void recordParseWarning(ParseWarningKind kind, String warning) { + String warningKey = kind == ParseWarningKind.DEFAULT ? warning : kind.name(); + if (recordedParseWarningKeys.contains(warningKey)) { + return; + } + if (recordedParseWarningKeys.size() >= MAX_PARSE_WARNINGS) { + if (!parseWarningsSuppressed) { + parseWarningsSuppressed = true; + emitParseWarning("Additional OneNote parse warnings were suppressed after " + + MAX_PARSE_WARNINGS + " distinct warnings"); + } + return; + } + recordedParseWarningKeys.add(warningKey); + emitParseWarning(warning); + } + + private void emitParseWarning(String warning) { + LOG.warn(warning); + if (parentMetadata == null) { + parseWarnings.add(warning); + } else { + parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + } + } + + /** + * Builds a map of object ID to object. The object groups are ordered from the oldest + * revision to the newest, so a newer version of an object wins over an older one. + */ + private Map indexObjectsById( + List objectGroups) { + Map objectsById = new HashMap<>(); + for (RevisionStoreObjectGroup objectGroup : objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + if (object.objectID != null) { + objectsById.put(object.objectID, object); + } + } + } + return objectsById; + } + + private void walkObject(RevisionStoreObject object, + Map objectsById, Set visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml, int depth) + throws SAXException, TikaException, IOException { + walkObject(object, objectsById, visited, authorRole, options, metadata, xhtml, + depth, null); + } + + private void walkObject(RevisionStoreObject object, + Map objectsById, Set visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml, int depth, + EmbeddedResourceInfo inheritedResourceInfo) + throws SAXException, TikaException, IOException { + if (object == null) { + return; + } + if (depth >= MAX_OBJECT_WALK_DEPTH) { + recordParseWarning("OneNote object traversal exceeded depth limit " + + MAX_OBJECT_WALK_DEPTH); + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + // one object may be referenced under several author roles (e.g. the same author + // as both AuthorOriginal and AuthorMostRecent) - record the role even though + // the subtree is not walked again + recordAuthors(object, authorRole); + return; + } + List actions = object.propertySet != null && + object.propertySet.objectSpaceObjectPropSet != null ? + collectObjectActions(object) : Collections.emptyList(); + EmbeddedResourceInfo resourceInfo = embeddedResourceInfo(actions); + if (resourceInfo == null) { + resourceInfo = inheritedResourceInfo; + } + if (resourceInfo == null && object.fileDataObject != null && object.objectID != null) { + resourceInfo = resourceInfoFromReferencingObject(object, objectsById, depth); + } + if (object.fileDataObject != null) { + // the object carries opaque binary data, e.g. an embedded image or file + handleEmbedded(object.fileDataObject.getData(), xhtml, resourceInfo, object.objectID); + } + if (object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + // An image node can reference the same picture twice: PictureContainer holds the + // canonical image data and WebPictureContainer14 holds a rendition derived from it + // (e.g. re-rendered when the picture was resized). Only emit the derived rendition + // when the canonical container is missing, so the picture is not extracted twice. + boolean hasPrimaryPicture = false; + for (PropertyAction action : actions) { + if (action.isChildReference && action.childReference != null && + action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer && + hasUsablePicture(action.childReference, objectsById, new HashSet<>(), depth)) { + hasPrimaryPicture = true; + break; + } + } + // The title structure of a page (StructureElementChildNodes) appears above the page + // body on screen, but is declared after the body child nodes. Emit it first so the + // text comes out in visual order. + for (PropertyAction action : actions) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.StructureElementChildNodes) { + processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, + depth, resourceInfo); + } + } + for (PropertyAction action : actions) { + if (hasPrimaryPicture && + action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) { + continue; + } + if (action.oneNotePropertyEnum != OneNotePropertyEnum.StructureElementChildNodes) { + processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, + depth, resourceInfo); + } + } + } + + private boolean hasUsablePicture(ExGuid objectId, + Map objectsById, + Set visited, int depth) { + if (depth >= MAX_OBJECT_WALK_DEPTH || objectId == null || !visited.add(objectId)) { + return false; + } + RevisionStoreObject object = objectsById.get(objectId); + if (object == null) { + return false; + } + byte[] data = object.fileDataObject == null ? null : object.fileDataObject.getData(); + if (data != null && data.length > 0) { + return true; + } + if (object.propertySet == null || object.propertySet.objectSpaceObjectPropSet == null) { + return false; + } + for (PropertyAction action : collectObjectActions(object)) { + if (action.isChildReference && action.childReference != null && + hasUsablePicture(action.childReference, objectsById, visited, depth + 1)) { + return true; + } } + return false; } + private EmbeddedResourceInfo resourceInfoFromReferencingObject( + RevisionStoreObject target, Map objectsById, int depth) { + if (depth >= MAX_OBJECT_WALK_DEPTH) { + return null; + } + EmbeddedResourceInfo typeOnlyFallback = null; + for (RevisionStoreObject candidate : objectsById.values()) { + if (candidate.propertySet == null || + candidate.propertySet.objectSpaceObjectPropSet == null) { + continue; + } + List candidateActions = collectObjectActions(candidate); + for (PropertyAction action : candidateActions) { + if (action.isChildReference && target.objectID.equals(action.childReference) && + (action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer || + action.oneNotePropertyEnum == OneNotePropertyEnum.EmbeddedFileContainer || + action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14)) { + EmbeddedResourceInfo info = embeddedResourceInfo(candidateActions); + if (info != null) { + return info; + } + if (typeOnlyFallback == null) { + typeOnlyFallback = resourceInfoForChild(action, null); + } + } + } + } + return typeOnlyFallback; + } + + private EmbeddedResourceInfo resourceInfoForChild(PropertyAction action, + EmbeddedResourceInfo parentInfo) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer || + action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) { + return parentInfo == null ? new EmbeddedResourceInfo(null, + TikaCoreProperties.EmbeddedResourceType.INLINE.toString()) : parentInfo; + } + if (action.oneNotePropertyEnum == OneNotePropertyEnum.EmbeddedFileContainer) { + return parentInfo == null ? new EmbeddedResourceInfo(null, + TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString()) : parentInfo; + } + return parentInfo; + } + + /** + * Metadata used when emitting an embedded resource. + */ + private static final class EmbeddedResourceInfo { + private final String name; + private final String type; + + private EmbeddedResourceInfo(String name, String type) { + this.name = name; + this.type = type; + } + } + + /** + * A property of an object, together with the object reference assigned to it if it is + * an object reference property. + */ + private static final class PropertyAction { + private final IProperty property; + private final PropertyType propertyType; + private final OneNotePropertyEnum oneNotePropertyEnum; + private final boolean isChildReference; + private final ExGuid childReference; + private final CellID spaceReference; + + PropertyAction(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, boolean isChildReference, + ExGuid childReference, CellID spaceReference) { + this.property = property; + this.propertyType = propertyType; + this.oneNotePropertyEnum = oneNotePropertyEnum; + this.isChildReference = isChildReference; + this.childReference = childReference; + this.spaceReference = spaceReference; + } + } + + /** + * Flattens the properties of an object, in order, into a list of actions. The result is + * cached per object; callers must not mutate the returned list. + */ + private List collectObjectActions(RevisionStoreObject object) { + List actions = objectActionsCache.get(object); + if (actions != null) { + return actions; + } + List referencedObjects = + object.referencedObjectID == null || object.referencedObjectID.content == null ? + Collections.emptyList() : object.referencedObjectID.content; + List referencedSpaces = object.referencedObjectSpacesID == null || + object.referencedObjectSpacesID.content == null ? Collections.emptyList() : + object.referencedObjectSpacesID.content; + actions = new ArrayList<>(); + collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects, + new int[]{0}, referencedSpaces, new int[]{0}, actions, 0); + objectActionsCache.put(object, actions); + return actions; + } + + /** + * Flattens the properties of a property set, in order, into a list of actions. + * Properties of type ObjectID or ArrayOfObjectIDs consume, in property order, the object + * references of the containing object, and properties of type ObjectSpaceID or + * ArrayOfObjectSpaceIDs consume the object space (cell) references (see MS-ONESTORE + * section 2.7.8), so the references must be assigned here, in property order, no matter + * in which order the actions are processed later. + */ + private void collectActions(PropertySet propertySet, List referencedObjects, + int[] referenceCursor, List referencedSpaces, + int[] spaceCursor, List actions, int depth) { + if (propertySet == null || propertySet.rgPrids == null || propertySet.rgData == null) { + return; + } + if (depth >= MAX_OBJECT_WALK_DEPTH) { + recordParseWarning("OneNote property traversal exceeded depth limit " + + MAX_OBJECT_WALK_DEPTH); + return; + } + for (int i = 0; i < propertySet.rgPrids.length && i < propertySet.rgData.size(); ++i) { + IProperty property = propertySet.rgData.get(i); + PropertyID propertyID = propertySet.rgPrids[i]; + PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); + OneNotePropertyEnum oneNotePropertyEnum = + OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); + if (propertyType == PropertyType.ObjectID) { + ExGuid childReference = nextReference(referencedObjects, referenceCursor); + if (childReference == null) { + recordParseWarning("OneNote object reference slot was exhausted"); + } + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, true, + childReference, null)); + } else if (propertyType == PropertyType.ArrayOfObjectIDs) { + int available = referencedObjects.size() - referenceCursor[0]; + int requestedCount = property instanceof ArrayNumber ? + Math.max(0, ((ArrayNumber) property).number) : 0; + int declaredCount = boundedAvailableReferenceCount(requestedCount, available); + int count = Math.min(declaredCount, MAX_REFERENCE_COUNT); + if (requestedCount > available && !warningsSaturated()) { + recordParseWarning(ParseWarningKind.OBJECT_REFERENCE_ARRAY_UNAVAILABLE, + "OneNote object reference array had unavailable entries " + + "(first occurrence: declared " + requestedCount + + " entries but only " + Math.max(available, 0) + + " were available)"); + } + if (declaredCount > count && !warningsSaturated()) { + recordParseWarning(ParseWarningKind.OBJECT_REFERENCE_ARRAY_CAPPED, + "Capping OneNote object reference array at " + + MAX_REFERENCE_COUNT + " entries"); + } + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + true, nextReference(referencedObjects, referenceCursor), null)); + } + referenceCursor[0] += declaredCount - count; + } else if (propertyType == PropertyType.ObjectSpaceID) { + CellID spaceReference = nextSpaceReference(referencedSpaces, spaceCursor); + if (spaceReference == null) { + recordParseWarning("OneNote object-space reference slot was exhausted"); + } + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, spaceReference)); + } else if (propertyType == PropertyType.ArrayOfObjectSpaceIDs) { + int available = referencedSpaces.size() - spaceCursor[0]; + int requestedCount = property instanceof ArrayNumber ? + Math.max(0, ((ArrayNumber) property).number) : 0; + int declaredCount = boundedAvailableReferenceCount(requestedCount, available); + int count = Math.min(declaredCount, MAX_REFERENCE_COUNT); + if (requestedCount > available && !warningsSaturated()) { + recordParseWarning(ParseWarningKind.OBJECT_SPACE_REFERENCE_ARRAY_UNAVAILABLE, + "OneNote object-space reference array had unavailable entries " + + "(first occurrence: declared " + requestedCount + + " entries but only " + Math.max(available, 0) + + " were available)"); + } + if (declaredCount > count && !warningsSaturated()) { + recordParseWarning(ParseWarningKind.OBJECT_SPACE_REFERENCE_ARRAY_CAPPED, + "Capping OneNote object-space reference array at " + + MAX_REFERENCE_COUNT + " entries"); + } + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + false, null, nextSpaceReference(referencedSpaces, spaceCursor))); + } + spaceCursor[0] += declaredCount - count; + } else if (propertyType == PropertyType.PropertySet) { + if (property instanceof PropertySet) { + collectActions((PropertySet) property, referencedObjects, referenceCursor, + referencedSpaces, spaceCursor, actions, depth + 1); + } + } else if (propertyType == PropertyType.ArrayOfPropertyValues) { + if (property instanceof PrtArrayOfPropertyValues && + ((PrtArrayOfPropertyValues) property).data != null) { + for (PropertySet nested : ((PrtArrayOfPropertyValues) property).data) { + collectActions(nested, referencedObjects, referenceCursor, + referencedSpaces, spaceCursor, actions, depth + 1); + } + } + } else { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, null)); + } + } + } + + private void processAction(PropertyAction action, + Map objectsById, Set visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml, int depth, + EmbeddedResourceInfo parentResourceInfo) + throws SAXException, TikaException, IOException { + if (action.spaceReference != null) { + // a reference to another object space (cell) - cells are walked separately + return; + } + if (action.isChildReference) { + AuthorRole childRole = AuthorRole.NONE; + if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { + childRole = AuthorRole.MOST_RECENT; + } else if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { + childRole = AuthorRole.ORIGINAL; + } + EmbeddedResourceInfo childResourceInfo = resourceInfoForChild(action, + parentResourceInfo); + RevisionStoreObject child = action.childReference == null ? null : + objectsById.get(action.childReference); + if (child == null && action.childReference != null && !warningsSaturated()) { + recordParseWarning("OneNote object " + action.childReference + + " could not be resolved"); + } + walkObject(child, objectsById, visited, childRole, options, metadata, xhtml, + depth + 1, childResourceInfo); + } else { + processPrimitiveProperty(action.property, action.propertyType, + action.oneNotePropertyEnum, authorRole, options, metadata, xhtml); + } + } + + private int boundedAvailableReferenceCount(int declaredCount, int remainingReferences) { + if (declaredCount <= 0 || remainingReferences <= 0) { + return 0; + } + return Math.min(declaredCount, remainingReferences); + } + + private ExGuid nextReference(List referencedObjects, int[] referenceCursor) { + if (referenceCursor[0] < referencedObjects.size()) { + return referencedObjects.get(referenceCursor[0]++); + } + return null; + } + + private CellID nextSpaceReference(List referencedSpaces, int[] spaceCursor) { + if (spaceCursor[0] < referencedSpaces.size()) { + return referencedSpaces.get(spaceCursor[0]++); + } + return null; + } + + private void processPrimitiveProperty(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, + AuthorRole authorRole, + OneNoteTreeWalkerOptions options, Metadata metadata, + XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { + long fullval = getScalar(property); + Instant instant = Instant.ofEpochSecond( + fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); + if (instant.isAfter(lastModifiedTimestamp)) { + lastModifiedTimestamp = instant; + } + metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp", + String.valueOf(lastModifiedTimestamp.toEpochMilli())); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long creationTs = scalar + TIME32_EPOCH_DIFF_1980; + if (creationTs < creationTimestamp) { + creationTimestamp = creationTs; + } + metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", String.valueOf(creationTimestamp)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long lastMod = scalar + TIME32_EPOCH_DIFF_1980; + if (lastMod > lastModified) { + lastModified = lastMod; + } + metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { + recordAuthor(decodeOneNoteText( + ((PrtFourBytesOfLengthFollowedByData) property).data), authorRole); + } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { + boolean isBinary = propertyIsBinary(oneNotePropertyEnum); + PrtFourBytesOfLengthFollowedByData dataProperty = + (PrtFourBytesOfLengthFollowedByData) property; + if ((dataProperty.data.length & 1) == 0 && + oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + emitParagraph(xhtml, new String(dataProperty.data, + StandardCharsets.UTF_16LE)); + } + } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { + emitParagraph(xhtml, new String(dataProperty.data, StandardCharsets.US_ASCII)); + } else if (!isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + emitParagraph(xhtml, new String(dataProperty.data, + StandardCharsets.UTF_16LE)); + } + } else { + if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { + handleRichEditTextUnicode(dataProperty.data, xhtml); + } else { + //TODO -- these seem to be somewhat broken font files and other + //odds and ends...what are they and how should we process them? + //handleEmbedded(content.size()); + } + } + } + } + + + private String decodeOneNoteText(byte[] bytes) { + return new String(bytes, StandardCharsets.UTF_16LE).replace("\u0000", ""); + } + + private void recordAuthor(String author, AuthorRole role) { + if (role == AuthorRole.MOST_RECENT) { + mostRecentAuthors.add(author); + } else if (role == AuthorRole.ORIGINAL) { + originalAuthors.add(author); + // the original authors are the creators of the content + authors.add(author); + } else { + authors.add(author); + } + } + + /** + * Records the Author properties of an already-visited object under the given role. + */ + private void recordAuthors(RevisionStoreObject object, AuthorRole role) { + // NONE also records (into authors) so the result is visit-order independent + if (object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + for (PropertyAction action : collectObjectActions(object)) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.Author && + action.property instanceof PrtFourBytesOfLengthFollowedByData) { + recordAuthor(decodeOneNoteText( + ((PrtFourBytesOfLengthFollowedByData) action.property).data), role); + } + } + } + + private EmbeddedResourceInfo embeddedResourceInfo(List actions) { + for (PropertyAction action : actions) { + if (action.property instanceof PrtFourBytesOfLengthFollowedByData && + (action.oneNotePropertyEnum == OneNotePropertyEnum.ImageFilename || + action.oneNotePropertyEnum == OneNotePropertyEnum.EmbeddedFileName)) { + byte[] bytes = ((PrtFourBytesOfLengthFollowedByData) action.property).data; + String name = sanitizeResourceName(new String(bytes, StandardCharsets.UTF_16LE) + .replace("\u0000", "")); + if (!name.isEmpty()) { + String type = action.oneNotePropertyEnum == OneNotePropertyEnum.ImageFilename ? + TikaCoreProperties.EmbeddedResourceType.INLINE.toString() : + TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString(); + return new EmbeddedResourceInfo(name, type); + } + } + } + return null; + } + + static String sanitizeResourceName(String name) { + name = name.replace('\\', '/'); + // a drive-relative name like C:pic.png has no slash, so strip the prefix separately + if (name.length() > 1 && name.charAt(1) == ':') { + name = name.substring(2); + } + int slash = name.lastIndexOf('/'); + name = slash >= 0 ? name.substring(slash + 1) : name; + return ".".equals(name) || "..".equals(name) ? "" : name; + } + + /** + * Hands the binary data of an embedded object (e.g. an image or an attached file) to the + * embedded document extractor. + */ + private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml, + EmbeddedResourceInfo resourceInfo, ExGuid objectID) + throws SAXException, IOException { + if (data == null || data.length == 0 || embeddedDocumentExtractor == null) { + return; + } + Metadata embeddedMetadata = new Metadata(); + embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, + resourceInfo == null ? TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString() : + resourceInfo.type); + if (resourceInfo != null && resourceInfo.name != null) { + embeddedMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, resourceInfo.name); + } + String relationshipID = objectID == null ? null : "_" + objectID.value + "_" + + objectID.guid; + if (relationshipID != null) { + embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, relationshipID); + } + AttributesImpl attributes = new AttributesImpl(); + attributes.addAttribute("", "class", "class", "CDATA", "embedded"); + if (relationshipID != null) { + attributes.addAttribute("", "id", "id", "CDATA", relationshipID); + } + xhtml.startElement("div", attributes); + xhtml.endElement("div"); + try (TikaInputStream tis = TikaInputStream.get(data)) { + if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) { + // only counts as content when the extractor accepts it, so a declined + // embedded object in an otherwise-empty file still triggers the fallback + contentEmitted = true; + embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml), + embeddedMetadata, false); + } + } catch (IOException e) { + EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata); + } + } private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml) throws SAXException, IOException, TikaException { @@ -285,13 +1032,37 @@ private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml) Matcher m = HYPERLINK_PATTERN.matcher(txt); if (m.find()) { xhtml.startElement("a", "href", m.group(1)); - xhtml.characters(m.group(2)); - xhtml.endElement("a"); + try { + xhtml.characters(m.group(2)); + } finally { + xhtml.endElement("a"); + } + contentEmitted = true; } else { - xhtml.startElement(P); - xhtml.characters(txt); + emitParagraph(xhtml, txt); + } + } + + private void emitParagraph(XHTMLContentHandler xhtml, String text) throws SAXException { + xhtml.startElement(P); + try { + xhtml.characters(text); + } finally { xhtml.endElement(P); } + // blank text does not count as content, so it cannot suppress the legacy fallback + if (!text.isBlank()) { + contentEmitted = true; + } + } + + /** + * Whether the tree walk emitted any content - text or an embedded object. When it did + * not, the caller can fall back to the legacy string dump so a degraded file still + * yields its text. + */ + public boolean hasEmittedContent() { + return contentEmitted; } private long getScalar(IProperty property) throws TikaException, IOException { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java index 25790ead8b2..d4a41b43d0b 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java @@ -17,9 +17,15 @@ package org.apache.tika.parser.microsoft.onenote.fsshttpb; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -31,6 +37,8 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestObjectGroupReferences; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexDataElementData; @@ -40,8 +48,14 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.HeaderCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.GuidUtil; public class MSOneStoreParser { + /** + * The root role declaration used for the encryption key of encrypted sections. + */ + private static final ExGuid ENCRYPTION_KEY_ROOT_EXGUID = + new ExGuid(3, UUID.fromString("4A3717F8-1C14-49E7-9526-81D942DE1741")); private final Set storageIndexHashTab = new HashSet<>(); // The DataElements of Storage Index private List storageIndexDataElements; @@ -49,15 +63,47 @@ public class MSOneStoreParser { private List storageManifestDataElements; // The DataElements of Cell Manifest private List cellManifestDataElements; + private Map cellManifestDataElementsById = new HashMap<>(); // The DataElements of Revision Manifest private List revisionManifestDataElements; + private Map revisionManifestDataElementsById = new HashMap<>(); // The DataElements of Object Group Data private List objectGroupDataElements; - // The DataElements of Object BLOB - private List objectBlOBElements; + // The DataElements of Object Group Data, keyed by their data element extended GUID + private Map objectGroupDataElementsById; + private final Map objectGroupsByDataElementId = + new HashMap<>(); + // The DataElements of Object BLOB, keyed by their data element extended GUID + private Map objectBlobElementsById; + + private static final class ObjectGroupCacheKey { + private final ExGuid dataElementId; + private final boolean encryption; + + private ObjectGroupCacheKey(ExGuid dataElementId, boolean encryption) { + this.dataElementId = dataElementId; + this.encryption = encryption; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ObjectGroupCacheKey)) { + return false; + } + ObjectGroupCacheKey that = (ObjectGroupCacheKey) other; + return encryption == that.encryption && Objects.equals(dataElementId, that.dataElementId); + } + + @Override + public int hashCode() { + return 31 * Objects.hashCode(dataElementId) + Boolean.hashCode(encryption); + } + } public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOException { + storageIndexHashTab.clear(); MSOneStorePackage msOneStorePackage = new MSOneStorePackage(); + objectGroupsByDataElementId.clear(); storageIndexDataElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.StorageIndexDataElementData) @@ -68,20 +114,48 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE cellManifestDataElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.CellManifestDataElementData) .collect(Collectors.toList()); + cellManifestDataElementsById.clear(); + for (DataElement element : cellManifestDataElements) { + cellManifestDataElementsById.putIfAbsent(element.dataElementExGuid, element); + } revisionManifestDataElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.RevisionManifestDataElementData) .collect(Collectors.toList()); + revisionManifestDataElementsById.clear(); + for (DataElement element : revisionManifestDataElements) { + revisionManifestDataElementsById.putIfAbsent(element.dataElementExGuid, element); + } objectGroupDataElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.ObjectGroupDataElementData) .collect(Collectors.toList()); - objectBlOBElements = dataElementPackage.dataElements.stream() + objectGroupDataElementsById = new HashMap<>(); + for (DataElement objectGroupElement : objectGroupDataElements) { + objectGroupDataElementsById.putIfAbsent(objectGroupElement.dataElementExGuid, + objectGroupElement); + } + List objectBlobElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.ObjectDataBLOBDataElementData) .collect(Collectors.toList()); + objectBlobElementsById = new HashMap<>(); + for (DataElement blobElement : objectBlobElements) { + objectBlobElementsById.putIfAbsent(blobElement.dataElementExGuid, blobElement); + } + if (storageIndexDataElements.isEmpty() || storageManifestDataElements.isEmpty()) { + throw new IOException("Missing OneNote storage index or storage manifest"); + } msOneStorePackage.storageIndex = (StorageIndexDataElementData) storageIndexDataElements.get(0).data; msOneStorePackage.storageManifest = (StorageManifestDataElementData) storageManifestDataElements.get(0).data; + if (msOneStorePackage.storageIndex == null || + msOneStorePackage.storageManifest == null) { + throw new IOException("Invalid OneNote storage index or storage manifest"); + } + if (msOneStorePackage.storageManifest.storageManifestRootDeclareList == null || + msOneStorePackage.storageManifest.storageManifestRootDeclareList.size() < 2) { + throw new IOException("Missing OneNote header or data-root declaration"); + } // Parse Header Cell CellID headerCellID = @@ -93,31 +167,50 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE if (headerCellStorageIndexCellMapping != null) { msOneStorePackage.headerCellCellManifest = this.findCellManifest(headerCellStorageIndexCellMapping.cellMappingExGuid); - StorageIndexRevisionMapping headerCellRevisionManifestMapping = - msOneStorePackage.findStorageIndexRevisionMapping( - msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision - .cellManifestCurrentRevisionExGuid); - msOneStorePackage.headerCellRevisionManifest = this.findRevisionManifestDataElement( - headerCellRevisionManifestMapping.revisionMappingExGuid); - msOneStorePackage.headerCell = - this.parseHeaderCell(msOneStorePackage.headerCellRevisionManifest); - - // Parse Data root - CellID dataRootCellID = - msOneStorePackage.storageManifest.storageManifestRootDeclareList.get(1).cellID; - storageIndexHashTab.add(dataRootCellID); - msOneStorePackage.dataRoot = this.parseObjectGroup(dataRootCellID, msOneStorePackage); - // Parse other data - for (StorageIndexCellMapping storageIndexCellMapping : msOneStorePackage.storageIndex - .storageIndexCellMappingList) { - if (!storageIndexHashTab.contains(storageIndexCellMapping.cellID)) { - msOneStorePackage.OtherFileNodeList.addAll( - this.parseObjectGroup(storageIndexCellMapping.cellID, - msOneStorePackage)); - storageIndexHashTab.add(storageIndexCellMapping.cellID); + if (msOneStorePackage.headerCellCellManifest != null && + msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision != null) { + StorageIndexRevisionMapping headerCellRevisionManifestMapping = + msOneStorePackage.findStorageIndexRevisionMapping( + msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision + .cellManifestCurrentRevisionExGuid); + if (headerCellRevisionManifestMapping != null) { + msOneStorePackage.headerCellRevisionManifest = + this.findRevisionManifestDataElement( + headerCellRevisionManifestMapping.revisionMappingExGuid); + if (msOneStorePackage.headerCellRevisionManifest != null) { + msOneStorePackage.headerCell = + this.parseHeaderCell(msOneStorePackage.headerCellRevisionManifest); + } } } } + + // Parse Data root independently of the header-cell metadata. A malformed header cell + // should not prevent valid section cells from being parsed. + CellID dataRootCellID = + msOneStorePackage.storageManifest.storageManifestRootDeclareList.get(1).cellID; + storageIndexHashTab.add(dataRootCellID); + RevisionStoreCell dataRootCell = this.parseCell(dataRootCellID, msOneStorePackage); + if (dataRootCell == null) { + throw new IOException("Could not parse the OneNote data root cell"); + } + msOneStorePackage.dataRootCell = dataRootCell; + // Parse other data + for (StorageIndexCellMapping storageIndexCellMapping : msOneStorePackage.storageIndex + .storageIndexCellMappingList) { + if (!storageIndexHashTab.contains(storageIndexCellMapping.cellID)) { + RevisionStoreCell cell = + this.parseCell(storageIndexCellMapping.cellID, msOneStorePackage); + // The storage index can retain a mapping for a deleted version context. + // Such an entry has no CellManifestDataElementData (often its mapping GUID + // is all zero) and therefore cannot contain current document content. + if (cell != null) { + msOneStorePackage.OtherFileNodeList.addAll(cell.objectGroups); + msOneStorePackage.cells.add(cell); + } + storageIndexHashTab.add(storageIndexCellMapping.cellID); + } + } return msOneStorePackage; } @@ -125,74 +218,187 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE * Find the CellManifestDataElementData * * @param cellMappingExtendedGUID The ExGuid of Cell Mapping Extended GUID. - * @return The CellManifestDataElementData instance. + * @return The CellManifestDataElementData instance, or {@code null} if it is absent. */ private CellManifestDataElementData findCellManifest(ExGuid cellMappingExtendedGUID) { - return (CellManifestDataElementData) this.cellManifestDataElements.stream() - .filter(d -> d.dataElementExGuid.equals(cellMappingExtendedGUID)).findFirst() - .orElse(new DataElement()).data; + DataElement element = cellManifestDataElementsById.get(cellMappingExtendedGUID); + return element == null ? null : (CellManifestDataElementData) element.data; } /** * Find the Revision Manifest from Data Elements. * * @param revisionMappingExtendedGUID The Revision Mapping Extended GUID. - * @return Returns the instance of RevisionManifestDataElementData + * @return The RevisionManifestDataElementData instance, or {@code null} if it is absent. */ private RevisionManifestDataElementData findRevisionManifestDataElement( ExGuid revisionMappingExtendedGUID) { - return (RevisionManifestDataElementData) this.revisionManifestDataElements.stream() - .filter(d -> d.dataElementExGuid.equals(revisionMappingExtendedGUID)).findFirst() - .orElse(new DataElement()).data; + DataElement element = revisionManifestDataElementsById.get(revisionMappingExtendedGUID); + return element == null ? null : (RevisionManifestDataElementData) element.data; } private HeaderCell parseHeaderCell(RevisionManifestDataElementData headerCellRevisionManifest) throws IOException { + if (headerCellRevisionManifest == null || + headerCellRevisionManifest.revisionManifestObjectGroupReferences == null || + headerCellRevisionManifest.revisionManifestObjectGroupReferences.isEmpty()) { + return null; + } ExGuid rootObjectId = headerCellRevisionManifest.revisionManifestObjectGroupReferences.get( 0).objectGroupExtendedGUID; - DataElement element = this.objectGroupDataElements.stream() - .filter(d -> d.dataElementExGuid.equals(rootObjectId)).findFirst() - .orElse(new DataElement()); - + DataElement element = objectGroupDataElementsById.get(rootObjectId); + if (element == null) { + return null; + } return HeaderCell.createInstance((ObjectGroupDataElementData) element.data); } - private List parseObjectGroup(CellID objectGroupCellID, - MSOneStorePackage msOneStorePackage) + private RevisionStoreCell parseCell(CellID objectGroupCellID, + MSOneStorePackage msOneStorePackage) throws IOException { StorageIndexCellMapping storageIndexCellMapping = msOneStorePackage.findStorageIndexCellMapping(objectGroupCellID); + if (storageIndexCellMapping == null) { + String warning = "Skipping OneNote cell " + objectGroupCellID + + ": no storage-index cell mapping"; + msOneStorePackage.recordParseWarning(warning); + return null; + } CellManifestDataElementData cellManifest = this.findCellManifest(storageIndexCellMapping.cellMappingExGuid); + if (cellManifest == null || cellManifest.cellManifestCurrentRevision == null) { + String warning = "Skipping OneNote cell " + objectGroupCellID + + ": no current cell manifest"; + msOneStorePackage.recordParseWarning(warning); + return null; + } List objectGroups = new ArrayList<>(); msOneStorePackage.cellManifests.add(cellManifest); StorageIndexRevisionMapping revisionMapping = msOneStorePackage.findStorageIndexRevisionMapping( cellManifest.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid); + if (revisionMapping == null) { + String warning = "Skipping OneNote cell " + objectGroupCellID + + ": no revision mapping"; + msOneStorePackage.recordParseWarning(warning); + return null; + } RevisionManifestDataElementData revisionManifest = findRevisionManifestDataElement(revisionMapping.revisionMappingExGuid); - msOneStorePackage.revisionManifests.add(revisionManifest); - RevisionManifestRootDeclare encryptionKeyRoot = - revisionManifest.revisionManifestRootDeclareList.stream() - .filter(r -> r.rootExGuid.equals(new ExGuid(3, - UUID.fromString("4A3717F8-1C14-49E7-9526-81D942DE1741")))) - .findFirst().orElse(null); - boolean isEncryption = encryptionKeyRoot != null; - for (RevisionManifestObjectGroupReferences objRef : - revisionManifest.revisionManifestObjectGroupReferences) { - ObjectGroupDataElementData dataObject = - (ObjectGroupDataElementData) objectGroupDataElements.stream() - .filter(d -> d.dataElementExGuid.equals(objRef.objectGroupExtendedGUID)) - .findFirst().get().data; - - RevisionStoreObjectGroup objectGroup = - RevisionStoreObjectGroup.createInstance(objRef.objectGroupExtendedGUID, - dataObject, isEncryption); - objectGroups.add(objectGroup); - } - - return objectGroups; + if (revisionManifest == null || revisionManifest.revisionManifest == null) { + String warning = "Skipping OneNote cell " + objectGroupCellID + + ": no revision manifest"; + msOneStorePackage.recordParseWarning(warning); + return null; + } + + // A revision manifest may only reference the object groups that were created or + // modified in that revision. The remaining object groups belong to the chain of + // base revisions (MS-FSSHTTPB "Base Revision ID"). Follow that chain and collect + // the object groups of every revision, oldest revision first, so no content is lost. + Deque revisionChain = new ArrayDeque<>(); + Set seenRevisionIds = new HashSet<>(); + while (revisionManifest != null) { + ExGuid revisionId = revisionManifest.revisionManifest.revisionID; + if (revisionId != null && !seenRevisionIds.add(revisionId)) { + // cycle guard - stop if we have already visited this revision + break; + } + revisionChain.addFirst(revisionManifest); + ExGuid baseRevisionId = revisionManifest.revisionManifest.baseRevisionID; + if (baseRevisionId == null || baseRevisionId.guid == null || + GuidUtil.emptyGuid().equals(baseRevisionId.guid)) { + break; + } + StorageIndexRevisionMapping baseRevisionMapping = + msOneStorePackage.findStorageIndexRevisionMapping(baseRevisionId); + revisionManifest = baseRevisionMapping == null ? null : + findRevisionManifestDataElement(baseRevisionMapping.revisionMappingExGuid); + } + + Set seenObjectGroupIds = new HashSet<>(); + // for each root role, the declaration made by the most recent revision wins + Map effectiveRootDeclares = new LinkedHashMap<>(); + for (RevisionManifestDataElementData manifest : revisionChain) { + msOneStorePackage.revisionManifests.add(manifest); + RevisionManifestRootDeclare encryptionKeyRoot = + manifest.revisionManifestRootDeclareList.stream() + .filter(r -> r.rootExGuid.equals(ENCRYPTION_KEY_ROOT_EXGUID)) + .findFirst().orElse(null); + boolean isEncryption = encryptionKeyRoot != null; + for (RevisionManifestRootDeclare rootDeclare : + manifest.revisionManifestRootDeclareList) { + if (!rootDeclare.rootExGuid.equals(ENCRYPTION_KEY_ROOT_EXGUID)) { + effectiveRootDeclares.put(rootDeclare.rootExGuid, rootDeclare); + } + } + for (RevisionManifestObjectGroupReferences objRef : + manifest.revisionManifestObjectGroupReferences) { + if (!seenObjectGroupIds.add(objRef.objectGroupExtendedGUID)) { + continue; + } + DataElement dataElement = + objectGroupDataElementsById.get(objRef.objectGroupExtendedGUID); + if (dataElement == null) { + msOneStorePackage.recordParseWarning( + "OneNote object group " + objRef.objectGroupExtendedGUID + + " could not be resolved"); + continue; + } + ObjectGroupDataElementData dataObject = + (ObjectGroupDataElementData) dataElement.data; + + ObjectGroupCacheKey cacheKey = + new ObjectGroupCacheKey(dataElement.dataElementExGuid, isEncryption); + RevisionStoreObjectGroup cachedObjectGroup = objectGroupsByDataElementId.get(cacheKey); + if (cachedObjectGroup == null) { + cachedObjectGroup = RevisionStoreObjectGroup.createInstance( + objRef.objectGroupExtendedGUID, dataObject, isEncryption, + objectBlobElementsById); + objectGroupsByDataElementId.put(cacheKey, cachedObjectGroup); + } + objectGroups.add(cachedObjectGroup.copy()); + } + } + + removeSupersededObjects(objectGroups); + + RevisionStoreCell cell = new RevisionStoreCell(); + cell.cellID = objectGroupCellID; + cell.objectGroups = objectGroups; + cell.rootDeclares = new ArrayList<>(effectiveRootDeclares.values()); + return cell; + } + + /** + * An object that is modified in a later revision appears again, with the same object ID, + * in that revision's object group. Keep only the newest version of each object, replacing + * the older version in place so the original object ordering is preserved. + * + * @param objectGroups The object groups ordered from the oldest revision to the newest. + */ + private void removeSupersededObjects(List objectGroups) { + Map> containingList = new HashMap<>(); + Map indexInList = new HashMap<>(); + for (RevisionStoreObjectGroup objectGroup : objectGroups) { + List objects = objectGroup.objects; + for (int i = 0; i < objects.size(); ) { + RevisionStoreObject object = objects.get(i); + ExGuid objectId = object.objectID; + if (objectId != null && containingList.containsKey(objectId)) { + // newer version of an already seen object - replace the older one in place + containingList.get(objectId).set(indexInList.get(objectId), object); + objects.remove(i); + } else { + if (objectId != null) { + containingList.put(objectId, objects); + indexInList.put(objectId, i); + } + ++i; + } + } + } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java index 68b1dab1692..7220f999bf2 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java @@ -41,16 +41,27 @@ public class PrtArrayOfPropertyValues implements IProperty { * @return */ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws IOException { + return doDeserializeFromByteArray(byteArray, startIndex, 0); + } + + public int doDeserializeFromByteArray(byte[] byteArray, int startIndex, int depth) + throws IOException { int index = startIndex; this.cProperties = BitConverter.toInt32(byteArray, index); index += 4; this.propertyID = new PropertyID(); int len = this.propertyID.doDeserializeFromByteArray(byteArray, index); index += len; + // each PropertySet consumes at least its 2-byte count, so a valid count cannot + // exceed the remaining bytes / 2; bounds the allocation on malformed counts + if (this.cProperties < 0 || this.cProperties > (byteArray.length - index) / 2) { + throw new IOException("prtArrayOfPropertyValues count " + this.cProperties + + " exceeds remaining data " + (byteArray.length - index)); + } this.data = new PropertySet[this.cProperties]; for (int i = 0; i < this.cProperties; i++) { this.data[i] = new PropertySet(); - int length = this.data[i].doDeserializeFromByteArray(byteArray, index); + int length = this.data[i].doDeserializeFromByteArray(byteArray, index, depth + 1); index += length; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtFourBytesOfLengthFollowedByData.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtFourBytesOfLengthFollowedByData.java index 8518be09b54..4eb0ca7b832 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtFourBytesOfLengthFollowedByData.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtFourBytesOfLengthFollowedByData.java @@ -43,6 +43,12 @@ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws I int index = startIndex; this.cb = (int) BitConverter.toUInt32(byteArray, startIndex); index += 4; + // copyOfRange zero-fills past the source end, so cb must be bounded by + // the remaining bytes or a malformed length allocates cb bytes of heap + if (this.cb < 0 || this.cb > byteArray.length - index) { + throw new IOException("prtFourBytesOfLengthFollowedByData length " + this.cb + + " exceeds remaining data " + (byteArray.length - index)); + } this.data = Arrays.copyOfRange(byteArray, index, index + this.cb); index += this.cb; diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/DataElement.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/DataElement.java index 2ae203fb2bb..96e94f0e598 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/DataElement.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/DataElement.java @@ -50,7 +50,8 @@ public class DataElement extends StreamObject { String className = DataElement.class.getPackage().getName() + "." + value.name(); try { - DATA_ELEMENT_DATA_TYPE_MAPPING.put(value, Class.forName(className)); + DATA_ELEMENT_DATA_TYPE_MAPPING.put(value, + Class.forName(className, false, DataElement.class.getClassLoader())); } catch (ClassNotFoundException e) { // This is OK, we are not pulling over every single class } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java index 1211d2bf9d1..d3a70780a9a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java @@ -19,8 +19,24 @@ /** * This class is used to represent the file data. */ -class FileDataObject { +public class FileDataObject { public ObjectGroupObjectBLOBDataDeclaration objectDataBLOBDeclaration; public ObjectGroupObjectDataBLOBReference objectDataBLOBReference; public DataElement objectDataBLOBDataElement; + + /** + * @return the opaque binary data of this file data object, or null if it could not be + * resolved. + */ + public byte[] getData() { + if (objectDataBLOBDataElement != null && + objectDataBLOBDataElement.data instanceof ObjectDataBLOBDataElementData) { + ObjectDataBLOBDataElementData blobData = + (ObjectDataBLOBDataElementData) objectDataBLOBDataElement.data; + if (blobData.objectDataBLOB != null) { + return blobData.objectDataBLOB.getData(); + } + } + return null; + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java new file mode 100644 index 00000000000..2873e1d013a --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BasicObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; + +/** + * Specifies an object data BLOB stream object - the opaque binary data of an object, + * e.g. an embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8. + */ +public class ObjectDataBLOB extends StreamObject { + public BinaryItem data; + + public ObjectDataBLOB() { + super(StreamObjectTypeHeaderStart.ObjectDataBLOB); + this.data = new BinaryItem(); + } + + /** + * @return the opaque binary data as a byte array, or null if not present. + */ + public byte[] getData() { + if (this.data == null || this.data.content == null) { + return null; + } + return ByteUtil.toByteArray(this.data.content); + } + + @Override + protected void deserializeItemsFromByteArray(byte[] byteArray, AtomicInteger currentIndex, + int lengthOfItems) + throws TikaException, IOException { + AtomicInteger index = new AtomicInteger(currentIndex.get()); + this.data = BasicObject.parse(byteArray, index, BinaryItem.class); + + if (index.get() - currentIndex.get() != lengthOfItems) { + throw new StreamObjectParseErrorException(currentIndex.get(), "ObjectDataBLOB", + "Stream object over-parse error", null); + } + + currentIndex.set(index.get()); + } + + @Override + protected int serializeItemsToByteList(List byteList) throws IOException { + int startPoint = byteList.size(); + if (this.data != null) { + byteList.addAll(this.data.serializeToByteList()); + } + return byteList.size() - startPoint; + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java new file mode 100644 index 00000000000..ff28e9d096d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tika.exception.TikaException; + +/** + * Object data BLOB data element - carries the opaque binary data of an object, e.g. an + * embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8. + */ +public class ObjectDataBLOBDataElementData extends DataElementData { + public ObjectDataBLOB objectDataBLOB; + + public ObjectDataBLOBDataElementData() { + this.objectDataBLOB = new ObjectDataBLOB(); + } + + @Override + public int deserializeDataElementDataFromByteArray(byte[] byteArray, int startIndex) + throws TikaException, IOException { + AtomicInteger index = new AtomicInteger(startIndex); + this.objectDataBLOB = StreamObject.getCurrent(byteArray, index, ObjectDataBLOB.class); + return index.get() - startIndex; + } + + @Override + public List serializeToByteList() throws TikaException, IOException { + return this.objectDataBLOB.serializeToByteList(); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySet.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySet.java index 45e1278f424..64276e5aa54 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySet.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySet.java @@ -37,6 +37,10 @@ * This class is used to represent a PropertySet. */ public class PropertySet implements IProperty { + // a nesting level costs ~14 file bytes, so unbounded depth lets a small + // crafted file drive a StackOverflowError that escapes catch(Exception) + public static final int MAX_PROPERTY_NESTING = 100; + public int cProperties; public PropertyID[] rgPrids; @@ -72,6 +76,14 @@ public List serializeToByteList() throws IOException { * @return Return the length in byte of the PropertySet. */ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws IOException { + return doDeserializeFromByteArray(byteArray, startIndex, 0); + } + + public int doDeserializeFromByteArray(byte[] byteArray, int startIndex, int depth) + throws IOException { + if (depth > MAX_PROPERTY_NESTING) { + throw new IOException("PropertySet nesting exceeds " + MAX_PROPERTY_NESTING); + } int index = startIndex; this.cProperties = BitConverter.toInt16(byteArray, startIndex); @@ -124,7 +136,16 @@ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws I break; } if (property != null) { - int len = property.doDeserializeFromByteArray(byteArray, index); + int len; + if (property instanceof PropertySet) { + len = ((PropertySet) property) + .doDeserializeFromByteArray(byteArray, index, depth + 1); + } else if (property instanceof PrtArrayOfPropertyValues) { + len = ((PrtArrayOfPropertyValues) property) + .doDeserializeFromByteArray(byteArray, index, depth + 1); + } else { + len = property.doDeserializeFromByteArray(byteArray, index); + } this.rgData.add(property); index += len; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java new file mode 100644 index 00000000000..188a2a45de9 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; + +/** + * The revision store content of a single cell (object space), together with the root object + * declarations of its current revision. The object groups are ordered from the oldest revision + * to the newest. + */ +public class RevisionStoreCell { + public CellID cellID; + public List objectGroups = new ArrayList<>(); + /** + * The effective root object declarations of the cell's current revision, i.e. for each + * root role the declaration made by the most recent revision in the base revision chain. + */ + public List rootDeclares = new ArrayList<>(); +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java index afd2f6c067c..fb16a8402a0 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java @@ -18,7 +18,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,13 +38,20 @@ public RevisionStoreObjectGroup(ExGuid objectGroupId) { public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, ObjectGroupDataElementData dataObject, - boolean isEncryption) throws IOException { + boolean isEncryption, + Map blobElements) + throws IOException { RevisionStoreObjectGroup objectGroup = new RevisionStoreObjectGroup(objectGroupId); - Map objectDict = new HashMap<>(); + // insertion-ordered so the object list (and any degraded-file fallback walk) is + // deterministic + Map objectDict = new LinkedHashMap<>(); if (!isEncryption) { RevisionStoreObject revisionObject = null; for (int i = 0; i < dataObject.objectGroupDeclarations.objectDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataList.size()) { + throw new IOException("Missing object data for object declaration " + i); + } ObjectGroupObjectDeclare objectDeclaration = dataObject.objectGroupDeclarations.objectDeclarationList.get(i); ObjectGroupObjectData objectData = @@ -63,16 +70,20 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, } else if (objectDeclaration.objectPartitionID.getDecodedValue() == 1) { revisionObject.propertySet = new PropertySetObject(objectDeclaration, objectData); - if (revisionObject.jcid.jcid.isFileData != 0) { - revisionObject.referencedObjectID = objectData.objectExGUIDArray; - revisionObject.referencedObjectSpacesID = objectData.cellIDArray; - } + // the object extended GUID array lists the objects referenced by this + // object, in the same order as the CompactIDs in the OID stream of the + // ObjectSpaceObjectPropSet - see MS-ONESTORE section 2.7.8 + revisionObject.referencedObjectID = objectData.objectExGUIDArray; + revisionObject.referencedObjectSpacesID = objectData.cellIDArray; } } for (int i = 0; i < dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) { + throw new IOException("Missing BLOB reference for object declaration " + i); + } ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration = dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get( i); @@ -87,17 +98,24 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, objectDict.get(objectGroupObjectBLOBDataDeclaration.objectExGUID); } if (objectGroupObjectBLOBDataDeclaration.objectPartitionID.getDecodedValue() == 2) { + revisionObject.objectID = objectGroupObjectBLOBDataDeclaration.objectExGUID; + revisionObject.objectGroupID = objectGroupId; revisionObject.fileDataObject = new FileDataObject(); revisionObject.fileDataObject.objectDataBLOBDeclaration = objectGroupObjectBLOBDataDeclaration; revisionObject.fileDataObject.objectDataBLOBReference = objectGroupObjectDataBLOBReference; + revisionObject.fileDataObject.objectDataBLOBDataElement = blobElements.get( + objectGroupObjectDataBLOBReference.blobExtendedGUID); } } objectGroup.objects.addAll(objectDict.values()); } else { for (int i = 0; i < dataObject.objectGroupDeclarations.objectDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataList.size()) { + throw new IOException("Missing object data for object declaration " + i); + } ObjectGroupObjectDeclare objectDeclaration = dataObject.objectGroupDeclarations.objectDeclarationList.get(i); ObjectGroupObjectData objectData = @@ -114,4 +132,17 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, return objectGroup; } + + /** + * Creates a cell-local wrapper around this parsed group. The object lists are copied so + * revision supersession filtering cannot mutate a cached group used by another cell. + * + * @return a shallow copy with independent mutable lists + */ + public RevisionStoreObjectGroup copy() { + RevisionStoreObjectGroup copy = new RevisionStoreObjectGroup(objectGroupID); + copy.objects.addAll(objects); + copy.encryptionObjects.addAll(encryptionObjects); + return copy; + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/StreamObject.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/StreamObject.java index 03bb1da92cb..6a250ba1242 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/StreamObject.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/StreamObject.java @@ -73,7 +73,8 @@ public abstract class StreamObject implements IFSSHTTPBSerializable { for (StreamObjectTypeHeaderStart value : StreamObjectTypeHeaderStart.values()) { String className = StreamObject.class.getPackage().getName() + "." + value.name(); try { - streamObjectTypeMapping.put(value, Class.forName(className)); + streamObjectTypeMapping.put(value, + Class.forName(className, false, StreamObject.class.getClassLoader())); } catch (ClassNotFoundException e) { // This is OK, we are not pulling over every single class } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/CellID.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/CellID.java index a352dc6315d..7b6dbcf8869 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/CellID.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/CellID.java @@ -105,6 +105,14 @@ public int hashCode() { return this.extendGUID1.hashCode() + this.extendGUID2.hashCode(); } + @Override + public String toString() { + return "CellID{" + + "extendGUID1=" + extendGUID1 + + ", extendGUID2=" + extendGUID2 + + '}'; + } + /** * This method is used to deserialize the CellID basic object from the specified byte array and start index. * diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/ExGuid.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/ExGuid.java index 4a7e149de23..42ff354b0ce 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/ExGuid.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/ExGuid.java @@ -144,6 +144,15 @@ public int hashCode() { return this.guid.hashCode() + Integer.valueOf(this.value).hashCode(); } + @Override + public String toString() { + return "ExGuid{" + + "type=" + type + + ", value=" + value + + ", guid=" + guid + + '}'; + } + /** * This method is used to deserialize the ExGuid basic object from the specified byte array and start index. * diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java index fe6d288330a..7e1e348e7d5 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java @@ -57,10 +57,16 @@ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws I int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfContextIDs count " + + this.header.count + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); - int count = compactID.doDeserializeFromByteArray(byteArray, startIndex); + int count = compactID.doDeserializeFromByteArray(byteArray, index); this.body[i] = compactID; index += count; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java index 7901dd015ed..53398663c36 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java @@ -58,10 +58,16 @@ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws I int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfOIDs count " + this.header.count + + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); - int count = compactID.doDeserializeFromByteArray(byteArray, startIndex); + int count = compactID.doDeserializeFromByteArray(byteArray, index); this.body[i] = compactID; index += count; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java index 61f7deda901..f37de26f765 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java @@ -57,6 +57,12 @@ public int doDeserializeFromByteArray(byte[] byteArray, int startIndex) throws I int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfOSIDs count " + this.header.count + + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java new file mode 100644 index 00000000000..0f21f861db5 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.exception.TikaException; + +public class GUIDTest { + + @Test + public void testParsesValidGuidRoundTrip() throws Exception { + byte[] valid = "{638DE92F-a6d4-4BC1-9A36-4AFC2511A5B7}" + .getBytes(StandardCharsets.UTF_16LE); + + assertEquals("{638DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}", + GUID.fromCurlyBraceUTF16Bytes(valid).toString()); + } + + @Test + public void testRejectsNonAsciiUnicodeDigits() { + // fullwidth '6' is a Unicode digit that Character.digit(c, 16) would accept + byte[] malformed = "{６38DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}" + .getBytes(StandardCharsets.UTF_16LE); + + assertThrows(TikaException.class, () -> GUID.fromCurlyBraceUTF16Bytes(malformed)); + } + + @Test + public void testRejectsMalformedCurlyBraceGuid() { + byte[] malformed = "{638DE92F-A6D4-4BC1-9IJ36-4AFC2511A5B7}" + .getBytes(StandardCharsets.UTF_16LE); + + assertThrows(TikaException.class, () -> GUID.fromCurlyBraceUTF16Bytes(malformed)); + } + + @Test + public void testRejectsInjectedDashes() { + String[] malformedGuids = { + "{-38DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}", + "{--8DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}", + "{---8DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}", + "{----E92F-A6D4-4BC1-9A36-4AFC2511A5B7}" + }; + for (String malformedGuid : malformedGuids) { + assertThrows(TikaException.class, + () -> GUID.fromCurlyBraceUTF16Bytes( + malformedGuid.getBytes(StandardCharsets.UTF_16LE))); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java index bab0a6fbcb0..e2c9e410ddf 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java @@ -19,23 +19,128 @@ import static org.apache.tika.parser.microsoft.onenote.OneNoteParser.ONE_NOTE_PREFIX; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.ContentHandler; import org.apache.tika.TikaTest; +import org.apache.tika.exception.TikaException; +import org.apache.tika.exception.TikaMemoryLimitException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.MSOneStorePackage; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; public class OneNoteParserTest extends TikaTest { //test recursive parser wrapper for image files + @Test + public void testFuzzerRegressionInputsFallBackToLegacyDump() throws Exception { + // a structural failure falls back to the legacy string dump; the failure is + // pinned in the parse-warning metadata + String[][] resources = { + {"testOneNote-fuzz1.one", "Missing dependent revision"}, + {"testOneNote-fuzz2.one", "unified property count"}, + {"testOneNote-fuzz3.one", "Invalid GUID string"} + }; + for (String[] resource : resources) { + InputStream input = getClass().getResourceAsStream("/test-documents/" + resource[0]); + assertNotNull(input, resource[0]); + try (InputStream stream = input; + TikaInputStream tis = TikaInputStream.get(stream)) { + Metadata metadata = new Metadata(); + new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata, + new ParseContext()); + assertTrue(Arrays.stream( + metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains(resource[1]) + && warning.contains("falling back to legacy text dump")), + () -> resource[0] + ": " + Arrays.toString(metadata.getValues( + TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))); + } + } + } + + @Test + public void testNoContentFsshttpbWalkFallsBackToLegacyDump(@TempDir Path tempDir) + throws Exception { + Path file = tempDir.resolve("empty-walk.one"); + // trailing newline: the dump excludes the file's final byte from scanning + Files.write(file, + "recoverable legacy dump text\n".getBytes(StandardCharsets.US_ASCII)); + Metadata metadata = new Metadata(); + StringWriter writer = new StringWriter(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(new ToTextContentHandler(writer), + metadata); + xhtml.startDocument(); + try (OneNoteDirectFileResource resource = + new OneNoteDirectFileResource(file.toFile())) { + // fresh package = the walk completed without emitting anything + OneNoteParser.legacyFallbackIfNoContent(new MSOneStorePackage(), metadata, xhtml, + resource); + assertTrue(writer.toString().contains("recoverable legacy dump text")); + assertTrue(metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING) + .contains("produced no content")); + + // pkg == null means the exception path already dumped - no second dump + StringWriter exceptionPath = new StringWriter(); + XHTMLContentHandler xhtml2 = new XHTMLContentHandler( + new ToTextContentHandler(exceptionPath), metadata); + xhtml2.startDocument(); + OneNoteParser.legacyFallbackIfNoContent(null, metadata, xhtml2, resource); + xhtml2.endDocument(); + assertTrue(exceptionPath.toString().isBlank()); + } + xhtml.endDocument(); + } + + @Test + public void testTruncatedFileFallsBackToLegacyDump(@TempDir Path tempDir) throws Exception { + byte[] full; + try (InputStream is = getClass() + .getResourceAsStream("/test-documents/testOneNote1.one")) { + full = is.readAllBytes(); + } + // keep the 1024-byte header plus a sliver of content so the root file node list + // is unreachable + Path truncated = tempDir.resolve("truncated.one"); + Files.write(truncated, Arrays.copyOf(full, 2048)); + + Metadata metadata = new Metadata(); + try (TikaInputStream tis = TikaInputStream.get(truncated)) { + new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata, + new ParseContext()); + } + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("falling back to legacy text dump"))); + } + /** * This is the sample document that is automatically created from onenote 2013. */ @@ -223,6 +328,217 @@ public void testOneNoteEmbeddedWordDoc() throws Exception { ml.get("Content-Type")))); } + @Test + public void testOneNoteEmbeddedImage() throws Exception { + List embedded = new ArrayList<>(); + List embeddedTypes = new ArrayList<>(); + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(InputStream stream, ContentHandler handler, + Metadata metadata, boolean outputHtml) throws IOException { + embedded.add(stream.readAllBytes()); + embeddedTypes.add(metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + }); + try (InputStream tis = getResourceAsStream("/test-documents/testOneNoteEmbeddedImage.one")) { + new OneNoteParser().parse(tis, new ToTextContentHandler(), new Metadata(), context); + } + + assertEquals(1, embedded.size()); + assertTrue(embedded.stream().anyMatch(bytes -> bytes.length > 1000), + () -> "embedded sizes: " + embedded.stream().map(bytes -> bytes.length).collect(Collectors.toList())); + assertTrue(embeddedTypes.contains("INLINE")); + } + + @Test + public void testOneNoteEmbeddedImageRecursiveMetadata() throws Exception { + List metadataList = getRecursiveMetadata("testOneNoteEmbeddedImage.one"); + + assertEquals(2, metadataList.size()); + Metadata embedded = metadataList.get(1); + assertEquals("INLINE", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertNotNull(embedded.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID)); + } + + @Test + public void testPropertyValueBudgetIsSharedAcrossCopiesAndResetsPerList( + @TempDir Path tempDir) throws Exception { + Path emptyFile = tempDir.resolve("empty"); + Files.write(emptyFile, ByteBuffer.allocate(36).order(ByteOrder.LITTLE_ENDIAN) + .putLong(FileNodeListHeader.UNIT_MAGIC_CONSTANT).putInt(0x10).putInt(0) + .putLong(-1).putInt(0).putLong(OneNotePtr.FOOTER_CONST).array()); + Method reservePropertyCount = OneNotePtr.class.getDeclaredMethod( + "reservePropertyCount", long.class, String.class); + reservePropertyCount.setAccessible(true); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(emptyFile.toFile())) { + OneNotePtr ptr = new OneNotePtr(new OneNoteDocument(), dif); + OneNotePtr copy = new OneNotePtr(ptr); + reservePropertyCount.invoke(ptr, 100_000L, "test"); + InvocationTargetException exception = assertThrows(InvocationTargetException.class, + () -> reservePropertyCount.invoke(copy, 1L, "test")); + assertTrue(exception.getCause() instanceof TikaMemoryLimitException); + + ptr.deserializeFileNodeList(new FileNodeList(), new FileNodePtr()); + reservePropertyCount.invoke(ptr, 1L, "test"); + } + } + + @Test + public void testFileNodeCycleIsReportedAndStopsTraversal(@TempDir Path tempDir) throws Exception { + FileNode fileNode = new FileNode().setGosid(ExtendedGUID.nil()); + fileNode.childFileNodeList.setFileNodeListHeader(new FileNodeListHeader(0, + FileNodeListHeader.UNIT_MAGIC_CONSTANT, 0x10, 0)); + fileNode.childFileNodeList.children.add(fileNode); + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Path emptyFile = Files.createFile(tempDir.resolve("empty")); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(emptyFile.toFile())) { + OneNoteTreeWalker walker = new OneNoteTreeWalker(new OneNoteTreeWalkerOptions(), + new OneNoteDocument(), dif, + new XHTMLContentHandler(new ToTextContentHandler(), metadata), + metadata, parseContext, null); + walker.walkFileNode(fileNode, null); + } + assertEquals(1, Arrays.stream( + metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .filter(warning -> warning.contains("file-node cycle detected")) + .count()); + } + + @Test + public void testDeepFileNodeChainIsDepthCapped(@TempDir Path tempDir) throws Exception { + // a long acyclic chain slips past the cycle guard - the depth cap must stop it + // before the recursion can overflow the stack + FileNode root = new FileNode().setGosid(ExtendedGUID.nil()); + FileNode current = root; + for (int i = 0; i < 600; i++) { + FileNode child = new FileNode().setGosid(ExtendedGUID.nil()); + current.childFileNodeList.setFileNodeListHeader(new FileNodeListHeader(0, + FileNodeListHeader.UNIT_MAGIC_CONSTANT, 0x10, 0)); + current.childFileNodeList.children.add(child); + current = child; + } + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Path emptyFile = Files.createFile(tempDir.resolve("empty")); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(emptyFile.toFile())) { + OneNoteTreeWalker walker = new OneNoteTreeWalker(new OneNoteTreeWalkerOptions(), + new OneNoteDocument(), dif, + new XHTMLContentHandler(new ToTextContentHandler(), metadata), + metadata, parseContext, null); + walker.walkFileNode(root, null); + } + assertEquals(1, Arrays.stream( + metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .filter(warning -> warning.contains("exceeded depth limit")) + .count()); + } + + /** + * Writes one minimal 56-byte file-node-list fragment: header, a single baseType-2 node + * whose child list is at {childStp, childCb}, a terminator, a nil next-fragment + * reference and the footer. + */ + private static void writeFileNodeListBlock(ByteBuffer buf, long childStp, int childCb) { + // id=0x10, size=16 (node header + 8-byte stp + 4-byte cb), stpFormat=0, cbFormat=0, + // baseType=2, reserved=1 + int fileNodeHeader = 0x10 | (16 << 10) | (2 << 27) | (1 << 31); + buf.putLong(FileNodeListHeader.UNIT_MAGIC_CONSTANT).putInt(0x10).putInt(0) + .putInt(fileNodeHeader).putLong(childStp).putInt(childCb) + .putInt(0) + .putLong(-1).putInt(0) + .putLong(OneNotePtr.FOOTER_CONST); + } + + @Test + public void testFileNodeListCycleFailsCleanly(@TempDir Path tempDir) throws Exception { + // one fragment holding a baseType-2 node whose child list points back at itself + ByteBuffer buf = ByteBuffer.allocate(56).order(ByteOrder.LITTLE_ENDIAN); + writeFileNodeListBlock(buf, 0, 56); + Path cyclic = tempDir.resolve("cyclic"); + Files.write(cyclic, buf.array()); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(cyclic.toFile())) { + OneNotePtr ptr = new OneNotePtr(new OneNoteDocument(), dif); + TikaException e = assertThrows(TikaException.class, + () -> ptr.deserializeFileNodeList(new FileNodeList(), new FileNodePtr())); + assertTrue(e.getMessage().contains("cycle"), e.getMessage()); + } + } + + @Test + public void testFileNodeListNestingIsDepthCapped(@TempDir Path tempDir) throws Exception { + // 120 lists, each holding one baseType-2 node pointing at the next list - acyclic, + // so only the depth cap can stop the recursion + int lists = 120; + ByteBuffer buf = ByteBuffer.allocate(56 * lists).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < lists; i++) { + long childStp = (i + 1 < lists ? i + 1 : i) * 56L; + writeFileNodeListBlock(buf, childStp, 56); + } + Path deep = tempDir.resolve("deep"); + Files.write(deep, buf.array()); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(deep.toFile())) { + OneNotePtr ptr = new OneNotePtr(new OneNoteDocument(), dif); + assertThrows(TikaMemoryLimitException.class, + () -> ptr.deserializeFileNodeList(new FileNodeList(), new FileNodePtr())); + } + } + + @Test + public void testFragmentChainCycleFailsCleanly(@TempDir Path tempDir) throws Exception { + // an empty fragment whose next-fragment reference points back at itself would + // previously loop forever + ByteBuffer buf = ByteBuffer.allocate(36).order(ByteOrder.LITTLE_ENDIAN) + .putLong(FileNodeListHeader.UNIT_MAGIC_CONSTANT).putInt(0x10).putInt(0) + .putLong(0).putInt(36) + .putLong(OneNotePtr.FOOTER_CONST); + Path cyclic = tempDir.resolve("cyclic-fragment"); + Files.write(cyclic, buf.array()); + try (OneNoteDirectFileResource dif = new OneNoteDirectFileResource(cyclic.toFile())) { + OneNotePtr ptr = new OneNotePtr(new OneNoteDocument(), dif); + TikaException e = assertThrows(TikaException.class, + () -> ptr.deserializeFileNodeList(new FileNodeList(), new FileNodePtr())); + assertTrue(e.getMessage().contains("fragment cycle"), e.getMessage()); + } + } + + @Test + public void testLegacyEmbeddedExtractionHonorsShouldParseEmbedded() throws Exception { + List offered = new ArrayList<>(); + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + offered.add(metadata); + return false; + } + + @Override + public void parseEmbedded(InputStream stream, ContentHandler handler, + Metadata metadata, boolean outputHtml) { + throw new AssertionError("must not parse embedded when shouldParseEmbedded" + + " returns false"); + } + }); + try (InputStream tis = + getResourceAsStream("/test-documents/testOneNoteEmbeddedWordDoc.one")) { + new OneNoteParser().parse(tis, new ToTextContentHandler(), new Metadata(), context); + } + assertTrue(offered.size() > 0); + // the legacy path offers every embedded object as an ATTACHMENT; the relationship + // id is set only when the object carries a non-nil gosid + for (Metadata offeredMetadata : offered) { + assertEquals(TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString(), + offeredMetadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + } + /** * Test a document pulled from Office 365 which stores the MS-ONESTORE document using the MS-FSSHTTPB * protocol. @@ -232,7 +548,14 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception { Metadata metadata = new Metadata(); String txt = getText("testOneNoteFromOffice365.one", metadata); - assertEquals(1, metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors").length); + // only the authors of the current content count - authors that only appear in + // older page version snapshots are not reported + assertEquals(Arrays.asList("Chang Du", "Du Chang"), + Arrays.asList(metadata.getValues(TikaCoreProperties.CREATOR))); + // both authors are referenced as AuthorMostRecent by current content; one of them + // is first visited under another role, so its most-recent role must still register + assertEquals(Arrays.asList("Chang Du", "Du Chang"), + Arrays.asList(metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors"))); assertEquals(Instant.ofEpochSecond(1636621406), Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "creationTimestamp")))); @@ -241,6 +564,9 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception { assertEquals(Instant.ofEpochSecond(1636621448), Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED)))); assertContains("Section1Page1Content", txt); + // content from revisions other than each cell's current revision manifest + assertContains("Section1Page2Content", txt); + assertTrue(txt.indexOf("Section1Page1Content") < txt.indexOf("Section1Page2Content")); } /** @@ -252,20 +578,23 @@ public void testOneNoteDocumentFromOffice365_2() throws Exception { Metadata metadata = new Metadata(); String txt = getText("testOneNoteFromOffice365-2.one", metadata); + assertEquals(List.of("Robert Lucarini"), + Arrays.asList(metadata.getValues(TikaCoreProperties.CREATOR))); List mostRecentAuthors = Arrays.asList(metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors")); assertContains( - "R\u0000o\u0000b\u0000e\u0000r\u0000t\u0000 \u0000L\u0000u\u0000c\u0000a" + - "\u0000r\u0000i\u0000n\u0000i\u0000\u0000\u0000", + "Robert Lucarini", mostRecentAuthors); assertEquals(Instant.ofEpochSecond(1591712300), Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "creationTimestamp")))); - assertEquals(Instant.ofEpochMilli(1623252330000L), + assertEquals(Instant.ofEpochMilli(1623597638000L), Instant.ofEpochMilli(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "lastModifiedTimestamp")))); assertEquals(Instant.ofEpochSecond(1623597587), Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED)))); assertContains("Section1Page1Content", txt); + // content from revisions other than each cell's current revision manifest + assertContains("Section1Page2Content", txt); } private void assertNoJunk(String txt) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java new file mode 100644 index 00000000000..ea5a4c986b7 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import org.xml.sax.ContentHandler; + +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.FileDataObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOB; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOBDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectBLOBDataDeclaration; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectDataBLOBReference; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StreamObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StreamObjectParseErrorException; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +/** + * Tests for object data BLOBs (embedded images and files) in the MS-FSSHTTPB packaged + * revision store. + */ +public class MSOneStoreBlobTest { + + private static final byte[] BLOB_BYTES = + "these are the bytes of an embedded image".getBytes(StandardCharsets.UTF_8); + + /** + * An object data BLOB data element must survive a serialization round trip. Before + * ObjectDataBLOBDataElementData existed, deserializing a data element of this type threw, + * so any OneNote file with an embedded image or file lost its structure entirely. + */ + @Test + public void testObjectDataBLOBDataElementRoundTrip() throws Exception { + DataElement reparsed = roundTripBlobElement(); + assertEquals(DataElementType.ObjectDataBLOBDataElementData, reparsed.dataElementType); + assertArrayEquals(BLOB_BYTES, + ((ObjectDataBLOBDataElementData) reparsed.data).objectDataBLOB.getData()); + } + + /** + * A revision store object whose BLOB reference resolves to an object data BLOB element + * must expose the BLOB bytes and hand them to the embedded document extractor during the + * tree walk. + */ + @Test + public void testBlobIsEmittedAsEmbeddedDocument() throws Exception { + DataElement blobElement = roundTripBlobElement(); + + ObjectGroupObjectBLOBDataDeclaration declaration = + new ObjectGroupObjectBLOBDataDeclaration(); + declaration.objectExGUID = new ExGuid(1, UUID.randomUUID()); + declaration.objectPartitionID.setDecodedValue(2); + ObjectGroupObjectDataBLOBReference blobReference = + new ObjectGroupObjectDataBLOBReference(); + blobReference.blobExtendedGUID = blobElement.dataElementExGuid; + + ObjectGroupDataElementData groupData = new ObjectGroupDataElementData(); + groupData.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList + .add(declaration); + groupData.objectGroupData.objectGroupObjectDataBLOBReferenceList.add(blobReference); + + RevisionStoreObjectGroup objectGroup = + RevisionStoreObjectGroup.createInstance(new ExGuid(2, UUID.randomUUID()), + groupData, false, + Collections.singletonMap(blobElement.dataElementExGuid, blobElement)); + assertEquals(1, objectGroup.objects.size()); + assertArrayEquals(BLOB_BYTES, objectGroup.objects.get(0).fileDataObject.getData()); + addImageContainer(objectGroup, objectGroup.objects.get(0), "picture.png"); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.OtherFileNodeList.add(objectGroup); + + List embedded = new ArrayList<>(); + List embeddedNames = new ArrayList<>(); + List embeddedTypes = new ArrayList<>(); + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(InputStream stream, ContentHandler handler, + Metadata metadata, boolean outputHtml) throws IOException { + embedded.add(stream.readAllBytes()); + embeddedNames.add(metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + embeddedTypes.add(metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + } + }); + + Metadata metadata = new Metadata(); + XHTMLContentHandler xhtml = + new XHTMLContentHandler(new ToTextContentHandler(new StringWriter()), metadata); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + + assertEquals(1, embedded.size()); + assertArrayEquals(BLOB_BYTES, embedded.get(0)); + assertEquals("picture.png", embeddedNames.get(0)); + assertEquals("INLINE", embeddedTypes.get(0)); + } + + private static void addImageContainer(RevisionStoreObjectGroup objectGroup, + RevisionStoreObject blob, String name) throws IOException { + RevisionStoreObject parent = new RevisionStoreObject(); + parent.objectID = new ExGuid(9, UUID.randomUUID()); + parent.objectGroupID = objectGroup.objectGroupID; + parent.referencedObjectID = new ExGUIDArray(); + parent.referencedObjectID.content = Collections.singletonList(blob.objectID); + + PropertySet propertySet = new PropertySet(); + propertySet.cProperties = 2; + PropertyID fileNameID = new PropertyID(); + fileNameID.type = PropertyType.FourBytesOfLengthFollowedByData.getIntVal(); + fileNameID.value = 0x1C001DD7; + PropertyID containerID = new PropertyID(); + containerID.type = PropertyType.ObjectID.getIntVal(); + containerID.value = 0x20001C3F; + propertySet.rgPrids = new PropertyID[]{fileNameID, containerID}; + PrtFourBytesOfLengthFollowedByData fileName = new PrtFourBytesOfLengthFollowedByData(); + fileName.data = (name + "\u0000").getBytes(StandardCharsets.UTF_16LE); + fileName.cb = fileName.data.length; + propertySet.rgData = Arrays.asList(fileName, new NoData()); + ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet(); + propSet.body = propertySet; + ObjectGroupObjectData objectData = new ObjectGroupObjectData(); + objectData.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80, + 0, 0, 0, 0})); + PropertySetObject propertySetObject = new PropertySetObject(null, objectData); + propertySetObject.objectSpaceObjectPropSet = propSet; + parent.propertySet = propertySetObject; + objectGroup.objects.add(0, parent); + } + + @Test + public void testMissingBlobDataAndMalformedLengthAreHandled() throws Exception { + FileDataObject fileData = new FileDataObject(); + assertNull(fileData.getData()); + + DataElement wrongType = new DataElement(); + fileData.objectDataBLOBDataElement = wrongType; + assertNull(fileData.getData()); + + ObjectDataBLOBDataElementData emptyBlobData = new ObjectDataBLOBDataElementData(); + emptyBlobData.objectDataBLOB = null; + wrongType.data = emptyBlobData; + assertNull(fileData.getData()); + + ObjectDataBLOB blob = new ObjectDataBLOB(); + blob.data = null; + assertNull(blob.getData()); + blob.data = new BinaryItem(); + blob.data.content = null; + assertNull(blob.getData()); + blob.data = new BinaryItem(); + blob.data.content.add((byte) 1); + assertFalse(blob.serializeToByteList().isEmpty()); + + ObjectDataBLOB validBlob = new ObjectDataBLOB(); + validBlob.data.content.add((byte) 1); + byte[] itemBytes = ByteUtil.toByteArray(validBlob.data.serializeToByteList()); + AtomicInteger index = new AtomicInteger(0); + java.lang.reflect.Method deserialize = ObjectDataBLOB.class.getDeclaredMethod( + "deserializeItemsFromByteArray", byte[].class, AtomicInteger.class, int.class); + deserialize.setAccessible(true); + InvocationTargetException exception = assertThrows(InvocationTargetException.class, + () -> deserialize.invoke(new ObjectDataBLOB(), itemBytes, index, + itemBytes.length + 1)); + assertTrue(exception.getCause() instanceof StreamObjectParseErrorException); + } + + @Test + public void testEmptyAndFailedEmbeddedDataAreIgnored() throws Exception { + MSOneStorePackage emptyPackage = packageWithFileData(new ObjectDataBLOBDataElementData()); + walkWithExtractor(emptyPackage, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + throw new AssertionError("empty data must not reach the extractor"); + } + + @Override + public void parseEmbedded(InputStream stream, ContentHandler handler, + Metadata metadata, boolean outputHtml) { + } + }); + + DataElement blobElement = roundTripBlobElement(); + MSOneStorePackage failedPackage = packageWithFileData( + (ObjectDataBLOBDataElementData) blobElement.data); + walkWithExtractor(failedPackage, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(InputStream stream, ContentHandler handler, + Metadata metadata, boolean outputHtml) throws IOException { + throw new IOException("synthetic embedded parse failure"); + } + }); + } + + @Test + public void testDeclarationWithoutObjectDataFailsCleanly() { + ObjectGroupDataElementData mismatched = new ObjectGroupDataElementData(); + ObjectGroupObjectDeclare declaration = new ObjectGroupObjectDeclare(); + declaration.objectPartitionID.setDecodedValue(1); + mismatched.objectGroupDeclarations.objectDeclarationList.add(declaration); + + IOException e = assertThrows(IOException.class, + () -> RevisionStoreObjectGroup.createInstance(new ExGuid(9, UUID.randomUUID()), + mismatched, false, Collections.emptyMap())); + assertTrue(e.getMessage().contains("Missing object data")); + } + + @Test + public void testEncryptedObjectGroupAndMissingBlobReference() throws Exception { + ObjectGroupDataElementData encryptedData = new ObjectGroupDataElementData(); + ObjectGroupObjectDeclare declaration = new ObjectGroupObjectDeclare(); + declaration.objectPartitionID.setDecodedValue(1); + ObjectGroupObjectData objectData = new ObjectGroupObjectData(); + objectData.data.content.add((byte) 42); + encryptedData.objectGroupDeclarations.objectDeclarationList.add(declaration); + encryptedData.objectGroupData.objectGroupObjectDataList.add(objectData); + + RevisionStoreObjectGroup encrypted = RevisionStoreObjectGroup.createInstance( + new ExGuid(3, UUID.randomUUID()), encryptedData, true, Collections.emptyMap()); + assertEquals(1, encrypted.encryptionObjects.size()); + + ObjectGroupDataElementData missingBlobData = new ObjectGroupDataElementData(); + ObjectGroupObjectBLOBDataDeclaration blobDeclaration = + new ObjectGroupObjectBLOBDataDeclaration(); + blobDeclaration.objectExGUID = new ExGuid(4, UUID.randomUUID()); + blobDeclaration.objectPartitionID.setDecodedValue(2); + ObjectGroupObjectDataBLOBReference reference = + new ObjectGroupObjectDataBLOBReference(); + reference.blobExtendedGUID = new ExGuid(5, UUID.randomUUID()); + missingBlobData.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList + .add(blobDeclaration); + missingBlobData.objectGroupData.objectGroupObjectDataBLOBReferenceList.add(reference); + RevisionStoreObjectGroup missing = RevisionStoreObjectGroup.createInstance( + new ExGuid(6, UUID.randomUUID()), missingBlobData, false, Collections.emptyMap()); + assertEquals(1, missing.objects.size()); + assertNull(missing.objects.get(0).fileDataObject.getData()); + } + + private MSOneStorePackage packageWithFileData(ObjectDataBLOBDataElementData blobData) { + DataElement element = new DataElement(); + element.data = blobData; + FileDataObject fileData = new FileDataObject(); + fileData.objectDataBLOBDataElement = element; + RevisionStoreObject object = new RevisionStoreObject(); + object.objectID = new ExGuid(7, UUID.randomUUID()); + object.fileDataObject = fileData; + RevisionStoreObjectGroup group = new RevisionStoreObjectGroup( + new ExGuid(8, UUID.randomUUID())); + group.objects.add(object); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.OtherFileNodeList.add(group); + return pkg; + } + + private void walkWithExtractor(MSOneStorePackage pkg, EmbeddedDocumentExtractor extractor) + throws Exception { + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, extractor); + Metadata metadata = new Metadata(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(new ToTextContentHandler(new StringWriter()), metadata); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + } + + private DataElement roundTripBlobElement() throws Exception { + ObjectDataBLOBDataElementData blobData = new ObjectDataBLOBDataElementData(); + for (byte b : BLOB_BYTES) { + blobData.objectDataBLOB.data.content.add(b); + } + DataElement blobElement = + new DataElement(DataElementType.ObjectDataBLOBDataElementData, blobData); + byte[] serialized = ByteUtil.toByteArray(blobElement.serializeToByteList()); + return StreamObject.getCurrent(serialized, new AtomicInteger(0), DataElement.class); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java new file mode 100644 index 00000000000..ab597063fc4 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java @@ -0,0 +1,846 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.StringWriter; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteParser; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.FileDataObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOB; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOBDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.ToXMLContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +public class MSOneStorePackageTest { + + @Test + public void testParseWarningsAreBoundedAcrossParserAndWalkPhases() throws Exception { + MSOneStorePackage pkg = new MSOneStorePackage(); + for (int i = 0; i < 99; i++) { + pkg.recordParseWarning("warning " + i); + } + pkg.recordParseWarning("duplicate warning"); + pkg.recordParseWarning("duplicate warning"); + RevisionStoreCell damagedCell = new RevisionStoreCell(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(1006); + damagedCell.rootDeclares.add(missingRoot); + pkg.cells.add(damagedCell); + + Metadata metadata = new Metadata(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, + new XHTMLContentHandler(new ToTextContentHandler(), metadata), + new ParseContext()); + + String[] warnings = metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING); + assertEquals(101, warnings.length); + assertEquals(1, Arrays.stream(warnings) + .filter(warning -> warning.equals("duplicate warning")).count()); + assertTrue(Arrays.stream(warnings).anyMatch(warning -> warning.contains("suppressed"))); + } + + @Test + public void testReferenceArrayReportsUnavailableAndCappedWarnings() throws Exception { + ExGuid rootId = id(2000); + List references = new ArrayList<>(100_001); + for (int i = 0; i < 100_001; i++) { + references.add(rootId); + } + RevisionStoreObject root = object(rootId, + propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(100_002))), references, Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = rootId; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + Metadata metadata = new Metadata(); + walk(pkg, metadata); + String[] warnings = metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING); + assertTrue(Arrays.stream(warnings) + .anyMatch(warning -> warning.contains("declared 100002 entries"))); + assertTrue(Arrays.stream(warnings) + .anyMatch(warning -> warning.contains("Capping OneNote object reference array"))); + } + + @Test + public void testPagesFollowSectionOrderAndDropOlderCellVersions() throws Exception { + ExGuid sectionRootId = id(1); + CellID pageOne = cell(10, 100); + CellID pageTwo = cell(20, 200); + CellID oldPageOne = cell(11, 100); + RevisionStoreCell pageTwoCell = cellWithText(pageTwo, "page two"); + RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page one"); + RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one"); + RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300), "unrelated"); + + RevisionStoreObject sectionRoot = object(sectionRootId, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, + new NoData())), + Collections.emptyList(), Arrays.asList(pageOne, pageTwo)); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(sectionRoot)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = sectionRootId; + section.rootDeclares.add(rootDeclare); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(1000); + section.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell, pageOneCell, unrelatedCell)); + + Metadata metadata = new Metadata(); + String text = walk(pkg, metadata); + assertTrue(text.indexOf("page one") >= 0); + assertTrue(text.indexOf("page two") >= 0); + assertTrue(text.indexOf("page one") < text.indexOf("page two")); + assertFalse(text.contains("old page one")); + assertTrue(text.contains("unrelated")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("could not be resolved"))); + } + + @Test + public void testUnresolvedRootsFallBackToAllObjects() throws Exception { + RevisionStoreCell cell = cellWithText(cell(1, 1), "fallback content"); + cell.rootDeclares.clear(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(999); + cell.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + + assertTrue(walk(pkg, metadata).contains("fallback content")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains(id(999).toString()))); + } + + @Test + public void testExhaustedObjectReferenceTriggersFallbackWarning() throws Exception { + RevisionStoreObject root = object(id(1004), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001D78, new NoData())), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreObject fallback = object(id(1005), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("fallback after exhausted reference"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, fallback)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + String text = walk(pkg, metadata); + + assertFalse(text.contains("fallback after exhausted reference")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("reference slot was exhausted"))); + } + + @Test + public void testExhaustedObjectSpaceReferenceWarns() throws Exception { + ExGuid sectionRootID = id(1007); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(object(sectionRootID, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData())), Collections.emptyList(), Collections.emptyList()))); + RevisionManifestRootDeclare sectionRoot = new RevisionManifestRootDeclare(); + sectionRoot.objectExGuid = sectionRootID; + section.rootDeclares.add(sectionRoot); + RevisionStoreCell page = cellWithText(cell(1008, 1009), "page after missing space"); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.add(page); + Metadata metadata = new Metadata(); + String text = walk(pkg, metadata); + + assertTrue(text.contains("page after missing space")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("object-space reference slot was exhausted"))); + } + + @Test + public void testMixedRootsDoNotTriggerAllObjectFallback() throws Exception { + RevisionStoreObject root = object(id(1001), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("resolved root"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreObject unrelated = object(id(1002), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("unrelated object"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, unrelated)); + RevisionManifestRootDeclare resolvedRoot = new RevisionManifestRootDeclare(); + resolvedRoot.objectExGuid = root.objectID; + cell.rootDeclares.add(resolvedRoot); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(1003); + cell.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + String text = walk(pkg, metadata); + + assertTrue(text.contains("resolved root")); + assertFalse(text.contains("unrelated object")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains(id(1003).toString()))); + } + + @Test + public void testOriginalAuthorBecomesCreator() throws Exception { + ExGuid authorId = id(701); + RevisionStoreObject root = object(id(700), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001D78, new NoData())), + Collections.singletonList(authorId), Collections.emptyList()); + RevisionStoreObject author = object(authorId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001D75, utf16Text("Иван Петров"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, author)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + walk(pkg, metadata); + assertEquals("Иван Петров", metadata.get(TikaCoreProperties.CREATOR)); + assertEquals("Иван Петров", metadata.get(OneNoteParser.ONE_NOTE_PREFIX + "originalAuthors")); + } + + @Test + public void testDualRoleAuthorRecordedForBothRoles() throws Exception { + ExGuid authorId = id(710); + // the same author object referenced as both AuthorOriginal and AuthorMostRecent + RevisionStoreObject root = object(id(711), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001D78, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x20001D79, new NoData())), + Arrays.asList(authorId, authorId), Collections.emptyList()); + RevisionStoreObject author = object(authorId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001D75, utf16Text("Single Author"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, author)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + walk(pkg, metadata); + + assertEquals("Single Author", metadata.get(TikaCoreProperties.CREATOR)); + assertEquals("Single Author", metadata.get(OneNoteParser.ONE_NOTE_PREFIX + "originalAuthors")); + assertEquals("Single Author", metadata.get(OneNoteParser.ONE_NOTE_PREFIX + "mostRecentAuthors")); + } + + @Test + public void testBlobOnlyRootWithDanglingContentRootWalksAllObjects() throws Exception { + RevisionStoreObject blobRoot = object(id(620), propertySet(), + Collections.emptyList(), Collections.emptyList()); + blobRoot.propertySet = null; + blobRoot.fileDataObject = fileData("blob root"); + RevisionStoreObject textObject = object(id(621), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("page body text"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(blobRoot, textObject)); + RevisionManifestRootDeclare blobDeclare = new RevisionManifestRootDeclare(); + blobDeclare.objectExGuid = blobRoot.objectID; + cell.rootDeclares.add(blobDeclare); + RevisionManifestRootDeclare danglingContentRoot = new RevisionManifestRootDeclare(); + danglingContentRoot.objectExGuid = id(622); + cell.rootDeclares.add(danglingContentRoot); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + + // the blob root alone cannot reach the page body; the dangling content root must + // trigger the walk-everything fallback so the body is not lost + assertTrue(walk(pkg, metadata).contains("page body text")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("walking all objects"))); + } + + @Test + public void testHasEmittedContentTracksTextAndEmptyWalks() throws Exception { + MSOneStorePackage withText = new MSOneStorePackage(); + withText.cells.add(cellWithText(cell(70, 71), "some text")); + walk(withText); + assertTrue(withText.hasEmittedContent()); + + MSOneStorePackage empty = new MSOneStorePackage(); + RevisionStoreCell danglingCell = new RevisionStoreCell(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(72); + danglingCell.rootDeclares.add(missingRoot); + empty.cells.add(danglingCell); + walk(empty); + assertFalse(empty.hasEmittedContent()); + } + + @Test + public void testSanitizeResourceNameKeepsBasenameOfPathShapedNames() { + assertEquals("pic 1.png", + MSOneStorePackage.sanitizeResourceName("D:\\images\\pic 1.png")); + assertEquals("pic.png", MSOneStorePackage.sanitizeResourceName("C:pic.png")); + assertEquals("a.png", MSOneStorePackage.sanitizeResourceName("/tmp/a.png")); + assertEquals("plain.png", MSOneStorePackage.sanitizeResourceName("plain.png")); + assertEquals("", MSOneStorePackage.sanitizeResourceName("..")); + assertEquals("", MSOneStorePackage.sanitizeResourceName("D:\\images\\..")); + } + + @Test + public void testBlobOnlyRootDoesNotFallBackToOtherObjects() throws Exception { + RevisionStoreObject blobRoot = object(id(610), propertySet(), + Collections.emptyList(), Collections.emptyList()); + blobRoot.propertySet = null; + blobRoot.fileDataObject = fileData("blob root"); + RevisionStoreObject textObject = object(id(611), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("fallback content"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(blobRoot, textObject)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = blobRoot.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertFalse(walk(pkg).contains("fallback content")); + } + + @Test + public void testPageMarkupIsBalancedAndPinned() throws Exception { + CellID pageID = cell(90, 91); + RevisionStoreCell page = cellWithText(pageID, "page text"); + ExGuid sectionRootID = id(92); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(object(sectionRootID, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData())), Collections.emptyList(), + Collections.singletonList(pageID)))); + RevisionManifestRootDeclare sectionRoot = new RevisionManifestRootDeclare(); + sectionRoot.objectExGuid = sectionRootID; + section.rootDeclares.add(sectionRoot); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.add(page); + + String xml = walkXml(pkg); + assertEquals(1, count(xml, "

")); + assertEquals(1, count(xml, "
")); + assertTrue(xml.contains("page text")); + } + + @Test + public void testPageMarkupClosesWhenWalkThrows() throws Exception { + CellID pageID = cell(95, 96); + RevisionStoreCell page = cellWithText(pageID, "page text"); + ExGuid sectionRootID = id(97); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(object(sectionRootID, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData())), Collections.emptyList(), + Collections.singletonList(pageID)))); + RevisionManifestRootDeclare sectionRoot = new RevisionManifestRootDeclare(); + sectionRoot.objectExGuid = sectionRootID; + section.rootDeclares.add(sectionRoot); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.add(page); + Metadata metadata = new Metadata(); + List elements = new ArrayList<>(); + DefaultHandler recordingHandler = new DefaultHandler() { + private final StringBuilder text = new StringBuilder(); + + @Override + public void startElement(String uri, String localName, String qName, + org.xml.sax.Attributes atts) { + elements.add(qName); + } + + @Override + public void endElement(String uri, String localName, String qName) { + elements.add("/" + qName); + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + text.append(ch, start, length); + if (text.indexOf("page text") >= 0) { + throw new SAXException("intentional test failure"); + } + } + }; + XHTMLContentHandler xhtml = new XHTMLContentHandler(recordingHandler, metadata); + xhtml.startDocument(); + + assertThrows(SAXException.class, () -> pkg.walkTree(new OneNoteTreeWalkerOptions(), + metadata, xhtml, new ParseContext())); + assertTrue(elements.contains("p")); + assertTrue(elements.contains("/p")); + assertTrue(elements.contains("div")); + assertTrue(elements.contains("/div")); + assertTrue(elements.indexOf("p") < elements.indexOf("/p")); + assertTrue(elements.indexOf("div") < elements.indexOf("/div")); + assertTrue(elements.indexOf("/p") < elements.indexOf("/div")); + } + + @Test + public void testRemoveSupersededObjectsKeepsNewestVersion() throws Exception { + ExGuid objectID = id(950); + RevisionStoreObject oldObject = object(objectID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("old"))), Collections.emptyList(), + Collections.emptyList()); + RevisionStoreObject newObject = object(objectID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("new"))), Collections.emptyList(), + Collections.emptyList()); + List groups = new ArrayList<>(Arrays.asList( + group(oldObject), group(newObject))); + Method removeSupersededObjects = MSOneStoreParser.class.getDeclaredMethod( + "removeSupersededObjects", List.class); + removeSupersededObjects.setAccessible(true); + + removeSupersededObjects.invoke(new MSOneStoreParser(), groups); + + assertEquals(1, groups.get(0).objects.size()); + assertSame(newObject, groups.get(0).objects.get(0)); + assertTrue(groups.get(1).objects.isEmpty()); + } + + @Test + public void testArrayCountsAndRecursionDepthAreBounded() throws Exception { + RevisionStoreObject hugeArrayRoot = object(id(600), propertySet( + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(Integer.MAX_VALUE)), + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + new NoData())), + Collections.emptyList(), Collections.emptyList()); + Method collectActions = MSOneStorePackage.class.getDeclaredMethod("collectActions", + PropertySet.class, List.class, int[].class, List.class, int[].class, List.class, + int.class); + collectActions.setAccessible(true); + List actions = new ArrayList<>(); + collectActions.invoke(new MSOneStorePackage(), + hugeArrayRoot.propertySet.objectSpaceObjectPropSet.body, + Collections.emptyList(), new int[]{0}, Collections.emptyList(), new int[]{0}, + actions, 0); + assertTrue(actions.isEmpty()); + actions.clear(); + collectActions.invoke(new MSOneStorePackage(), + hugeArrayRoot.propertySet.objectSpaceObjectPropSet.body, + Collections.singletonList(id(601)), new int[]{0}, Collections.emptyList(), + new int[]{0}, actions, 1000); + assertTrue(actions.isEmpty()); + + Method collectReferencedCells = MSOneStorePackage.class.getDeclaredMethod( + "collectReferencedCells", RevisionStoreObject.class, Map.class, Set.class, + List.class, int.class); + collectReferencedCells.setAccessible(true); + RevisionStoreObject referencedRoot = object(id(602), propertySet( + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData())), + Collections.emptyList(), Collections.singletonList(cell(603, 604))); + List orderedCellIds = new ArrayList<>(); + collectReferencedCells.invoke(new MSOneStorePackage(), referencedRoot, + new java.util.HashMap<>(), new java.util.HashSet<>(), orderedCellIds, 1000); + assertTrue(orderedCellIds.isEmpty()); + + PropertySet alignedSet = propertySet( + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(100001)), + new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData())); + actions.clear(); + collectActions.invoke(new MSOneStorePackage(), alignedSet, + Collections.nCopies(100001, id(601)), new int[]{0}, Collections.emptyList(), + new int[]{0}, actions, 0); + assertEquals(100001, actions.size()); + java.lang.reflect.Field childReference = actions.get(actions.size() - 1).getClass() + .getDeclaredField("childReference"); + childReference.setAccessible(true); + assertNull(childReference.get(actions.get(actions.size() - 1))); + + Method walkObject = MSOneStorePackage.class.getDeclaredMethod("walkObject", + RevisionStoreObject.class, Map.class, Set.class, + Class.forName(MSOneStorePackage.class.getName() + "$AuthorRole"), + OneNoteTreeWalkerOptions.class, Metadata.class, XHTMLContentHandler.class, + int.class); + walkObject.setAccessible(true); + // an object that emits text when walked, so the depth-capped walk's blank + // output proves the cap fired rather than the setup having nothing to emit + RevisionStoreObject textRoot = object(id(605), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("depth capped text"))), + Collections.emptyList(), Collections.emptyList()); + Metadata metadata = new Metadata(); + StringWriter writer = new StringWriter(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(new ToTextContentHandler(writer), metadata); + xhtml.startDocument(); + walkObject.invoke(new MSOneStorePackage(), textRoot, + new java.util.HashMap<>(), new java.util.HashSet<>(), null, + new OneNoteTreeWalkerOptions(), metadata, xhtml, 1000); + assertTrue(writer.toString().isBlank()); + walkObject.invoke(new MSOneStorePackage(), textRoot, + new java.util.HashMap<>(), new java.util.HashSet<>(), null, + new OneNoteTreeWalkerOptions(), metadata, xhtml, 0); + xhtml.endDocument(); + assertTrue(writer.toString().contains("depth capped text")); + } + + @Test + public void testDanglingPrimaryPictureDoesNotSuppressWebPicture() throws Exception { + ExGuid missingPictureID = id(50); + ExGuid webPictureID = id(51); + RevisionStoreObject root = object(id(52), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())), + Arrays.asList(missingPictureID, webPictureID), Collections.emptyList()); + RevisionStoreObject webPicture = object(webPictureID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("derived picture"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, webPicture)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + String text = walk(pkg); + assertTrue(text.contains("derived picture"), text); + } + + @Test + public void testContentlessPrimaryPictureDoesNotSuppressDerivedPicture() throws Exception { + ExGuid pictureID = id(40); + ExGuid webPictureID = id(41); + RevisionStoreObject root = object(id(42), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())), + Arrays.asList(pictureID, webPictureID), Collections.emptyList()); + RevisionStoreObject picture = object(pictureID, propertySet(), + Collections.emptyList(), Collections.emptyList()); + picture.propertySet.objectSpaceObjectPropSet.body = null; + RevisionStoreObject webPicture = object(webPictureID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("derived picture"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, picture, webPicture)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertTrue(walk(pkg).contains("derived picture")); + } + + @Test + public void testUsablePrimaryPictureSuppressesDerivedPicture() throws Exception { + ExGuid pictureID = id(80); + ExGuid webPictureID = id(81); + RevisionStoreObject root = object(id(82), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())), + Arrays.asList(pictureID, webPictureID), Collections.emptyList()); + RevisionStoreObject picture = object(pictureID, propertySet(), + Collections.emptyList(), Collections.emptyList()); + picture.fileDataObject = fileData("primary image"); + RevisionStoreObject webPicture = object(webPictureID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("derived picture"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, picture, webPicture)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertFalse(walk(pkg).contains("derived picture")); + } + + @Test + public void testNestedPropertySetsAndMissingReferencesAreTraversedSafely() throws Exception { + ExGuid childId = id(2); + PropertySet nested = propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("nested text"))); + PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues(); + array.data = new PropertySet[]{propertySet(new PropertySpec( + PropertyType.FourBytesOfLengthFollowedByData, 0x1C003498, text("array text")))}; + RevisionStoreObject root = object(id(1), propertySet( + new PropertySpec(PropertyType.PropertySet, 0, nested), + new PropertySpec(PropertyType.ArrayOfPropertyValues, 0, array), + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(2)), + new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, new NoData()), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001DD7, bytes((byte) 'u', (byte) 0, (byte) 1)), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001C22, bytes((byte) 'h', (byte) 0, (byte) 'i', (byte) 0, + (byte) 0, (byte) 0))), + Collections.singletonList(childId), Collections.singletonList(cell(50, 51))); + RevisionStoreObject child = object(childId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("child text"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, child)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + String text = walk(pkg); + assertTrue(text.contains("nested text")); + assertTrue(text.contains("array text")); + assertTrue(text.contains("child text")); + assertTrue(text.contains("u")); + assertTrue(text.contains("hi")); + } + + private static String walk(MSOneStorePackage pkg) throws Exception { + Metadata metadata = new Metadata(); + return walk(pkg, metadata); + } + + private static String walk(MSOneStorePackage pkg, Metadata metadata) throws Exception { + ParseContext context = new ParseContext(); + StringWriter writer = new StringWriter(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(new ToTextContentHandler(writer), metadata); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + return writer.toString(); + } + + private static String walkXml(MSOneStorePackage pkg) throws Exception { + Metadata metadata = new Metadata(); + ParseContext context = new ParseContext(); + ToXMLContentHandler xml = new ToXMLContentHandler(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(xml, metadata); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + return xml.toString(); + } + + private static int count(String value, String needle) { + return value.split(java.util.regex.Pattern.quote(needle), -1).length - 1; + } + + private static RevisionStoreCell cellWithText(CellID cellID, String value) throws Exception { + RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()), + propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text(value))), Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.cellID = cellID; + cell.objectGroups.add(group(object)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = object.objectID; + cell.rootDeclares.add(rootDeclare); + return cell; + } + + private static RevisionStoreObject object(ExGuid objectID, PropertySet body, + List references, List spaces) + throws Exception { + RevisionStoreObject object = new RevisionStoreObject(); + object.objectID = objectID; + PropertySetObject propertySetObject = new PropertySetObject(null, emptyObjectData()); + ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet(); + propSet.body = body; + propertySetObject.objectSpaceObjectPropSet = propSet; + object.propertySet = propertySetObject; + if (!references.isEmpty()) { + object.referencedObjectID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray(); + object.referencedObjectID.content = references; + } + if (!spaces.isEmpty()) { + object.referencedObjectSpacesID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellIDArray(); + object.referencedObjectSpacesID.content = spaces; + } + return object; + } + + private static FileDataObject fileData(String value) { + ObjectDataBLOB blob = new ObjectDataBLOB(); + blob.data.content.addAll(ByteUtil.toListOfByte(value.getBytes(StandardCharsets.UTF_8))); + ObjectDataBLOBDataElementData blobData = new ObjectDataBLOBDataElementData(); + blobData.objectDataBLOB = blob; + DataElement element = new DataElement(); + element.dataElementType = DataElementType.ObjectDataBLOBDataElementData; + element.data = blobData; + FileDataObject fileData = new FileDataObject(); + fileData.objectDataBLOBDataElement = element; + return fileData; + } + + private static RevisionStoreObjectGroup group(RevisionStoreObject... objects) { + RevisionStoreObjectGroup group = new RevisionStoreObjectGroup(id(500)); + group.objects.addAll(Arrays.asList(objects)); + return group; + } + + private static PropertySet propertySet(PropertySpec... specs) { + PropertySet set = new PropertySet(); + set.cProperties = specs.length; + set.rgPrids = new PropertyID[specs.length]; + set.rgData = new ArrayList<>(); + for (int i = 0; i < specs.length; i++) { + set.rgPrids[i] = propertyID(specs[i].type, specs[i].value); + set.rgData.add(specs[i].property); + } + return set; + } + + private static PropertyID propertyID(PropertyType type, int value) { + PropertyID id = new PropertyID(); + id.type = type.getIntVal(); + id.value = value; + return id; + } + + private static PrtFourBytesOfLengthFollowedByData bytes(byte... value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value; + data.cb = data.data.length; + return data; + } + + private static PrtFourBytesOfLengthFollowedByData text(String value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value.getBytes(StandardCharsets.US_ASCII); + data.cb = data.data.length; + return data; + } + + private static PrtFourBytesOfLengthFollowedByData utf16Text(String value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = (value + "\u0000").getBytes(StandardCharsets.UTF_16LE); + data.cb = data.data.length; + return data; + } + + private static ArrayNumber arrayNumber(int number) { + ArrayNumber array = new ArrayNumber(); + array.number = number; + return array; + } + + private static ObjectGroupObjectData emptyObjectData() { + ObjectGroupObjectData data = new ObjectGroupObjectData(); + data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80, + 0, 0, 0, 0})); + return data; + } + + private static CellID cell(int first, int second) { + return new CellID(id(first), id(second)); + } + + private static ExGuid id(int value) { + return new ExGuid(value, UUID.nameUUIDFromBytes(("id-" + value).getBytes(StandardCharsets.UTF_8))); + } + + private static final class PropertySpec { + private final PropertyType type; + private final int value; + private final IProperty property; + + private PropertySpec(PropertyType type, int value, IProperty property) { + this.type = type; + this.value = value; + this.property = property; + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java new file mode 100644 index 00000000000..bfbb34c8b3d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestCurrentRevision; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestObjectGroupReferences; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexRevisionMapping; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +public class MSOneStoreParserTest { + + @Test + public void testStorageMappingIndexes() { + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.storageIndex = new StorageIndexDataElementData(); + CellID cellID = cell(700); + StorageIndexCellMapping cellMapping = new StorageIndexCellMapping(); + cellMapping.cellID = cellID; + pkg.storageIndex.storageIndexCellMappingList.add(cellMapping); + + ExGuid revisionID = id(701); + StorageIndexRevisionMapping revisionMapping = new StorageIndexRevisionMapping(); + revisionMapping.revisionExGuid = revisionID; + pkg.storageIndex.storageIndexRevisionMappingList.add(revisionMapping); + + assertSame(cellMapping, pkg.findStorageIndexCellMapping(cellID)); + assertSame(revisionMapping, pkg.findStorageIndexRevisionMapping(revisionID)); + + StorageIndexCellMapping addedAfterIndexing = new StorageIndexCellMapping(); + addedAfterIndexing.cellID = cell(702); + pkg.storageIndex.storageIndexCellMappingList.add(addedAfterIndexing); + assertNull(pkg.findStorageIndexCellMapping(addedAfterIndexing.cellID)); + } + + @Test + public void testMissingRootsAndRevisionMappingsReturnNoCell() throws Exception { + CellID cellID = cell(1); + ExGuid cellMappingID = id(2); + ExGuid missingRevisionID = id(3); + ExGuid revisionMappingID = id(4); + + MSOneStoreParser noMappingParser = parserWithEmptyIndexes(); + assertParseWarning(noMappingParser, new MSOneStorePackage(), cellID, + "no storage-index cell mapping"); + + MSOneStoreParser noManifestParser = parserWithEmptyIndexes(); + MSOneStorePackage noManifestPackage = packageWithCellMapping(cellID, cellMappingID); + assertParseWarning(noManifestParser, noManifestPackage, cellID, + "no current cell manifest"); + + MSOneStoreParser noCurrentParser = parserWithEmptyIndexes(); + DataElement currentlessManifest = cellManifest(cellMappingID, null); + set(noCurrentParser, "cellManifestDataElements", Arrays.asList(currentlessManifest)); + set(noCurrentParser, "cellManifestDataElementsById", + Collections.singletonMap(cellMappingID, currentlessManifest)); + assertParseWarning(noCurrentParser, + packageWithCellMapping(cellID, cellMappingID), cellID, + "no current cell manifest"); + + MSOneStoreParser noRevisionMappingParser = parserWithEmptyIndexes(); + DataElement manifest = cellManifest(cellMappingID, missingRevisionID); + set(noRevisionMappingParser, "cellManifestDataElements", Arrays.asList(manifest)); + set(noRevisionMappingParser, "cellManifestDataElementsById", + Collections.singletonMap(cellMappingID, manifest)); + assertParseWarning(noRevisionMappingParser, + packageWithCellMapping(cellID, cellMappingID), cellID, "no revision mapping"); + + MSOneStoreParser noRevisionManifestParser = parserWithEmptyIndexes(); + set(noRevisionManifestParser, "cellManifestDataElements", Arrays.asList(manifest)); + set(noRevisionManifestParser, "cellManifestDataElementsById", + Collections.singletonMap(cellMappingID, manifest)); + MSOneStorePackage noRevisionManifestPackage = packageWithCellMapping(cellID, + cellMappingID); + StorageIndexRevisionMapping revisionMapping = new StorageIndexRevisionMapping(); + revisionMapping.revisionExGuid = missingRevisionID; + revisionMapping.revisionMappingExGuid = revisionMappingID; + noRevisionManifestPackage.storageIndex.storageIndexRevisionMappingList.add(revisionMapping); + assertParseWarning(noRevisionManifestParser, noRevisionManifestPackage, cellID, + "no revision manifest"); + } + + @Test + public void testRevisionChainStopsCyclesAndDeduplicatesObjectGroups() throws Exception { + MSOneStoreParser parser = new MSOneStoreParser(); + set(parser, "cellManifestDataElements", new java.util.ArrayList<>()); + set(parser, "revisionManifestDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElementsById", new java.util.HashMap<>()); + set(parser, "objectBlobElementsById", new java.util.HashMap<>()); + + CellID cellID = cell(20); + ExGuid cellMappingID = id(21); + ExGuid currentMappingID = id(22); + ExGuid oldMappingID = id(23); + ExGuid currentRevisionID = id(24); + ExGuid oldRevisionID = id(25); + ExGuid currentGroupID = id(26); + ExGuid oldGroupID = id(27); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.storageIndex = new StorageIndexDataElementData(); + StorageIndexCellMapping cellMapping = new StorageIndexCellMapping(); + cellMapping.cellID = cellID; + cellMapping.cellMappingExGuid = cellMappingID; + pkg.storageIndex.storageIndexCellMappingList.add(cellMapping); + StorageIndexRevisionMapping currentMapping = new StorageIndexRevisionMapping(); + currentMapping.revisionMappingExGuid = currentMappingID; + currentMapping.revisionExGuid = currentRevisionID; + StorageIndexRevisionMapping oldMapping = new StorageIndexRevisionMapping(); + oldMapping.revisionMappingExGuid = oldMappingID; + oldMapping.revisionExGuid = oldRevisionID; + pkg.storageIndex.storageIndexRevisionMappingList.add(currentMapping); + pkg.storageIndex.storageIndexRevisionMappingList.add(oldMapping); + + DataElement cellManifestElement = new DataElement(); + cellManifestElement.dataElementExGuid = cellMappingID; + CellManifestDataElementData cellManifest = new CellManifestDataElementData(); + cellManifest.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid = currentRevisionID; + cellManifestElement.data = cellManifest; + set(parser, "cellManifestDataElements", Arrays.asList(cellManifestElement)); + set(parser, "cellManifestDataElementsById", + Collections.singletonMap(cellManifestElement.dataElementExGuid, + cellManifestElement)); + + RevisionManifestDataElementData current = revision(currentRevisionID, oldRevisionID, + currentGroupID, currentGroupID, id(99)); + RevisionManifestDataElementData old = revision(oldRevisionID, currentRevisionID, + oldGroupID); + DataElement currentElement = new DataElement(); + currentElement.dataElementExGuid = currentMappingID; + currentElement.data = current; + DataElement oldElement = new DataElement(); + oldElement.dataElementExGuid = oldMappingID; + oldElement.data = old; + set(parser, "revisionManifestDataElements", Arrays.asList(currentElement, oldElement)); + Map revisionManifests = new HashMap<>(); + revisionManifests.put(currentElement.dataElementExGuid, currentElement); + revisionManifests.put(oldElement.dataElementExGuid, oldElement); + set(parser, "revisionManifestDataElementsById", revisionManifests); + + DataElement currentGroup = new DataElement(); + currentGroup.dataElementExGuid = currentGroupID; + currentGroup.data = new ObjectGroupDataElementData(); + DataElement oldGroup = new DataElement(); + oldGroup.dataElementExGuid = oldGroupID; + oldGroup.data = new ObjectGroupDataElementData(); + set(parser, "objectGroupDataElements", Arrays.asList(currentGroup, oldGroup)); + java.util.Map objectGroups = new java.util.HashMap<>(); + objectGroups.put(currentGroupID, currentGroup); + objectGroups.put(oldGroupID, oldGroup); + set(parser, "objectGroupDataElementsById", objectGroups); + + RevisionStoreCell result = parseCell(parser, cellID, pkg); + assertEquals(2, result.objectGroups.size()); + + old.revisionManifest.baseRevisionID = id(1001); + RevisionStoreCell missingBaseResult = parseCell(parser, cellID, pkg); + assertEquals(2, missingBaseResult.objectGroups.size()); + } + + private static MSOneStoreParser parserWithEmptyIndexes() throws Exception { + MSOneStoreParser parser = new MSOneStoreParser(); + set(parser, "cellManifestDataElements", new java.util.ArrayList<>()); + set(parser, "revisionManifestDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElementsById", new java.util.HashMap<>()); + set(parser, "objectBlobElementsById", new java.util.HashMap<>()); + return parser; + } + + private static MSOneStorePackage packageWithCellMapping(CellID cellID, ExGuid mappingID) { + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.storageIndex = new StorageIndexDataElementData(); + StorageIndexCellMapping mapping = new StorageIndexCellMapping(); + mapping.cellID = cellID; + mapping.cellMappingExGuid = mappingID; + pkg.storageIndex.storageIndexCellMappingList.add(mapping); + return pkg; + } + + private static DataElement cellManifest(ExGuid mappingID, ExGuid revisionID) { + DataElement element = new DataElement(); + element.dataElementExGuid = mappingID; + CellManifestDataElementData data = new CellManifestDataElementData(); + data.cellManifestCurrentRevision = null; + if (revisionID != null) { + data.cellManifestCurrentRevision = new CellManifestCurrentRevision(); + data.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid = revisionID; + } + element.data = data; + return element; + } + + private static void assertParseWarning(MSOneStoreParser parser, MSOneStorePackage pkg, + CellID cellID, String expected) throws Exception { + assertNull(parseCell(parser, cellID, pkg)); + Metadata metadata = new Metadata(); + ParseContext context = new ParseContext(); + XHTMLContentHandler xhtml = new XHTMLContentHandler(new ToTextContentHandler(), metadata); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains(expected)), expected); + } + + private static RevisionManifestDataElementData revision(ExGuid revisionID, + ExGuid baseRevisionID, + ExGuid... groups) { + RevisionManifestDataElementData data = new RevisionManifestDataElementData(); + data.revisionManifest.revisionID = revisionID; + data.revisionManifest.baseRevisionID = baseRevisionID; + for (ExGuid group : groups) { + RevisionManifestObjectGroupReferences reference = + new RevisionManifestObjectGroupReferences(); + reference.objectGroupExtendedGUID = group; + data.revisionManifestObjectGroupReferences.add(reference); + } + return data; + } + + private static RevisionStoreCell parseCell(MSOneStoreParser parser, CellID cellID, + MSOneStorePackage pkg) throws Exception { + Method method = MSOneStoreParser.class.getDeclaredMethod("parseCell", CellID.class, + MSOneStorePackage.class); + method.setAccessible(true); + return (RevisionStoreCell) method.invoke(parser, cellID, pkg); + } + + private static void set(MSOneStoreParser parser, String field, Object value) throws Exception { + java.lang.reflect.Field declared = MSOneStoreParser.class.getDeclaredField(field); + declared.setAccessible(true); + declared.set(parser, value); + } + + private static CellID cell(int value) { + return new CellID(id(value), id(value + 1)); + } + + private static ExGuid id(int value) { + return new ExGuid(value, UUID.nameUUIDFromBytes( + ("parser-" + value).getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/PropertyDeserializationBoundsTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/PropertyDeserializationBoundsTest.java new file mode 100644 index 00000000000..73d5ecd3952 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/PropertyDeserializationBoundsTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectStreamHeader; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectStreamOfContextIDs; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectStreamOfOIDs; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectStreamOfOSIDs; + +/** + * File-derived counts and nesting in the fsshttpb property deserializers must be bounded - + * a malformed count would otherwise allocate up to gigabytes and a deeply nested property + * set would overflow the stack, both Errors that escape the parser's Exception fallback. + */ +public class PropertyDeserializationBoundsTest { + + @Test + public void testFourBytesOfLengthRejectsLengthBeyondRemainingData() { + byte[] bytes = concat(int32(1000), new byte[2]); + IOException e = assertThrows(IOException.class, + () -> new PrtFourBytesOfLengthFollowedByData() + .doDeserializeFromByteArray(bytes, 0)); + assertTrue(e.getMessage().contains("exceeds remaining data"), e.getMessage()); + } + + @Test + public void testFourBytesOfLengthRejectsNegativeLength() { + byte[] bytes = concat(int32(0xF0000000), new byte[8]); + assertThrows(IOException.class, + () -> new PrtFourBytesOfLengthFollowedByData() + .doDeserializeFromByteArray(bytes, 0)); + } + + @Test + public void testFourBytesOfLengthAcceptsExactFit() throws IOException { + byte[] bytes = concat(int32(3), new byte[]{1, 2, 3}); + PrtFourBytesOfLengthFollowedByData property = new PrtFourBytesOfLengthFollowedByData(); + assertEquals(7, property.doDeserializeFromByteArray(bytes, 0)); + assertEquals(3, property.data.length); + } + + @Test + public void testArrayOfPropertyValuesRejectsCountBeyondRemainingData() throws IOException { + byte[] bytes = concat(int32(Integer.MAX_VALUE), + propertyId(PropertyType.PropertySet.getIntVal())); + IOException e = assertThrows(IOException.class, + () -> new PrtArrayOfPropertyValues().doDeserializeFromByteArray(bytes, 0)); + assertTrue(e.getMessage().contains("exceeds remaining data"), e.getMessage()); + } + + @Test + public void testArrayOfPropertyValuesAcceptsSmallCount() throws IOException { + // one element: an empty PropertySet (int16 count of 0) + byte[] bytes = concat(int32(1), propertyId(PropertyType.PropertySet.getIntVal()), + new byte[]{0, 0}); + PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues(); + array.doDeserializeFromByteArray(bytes, 0); + assertEquals(1, array.data.length); + } + + @Test + public void testStreamOfIdsRejectCountBeyondRemainingData() throws IOException { + byte[] bytes = streamHeader(0xFFFFFF); + assertThrows(IOException.class, + () -> new ObjectSpaceObjectStreamOfOIDs().doDeserializeFromByteArray(bytes, 0)); + assertThrows(IOException.class, + () -> new ObjectSpaceObjectStreamOfOSIDs().doDeserializeFromByteArray(bytes, 0)); + assertThrows(IOException.class, + () -> new ObjectSpaceObjectStreamOfContextIDs() + .doDeserializeFromByteArray(bytes, 0)); + } + + @Test + public void testStreamOfOidsAcceptsSmallCount() throws IOException { + // one CompactID, 4 bytes + byte[] bytes = concat(streamHeader(1), new byte[4]); + ObjectSpaceObjectStreamOfOIDs stream = new ObjectSpaceObjectStreamOfOIDs(); + stream.doDeserializeFromByteArray(bytes, 0); + assertEquals(1, stream.body.length); + } + + @Test + public void testPropertySetNestingIsDepthCapped() throws IOException { + byte[] bytes = nestedPropertySet(PropertySet.MAX_PROPERTY_NESTING + 50); + IOException e = assertThrows(IOException.class, + () -> new PropertySet().doDeserializeFromByteArray(bytes, 0)); + assertTrue(e.getMessage().contains("nesting exceeds"), e.getMessage()); + } + + @Test + public void testPropertySetShallowNestingParses() throws IOException { + PropertySet propertySet = new PropertySet(); + propertySet.doDeserializeFromByteArray(nestedPropertySet(3), 0); + assertEquals(1, propertySet.rgData.size()); + } + + @Test + public void testNestingThroughArrayOfPropertyValuesIsDepthCapped() throws IOException { + // alternate PropertySet -> ArrayOfPropertyValues -> PropertySet -> ... + byte[] bytes = new byte[]{0, 0}; + for (int i = 0; i < PropertySet.MAX_PROPERTY_NESTING + 50; i++) { + bytes = concat(new byte[]{1, 0}, + propertyId(PropertyType.ArrayOfPropertyValues.getIntVal()), int32(1), + propertyId(PropertyType.PropertySet.getIntVal()), bytes); + } + byte[] finalBytes = bytes; + IOException e = assertThrows(IOException.class, + () -> new PropertySet().doDeserializeFromByteArray(finalBytes, 0)); + assertTrue(e.getMessage().contains("nesting exceeds"), e.getMessage()); + } + + private static byte[] nestedPropertySet(int depth) throws IOException { + // innermost: empty PropertySet; each wrapper declares one PropertySet-typed property + byte[] bytes = new byte[]{0, 0}; + for (int i = 0; i < depth; i++) { + bytes = concat(new byte[]{1, 0}, + propertyId(PropertyType.PropertySet.getIntVal()), bytes); + } + return bytes; + } + + private static byte[] propertyId(int type) throws IOException { + PropertyID propertyID = new PropertyID(); + propertyID.id = 1; + propertyID.type = type; + return toArray(propertyID.serializeToByteList()); + } + + private static byte[] streamHeader(int count) throws IOException { + ObjectSpaceObjectStreamHeader header = new ObjectSpaceObjectStreamHeader(); + header.count = count; + return toArray(header.serializeToByteList()); + } + + private static byte[] int32(int value) { + return new byte[]{(byte) value, (byte) (value >> 8), (byte) (value >> 16), + (byte) (value >> 24)}; + } + + private static byte[] toArray(List bytes) { + byte[] result = new byte[bytes.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = bytes.get(i); + } + return result; + } + + private static byte[] concat(byte[]... parts) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + for (byte[] part : parts) { + bos.writeBytes(part); + } + return bos.toByteArray(); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz1.one b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz1.one new file mode 100644 index 00000000000..857ac4f7d35 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz1.one differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz2.one b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz2.one new file mode 100644 index 00000000000..a038f43984d Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz2.one differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz3.one b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz3.one new file mode 100644 index 00000000000..35539efef8f Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNote-fuzz3.one differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one new file mode 100644 index 00000000000..2e2b1229525 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one differ