diff --git a/CHANGES.txt b/CHANGES.txt index 3b26ecb220..430c7f7fee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,17 @@ +Release 4.0.1 - ??? + + * Pipes now carries the caller-supplied Content-Type across the worker's + fresh-metadata boundary as a soft detection hint, so every forked-parse + endpoint (/tika, /meta, /rmeta, /unpack, /async, /pipes, plus tika-grpc + and embedded PipesForkParser) can route on a client Content-Type, not + only on the filename. Detection keeps the hint only when it equals or + specializes the content-detected type (e.g. refining image/tiff to + image/x-canon-cr2); for bytes with no magic it can select any type, + matching the routing power the filename already had. The + CONTENT_TYPE_USER_OVERRIDE key is deliberately not carried, so the hint + cannot force an unrelated type (TIKA-4825). + + Release 4.0.0 - 8/18/2026 This section is the complete delta from 3.x. It includes everything first diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc index 1f11111719..6ddfa42512 100644 --- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc +++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc @@ -428,6 +428,15 @@ Transport headers are unaffected: `Content-Disposition`, `Content-Type` and `Content-Length` still describe the payload and still influence detection. +NOTE: The way `Content-Type` influences detection changed. In 3.x, parsing ran +in-process and a request `Content-Type` acted as a hard override that forced the +type. In 4.x, parsing runs in a forked worker and the header is carried across as +a *soft* hint: detection keeps it only when it equals or specializes the type +detected from the content, and otherwise ignores it (TIKA-4825). A 3.x client +that forced an unrelated type onto arbitrary bytes (for example `text/plain`) +will now see that type ignored in favor of content-based detection. Supply the +correct `Content-Type` (or a filename) to refine within the detected hierarchy. + === Pipes Configuration (for `/pipes` and `/async`) No pipes or fetcher configuration is required to start the server: the default-on endpoints diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java index 2ad0647544..f08f3fed3c 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java @@ -36,6 +36,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.UnpackHandler; import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.metadata.writelimiter.MetadataWriteLimiterFactory; @@ -487,18 +488,8 @@ protected ParseDataOrPipesResult parseFromTuple() throws TikaException, Interrup } // Use newMetadata() to apply any configured write limits Metadata metadata = localContext.newMetadata(); - // Carry the caller-supplied resource name across the fresh-metadata boundary so - // detection, suffix selection, and the Frictionless manifest's name field see - // the logical filename rather than whatever the fetcher's path happens to be - // (e.g., a server-side spool prefix). TikaInputStream.get(path, metadata) - // already honors a pre-set RESOURCE_NAME_KEY. - Metadata tupleMetadata = fetchEmitTuple.getMetadata(); - String suppliedName = tupleMetadata == null - ? null - : tupleMetadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); - if (!StringUtils.isBlank(suppliedName)) { - metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, suppliedName); - } + // Carry the caller's resource name and Content-Type detection hints (see javadoc). + carryCallerHints(fetchEmitTuple.getMetadata(), metadata); FetchHandler.TisOrResult tisOrResult = fetchHandler.fetch(fetchEmitTuple, metadata, localContext); if (tisOrResult.pipesResult() != null) { return new ParseDataOrPipesResult(null, tisOrResult.pipesResult()); @@ -516,7 +507,33 @@ protected ParseDataOrPipesResult parseFromTuple() throws TikaException, Interrup } } - + /** + * Carries the caller-supplied detection hints from the tuple metadata across the + * fresh-metadata boundary into the metadata used for fetch and detection. + *

+ * Only the resource name and the {@code Content-Type} soft hint are carried. + * {@code Content-Type} is applied by {@code MimeTypes.detect} via {@code applyHint}, + * which keeps it only when it equals or specializes the magic-detected type (e.g. + * {@code image/tiff} -> {@code image/x-raw-nikon} for a NEF supplied without a + * filename). The {@code CONTENT_TYPE_USER_OVERRIDE} key is deliberately NOT carried: + * it short-circuits detection unconditionally and would let a caller force any type. + * + * @param tupleMetadata the caller-supplied metadata (may be null) + * @param target the fresh metadata used for fetch and detection + */ + static void carryCallerHints(Metadata tupleMetadata, Metadata target) { + if (tupleMetadata == null) { + return; + } + String suppliedName = tupleMetadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); + if (!StringUtils.isBlank(suppliedName)) { + target.set(TikaCoreProperties.RESOURCE_NAME_KEY, suppliedName); + } + String suppliedContentType = tupleMetadata.get(HttpHeaders.CONTENT_TYPE); + if (!StringUtils.isBlank(suppliedContentType)) { + target.set(HttpHeaders.CONTENT_TYPE, suppliedContentType); + } + } private ParseContext setupParseContext() throws TikaException, IOException { // ContentHandlerFactory and ParseMode are retrieved from ParseContext in ParseHandler. diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PipesWorkerCallerHintsTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PipesWorkerCallerHintsTest.java new file mode 100644 index 0000000000..e5c5c14784 --- /dev/null +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PipesWorkerCallerHintsTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.core.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeTypes; +import org.apache.tika.parser.ParseContext; + +/** + * Unit tests for {@link PipesWorker#carryCallerHints(Metadata, Metadata)}, which carries the + * caller-supplied detection hints across the worker's fresh-metadata boundary. + */ +public class PipesWorkerCallerHintsTest { + + @Test + public void testCarriesResourceNameAndContentType() { + Metadata tuple = new Metadata(); + tuple.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.nef"); + tuple.set(HttpHeaders.CONTENT_TYPE, "image/x-raw-nikon"); + + Metadata target = new Metadata(); + PipesWorker.carryCallerHints(tuple, target); + + assertEquals("photo.nef", target.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + assertEquals("image/x-raw-nikon", target.get(HttpHeaders.CONTENT_TYPE)); + } + + /** + * The Content-Type is carried only as a soft hint. The unconditional override keys + * must never be carried, or a caller could force any type past detection. + */ + @Test + public void testDoesNotCarryOverrides() { + Metadata tuple = new Metadata(); + tuple.set(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE, "image/x-raw-nikon"); + tuple.set(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE, "image/x-raw-nikon"); + + Metadata target = new Metadata(); + PipesWorker.carryCallerHints(tuple, target); + + assertNull(target.get(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE)); + assertNull(target.get(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE)); + assertNull(target.get(HttpHeaders.CONTENT_TYPE)); + } + + @Test + public void testNullTupleIsNoOp() { + Metadata target = new Metadata(); + target.set(TikaCoreProperties.RESOURCE_NAME_KEY, "keep.me"); + PipesWorker.carryCallerHints(null, target); + assertEquals("keep.me", target.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + } + + @Test + public void testBlankValuesNotCarried() { + Metadata tuple = new Metadata(); + tuple.set(TikaCoreProperties.RESOURCE_NAME_KEY, " "); + tuple.set(HttpHeaders.CONTENT_TYPE, ""); + + Metadata target = new Metadata(); + PipesWorker.carryCallerHints(tuple, target); + + assertNull(target.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + assertNull(target.get(HttpHeaders.CONTENT_TYPE)); + } + + //content that magic-detects as image/tiff (little-endian TIFF marker, no CR2 marker) + private static final byte[] TIFF_BYTES = {'I', 'I', 0x2A, 0x00, 0, 0, 0, 8}; + + private static MediaType detectWithCarriedContentType(String contentType) throws Exception { + Metadata tuple = new Metadata(); + tuple.set(HttpHeaders.CONTENT_TYPE, contentType); + Metadata target = new Metadata(); + PipesWorker.carryCallerHints(tuple, target); + try (TikaInputStream tis = TikaInputStream.get(TIFF_BYTES)) { + return MimeTypes.getDefaultMimeTypes().detect(tis, target, new ParseContext()); + } + } + + /** + * A carried Content-Type that specializes the content-detected type refines detection. + * image/x-canon-cr2 is a sub-class-of image/tiff, and these bytes lack the CR2 marker. + */ + @Test + public void testSpecializingContentTypeRefinesDetection() throws Exception { + assertEquals(MediaType.image("x-canon-cr2"), + detectWithCarriedContentType("image/x-canon-cr2")); + } + + /** + * Security boundary: a carried Content-Type that does NOT specialize the content-detected + * type is ignored, so a caller cannot force an unrelated type onto the document. + */ + @Test + public void testNonSpecializingContentTypeIgnored() throws Exception { + assertEquals(MediaType.image("tiff"), detectWithCarriedContentType("audio/mpeg")); + assertEquals(MediaType.image("tiff"), detectWithCarriedContentType("not-a-media-type")); + } +} diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java index b14f2f60e4..c0e5b8226b 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java @@ -191,7 +191,11 @@ public void testEmptyParser() throws Exception { } - // A truncated document isn't a process failure -- NOT_FOUND, not BAD_REQUEST. + // Since TIKA-4825 the caller-supplied Content-Type is carried into detection, so an + // explicit application/mock+xml routes the (truncated) document to the mock parser, + // which cannot parse the incomplete XML. That container exception maps to 422 for the + // bare-field endpoint (which has no envelope to embed it in). Without a Content-Type the + // truncated bytes detect as generic XML and still yield NOT_FOUND -- see testMetaNoType. @Test public void testMeta() throws Exception { InputStream stream = ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD); @@ -201,6 +205,19 @@ public void testMeta() throws Exception { .type("application/mock+xml") .accept(MediaType.TEXT_PLAIN) .put(copy(stream, 100)); + assertEquals(422, response.getStatus()); + } + + // A truncated document with no forcing Content-Type isn't a process failure -- + // NOT_FOUND (field missing), not BAD_REQUEST. + @Test + public void testMetaNoType() throws Exception { + InputStream stream = ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD); + + Response response = WebClient + .create(endPoint + "/meta" + "/Author") + .accept(MediaType.TEXT_PLAIN) + .put(copy(stream, 100)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus()); String msg = getStringFromInputStream((InputStream) response.getEntity()); assertEquals("Failed to get metadata field Author", msg);