From ca52a2cbc245a8d20418d71a3bafe713e2577110 Mon Sep 17 00:00:00 2001 From: Matthias Stemmler Date: Tue, 4 Aug 2026 13:13:10 +0200 Subject: [PATCH 1/2] Add CorpusStorage::find_extra returning the query alternative for each match --- CHANGELOG.md | 5 + graphannis/src/annis/db/corpusstorage.rs | 124 +++++++++---- ...__tests__find_extra_with_alternatives.snap | 58 ++++++ .../src/annis/db/corpusstorage/tests.rs | 169 ++++++++++++++++++ graphannis/src/annis/db/plan.rs | 71 +++++--- graphannis/src/annis/types.rs | 21 +++ graphannis/src/lib.rs | 2 +- 7 files changed, 387 insertions(+), 63 deletions(-) create mode 100644 graphannis/src/annis/db/corpusstorage/snapshots/graphannis__annis__db__corpusstorage__tests__find_extra_with_alternatives.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0f18480..9ec2ba9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added the `CorpusStorage::find_extra` method, which returns for each match the + alternative of the query that produced it in addition to the match ID. + ## [4.1.5] - 2026-06-25 ### Fixed diff --git a/graphannis/src/annis/db/corpusstorage.rs b/graphannis/src/annis/db/corpusstorage.rs index 840222f98..52d906360 100644 --- a/graphannis/src/annis/db/corpusstorage.rs +++ b/graphannis/src/annis/db/corpusstorage.rs @@ -14,7 +14,7 @@ use crate::annis::types::{ CorpusConfiguration, CorpusSizeUnit, FrequencyTable, FrequencyTableRow, QueryAttributeDescription, }; -use crate::annis::types::{CorpusSizeInfo, CountExtra}; +use crate::annis::types::{CorpusSizeInfo, CountExtra, MatchExtra}; use crate::annis::util::TimeoutCheck; use crate::annis::util::quicksort; use crate::{AnnotationGraph, graph::Match}; @@ -406,7 +406,9 @@ fn new_vector_with_memory_aligned_capacity(expected_len: usize) -> Vec { Vec::with_capacity(aligned_memory_size / std::mem::size_of::()) } -type FindIterator<'a> = Box> + 'a>; +/// Iterator over the matches of a `find` query +/// and the index of the query alternative that produced each match. +type FindIterator<'a> = Box> + 'a>; impl CorpusStorage { /// Create a new instance with a maximum size for the internal corpus cache. @@ -1735,36 +1737,36 @@ impl CorpusStorage { // If the output is already sorted correctly, directly return the iterator. // Quirks mode may change the order of the results, thus don't use the shortcut // if quirks mode is active. - Box::from(plan) + Box::from(plan.matches_with_alternative()) } else { let estimated_result_size = plan.estimated_output_size(); let btree_config = BtreeConfig::default() .fixed_key_size(size_of::()) .max_value_size(512); let mut anno_key_symbols: SymbolTable = SymbolTable::new(); - let mut tmp_results: BtreeIndex> = + let mut tmp_results: BtreeIndex)> = BtreeIndex::with_capacity(btree_config, estimated_result_size)?; if find_arguments.order == ResultOrder::Randomized { // Use a unique random index for each match to force a random order let mut rng = rand::rng(); - for mgroup in plan { - let mgroup = mgroup?; + for m in plan.matches_with_alternative() { + let (alternative, mgroup) = m?; let mut idx: u64 = rng.random(); while tmp_results.contains_key(&(idx as usize))? { idx = rng.random(); } let m = match_group_with_symbol_ids(&mgroup, &mut anno_key_symbols)?; - tmp_results.insert(idx as usize, m)?; + tmp_results.insert(idx as usize, (alternative, m))?; } } else { // Insert results in the order as they are given by the iterator - for (idx, mgroup) in plan.enumerate() { - let mgroup = mgroup?; + for (idx, m) in plan.matches_with_alternative().enumerate() { + let (alternative, mgroup) = m?; // add all matches to temporary container let m = match_group_with_symbol_ids(&mgroup, &mut anno_key_symbols)?; - tmp_results.insert(idx, m)?; + tmp_results.insert(idx, (alternative, m))?; } let token_helper = TokenHelper::new(db).ok(); @@ -1782,12 +1784,12 @@ impl CorpusStorage { let gs_order = db.get_graphstorage(&component_order); let mut cache = SortCache::new(gs_order); - let order_func = |m1: &Vec<(NodeID, usize)>, - m2: &Vec<(NodeID, usize)>| + let order_func = |m1: &(usize, Vec<(NodeID, usize)>), + m2: &(usize, Vec<(NodeID, usize)>)| -> Result { // Get matches from symbol ID - let m1 = match_group_resolve_symbol_ids(m1, &anno_key_symbols)?; - let m2 = match_group_resolve_symbol_ids(m2, &anno_key_symbols)?; + let m1 = match_group_resolve_symbol_ids(&m1.1, &anno_key_symbols)?; + let m2 = match_group_resolve_symbol_ids(&m2.1, &anno_key_symbols)?; // Compare the matches if find_arguments.order == ResultOrder::Inverted { @@ -1826,17 +1828,13 @@ impl CorpusStorage { quicksort::sort_first_n_items(&mut tmp_results, sort_size, order_func)?; } expected_size = Some(tmp_results.len()); - let iterator = tmp_results.into_iter()?.map(move |unresolved_match_group| { - match unresolved_match_group { - Ok((_idx, unresolved_match_group)) => { - let result = match_group_resolve_symbol_ids( - &unresolved_match_group, - &anno_key_symbols, - )?; - Ok(result) - } - Err(e) => Err(e.into()), + let iterator = tmp_results.into_iter()?.map(move |entry| match entry { + Ok((_idx, (alternative, unresolved_match_group))) => { + let result = + match_group_resolve_symbol_ids(&unresolved_match_group, &anno_key_symbols)?; + Ok((alternative, result)) } + Err(e) => Err(e.into()), }); Box::from(iterator) }; @@ -1844,13 +1842,14 @@ impl CorpusStorage { Ok((base_it, expected_size)) } - fn find_in_single_corpus>( + fn find_in_single_corpus, T>( &self, query: &SearchQuery, corpus_name: &str, find_arguments: FindArguments, timeout: TimeoutCheck, - ) -> Result<(Vec, usize)> { + map_match: impl Fn(MatchExtra) -> T, + ) -> Result<(Vec, usize)> { let prep = self.prepare_query(corpus_name, query.query, query.query_language, |db| { let mut additional_components = vec![Component::new( AnnotationComponentType::Ordering, @@ -1884,7 +1883,7 @@ impl CorpusStorage { timeout, )?; - let mut results: Vec = if let Some(expected_size) = expected_size { + let mut results: Vec = if let Some(expected_size) = expected_size { new_vector_with_memory_aligned_capacity(expected_size) } else if let Some(limit) = find_arguments.limit { new_vector_with_memory_aligned_capacity(limit) @@ -1901,15 +1900,14 @@ impl CorpusStorage { timeout.check()?; } } - let base_it: Box>> = - if let Some(limit) = find_arguments.limit { - Box::new(base_it.take(limit)) - } else { - Box::new(base_it) - }; + let base_it: FindIterator = if let Some(limit) = find_arguments.limit { + Box::new(base_it.take(limit)) + } else { + Box::new(base_it) + }; for (match_nr, m) in base_it.enumerate() { - let m = m?; + let (alternative, m) = m?; let mut match_desc = String::new(); let mut any_nodes_added = false; @@ -1971,7 +1969,10 @@ impl CorpusStorage { } } } - results.push(match_desc); + results.push(map_match(MatchExtra { + match_id: match_desc, + alternative, + })); if match_nr % 1_000 == 0 { timeout.check()?; } @@ -1991,6 +1992,8 @@ impl CorpusStorage { /// /// Returns a vector of match IDs, where each match ID consists of the matched node annotation identifiers separated by spaces. /// You can use the [subgraph(...)](#method.subgraph) method to get the subgraph for a single match described by the node annnotation identifiers. + /// + /// In order to obtain additional information for each match, see the [find_extra(...)](#method.find_extra) method. pub fn find>( &self, query: SearchQuery, @@ -1998,6 +2001,40 @@ impl CorpusStorage { limit: Option, order: ResultOrder, ) -> Result> { + self.find_mapped(query, offset, limit, order, |m| m.match_id) + } + + /// Find all results for a `query` and return the match ID and additional information for each result. + /// + /// The query is paginated and an offset and limit can be specified. + /// + /// - `query` - The search query definition. + /// - `offset` - Skip the `n` first results, where `n` is the offset. + /// - `limit` - Return at most `n` matches, where `n` is the limit. Use `None` to allow unlimited result sizes. + /// - `order` - Specify the order of the matches. + /// + /// Returns a vector of [`MatchExtra`], which contains the same match ID as returned by [find(...)](#method.find) + /// together with additional information about the match. + /// + /// If you are just interested in the match IDs, use the [find(...)](#method.find) method instead. + pub fn find_extra>( + &self, + query: SearchQuery, + offset: usize, + limit: Option, + order: ResultOrder, + ) -> Result> { + self.find_mapped(query, offset, limit, order, |m| m) + } + + fn find_mapped, T: Clone>( + &self, + query: SearchQuery, + offset: usize, + limit: Option, + order: ResultOrder, + map_match: impl Fn(MatchExtra) -> T, + ) -> Result> { let timeout = TimeoutCheck::new(query.timeout); // Sort corpus names @@ -2016,7 +2053,13 @@ impl CorpusStorage { match corpus_names.len() { 0 => Ok(Vec::new()), 1 => self - .find_in_single_corpus(&query, corpus_names[0].as_str(), find_arguments, timeout) + .find_in_single_corpus( + &query, + corpus_names[0].as_str(), + find_arguments, + timeout, + map_match, + ) .map(|r| r.0), _ => { if order == ResultOrder::Randomized { @@ -2033,8 +2076,13 @@ impl CorpusStorage { let mut result = Vec::new(); for cn in corpus_names { - let (single_result, skipped) = - self.find_in_single_corpus(&query, cn.as_ref(), find_arguments, timeout)?; + let (single_result, skipped) = self.find_in_single_corpus( + &query, + cn.as_ref(), + find_arguments, + timeout, + &map_match, + )?; // Adjust limit and offset according to the found matches for the next corpus. let single_result_length = single_result.len(); diff --git a/graphannis/src/annis/db/corpusstorage/snapshots/graphannis__annis__db__corpusstorage__tests__find_extra_with_alternatives.snap b/graphannis/src/annis/db/corpusstorage/snapshots/graphannis__annis__db__corpusstorage__tests__find_extra_with_alternatives.snap new file mode 100644 index 000000000..011764125 --- /dev/null +++ b/graphannis/src/annis/db/corpusstorage/snapshots/graphannis__annis__db__corpusstorage__tests__find_extra_with_alternatives.snap @@ -0,0 +1,58 @@ +--- +source: graphannis/src/annis/db/corpusstorage/tests.rs +expression: results +--- +[ + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok1", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok3", + alternative: 1, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok5", + alternative: 1, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc1#sTok8", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok1", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok3", + alternative: 1, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus1/doc2#sTok8", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok1", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok3", + alternative: 1, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc3#sTok8", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok1", + alternative: 0, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok3", + alternative: 1, + }, + MatchExtra { + match_id: "salt::pos::rootCorpus/subCorpus2/doc4#sTok8", + alternative: 0, + }, +] diff --git a/graphannis/src/annis/db/corpusstorage/tests.rs b/graphannis/src/annis/db/corpusstorage/tests.rs index 85de8faae..7122a85ce 100644 --- a/graphannis/src/annis/db/corpusstorage/tests.rs +++ b/graphannis/src/annis/db/corpusstorage/tests.rs @@ -1118,6 +1118,175 @@ fn find_with_multiple_corpora() { assert_debug_snapshot!("find_with_multiple_corpora_inverted_5", results); } +#[test] +fn find_extra_with_alternatives() { + let tmp = tempfile::tempdir().unwrap(); + let cargo_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + let cs = CorpusStorage::with_auto_cache_size(tmp.path(), true).unwrap(); + cs.import_from_fs( + &cargo_dir.join("tests/SaltSampleCorpus.graphml"), + ImportFormat::GraphML, + Some("SaltSampleCorpus".into()), + false, + true, + |_| {}, + ) + .unwrap(); + + let query = |query: &'static str| SearchQuery { + corpus_names: &["SaltSampleCorpus"], + query, + query_language: QueryLanguage::AQL, + timeout: None, + }; + + // Execute a query with two alternatives that have 8 and 5 matches. + // When sorted, the matches of both alternatives are interleaved and each + // match must be attributed to the alternative that produced it. + let q = query("pos=\"VBZ\" | pos=\"NN\""); + let results = cs + .find_extra(q.clone(), 0, None, ResultOrder::Normal) + .unwrap(); + assert_eq!(13, results.len()); + assert_debug_snapshot!("find_extra_with_alternatives", results); + + // The match IDs must be the same as the ones returned by find + let match_ids: Vec<_> = results.iter().map(|m| m.match_id.clone()).collect(); + let find_results = cs.find(q, 0, None, ResultOrder::Normal).unwrap(); + assert_eq!(find_results, match_ids); + + // A randomized order takes another code path, but must return the same + // matches with the same alternatives + let mut randomized_results = cs + .find_extra( + query("pos=\"VBZ\" | pos=\"NN\""), + 0, + None, + ResultOrder::Randomized, + ) + .unwrap(); + randomized_results.sort_by(|m1, m2| m1.match_id.cmp(&m2.match_id)); + let mut sorted_results = results.clone(); + sorted_results.sort_by(|m1, m2| m1.match_id.cmp(&m2.match_id)); + assert_eq!(sorted_results, randomized_results); + + // A query with a single alternative is executed without sorting the + // results, so it takes a different code path + let results = cs + .find_extra(query("pos=\"VBZ\""), 0, None, ResultOrder::NotSorted) + .unwrap(); + assert_eq!(8, results.len()); + assert!(results.iter().all(|m| m.alternative == 0)); + + // The first alternative is skipped because no execution node can be built + // for its invalid regular expression. The matches of the second alternative + // must still be attributed to it. + let results = cs + .find_extra( + query("pos=/[/ . node | pos=\"VBZ\""), + 0, + None, + ResultOrder::Normal, + ) + .unwrap(); + assert_eq!(8, results.len()); + assert!(results.iter().all(|m| m.alternative == 1)); + + // If no execution node can be built for any alternative, the query yields + // no results at all + let results = cs + .find_extra( + query("pos=/[/ . node | pos=/(/ . node"), + 0, + None, + ResultOrder::Normal, + ) + .unwrap(); + assert!(results.is_empty()); + + // The second alternative matches a superset of the first one + let matches_first = cs + .find(query("pos=\"VBZ\""), 0, None, ResultOrder::Normal) + .unwrap(); + let matches_second = cs + .find(query("pos=/V.*/"), 0, None, ResultOrder::Normal) + .unwrap(); + assert!(matches_first.len() < matches_second.len()); + + let results = cs + .find_extra( + query("pos=\"VBZ\" | pos=/V.*/"), + 0, + None, + ResultOrder::Normal, + ) + .unwrap(); + + // Matches produced by both alternatives must be returned only once ... + assert_eq!(matches_second.len(), results.len()); + + // ... and be attributed to the first alternative that produces them + for m in &results { + let expected_alternative = usize::from(!matches_first.contains(&m.match_id)); + assert_eq!(expected_alternative, m.alternative, "for {}", m.match_id); + } + assert!(results.iter().any(|m| m.alternative == 1)); +} + +#[test] +fn find_extra_with_multiple_corpora() { + let tmp = tempfile::tempdir().unwrap(); + let cargo_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + let cs = CorpusStorage::with_auto_cache_size(tmp.path(), true).unwrap(); + // Import the sample corpus with different names + let mut corpus_names = Vec::new(); + for i in 0..3 { + let corpus_name = format!("{i}"); + cs.import_from_fs( + &cargo_dir.join("tests/SaltSampleCorpus.graphml"), + ImportFormat::GraphML, + Some(corpus_name.clone()), + false, + true, + |_| {}, + ) + .unwrap(); + corpus_names.push(corpus_name); + } + + // Execute a query with two alternatives that have 8 and 5 matches inside + // each corpus + let q = SearchQuery { + corpus_names: &corpus_names, + query: "pos=\"VBZ\" | pos=\"NN\"".into(), + query_language: QueryLanguage::AQL, + timeout: None, + }; + + for order in [ResultOrder::Normal, ResultOrder::Inverted] { + let all_results = cs.find_extra(q.clone(), 0, None, order).unwrap(); + assert_eq!(39, all_results.len()); + assert!(all_results.iter().any(|m| m.alternative == 0)); + assert!(all_results.iter().any(|m| m.alternative == 1)); + + // The match IDs must be the same as the ones returned by find + let match_ids: Vec<_> = all_results.iter().map(|m| m.match_id.clone()).collect(); + assert_eq!(cs.find(q.clone(), 0, None, order).unwrap(), match_ids); + + // Paginated queries must return the same matches with the same + // alternatives, also when the pagination crosses a corpus boundary + for (offset, limit) in [(0, 5), (5, 10), (11, 4), (30, 20)] { + let results = cs + .find_extra(q.clone(), offset, Some(limit), order) + .unwrap(); + let expected = &all_results[offset..(offset + limit).min(all_results.len())]; + assert_eq!(expected, results, "for offset {offset} and limit {limit}"); + } + } +} + fn compare_edge_annos( annos1: &dyn EdgeAnnotationStorage, annos2: &dyn EdgeAnnotationStorage, diff --git a/graphannis/src/annis/db/plan.rs b/graphannis/src/annis/db/plan.rs index 00d58c979..cc52f2372 100644 --- a/graphannis/src/annis/db/plan.rs +++ b/graphannis/src/annis/db/plan.rs @@ -16,6 +16,8 @@ use transient_btree_index::{BtreeConfig, BtreeIndex}; pub struct ExecutionPlan<'a> { plans: Vec> + 'a>>, + /// The index of each plan's alternative in the original disjunction. + alternative: Vec, current_plan: usize, descriptions: Vec>, inverse_node_pos: Vec>>, @@ -32,9 +34,10 @@ impl<'a> ExecutionPlan<'a> { timeout: TimeoutCheck, ) -> Result> { let mut plans: Vec> + 'a>> = Vec::new(); + let mut alternative = Vec::new(); let mut descriptions = Vec::new(); let mut inverse_node_pos = Vec::new(); - for alt in &query.alternatives { + for (i, alt) in query.alternatives.iter().enumerate() { let p = alt.make_exec_node(db, config, timeout); if let Ok(p) = p { descriptions.push(p.get_desc().cloned()); @@ -66,6 +69,7 @@ impl<'a> ExecutionPlan<'a> { } plans.push(p); + alternative.push(i); } else if let Err(e) = p && let GraphAnnisError::AQLSemanticError(_) = &e { @@ -77,11 +81,13 @@ impl<'a> ExecutionPlan<'a> { // add a dummy execution step that yields no results let no_results_exec = EmptyResultSet {}; plans.push(Box::new(no_results_exec)); + alternative.push(0); descriptions.push(None); } let btree_config = BtreeConfig::default().fixed_value_size(std::mem::size_of::()); Ok(ExecutionPlan { current_plan: 0, + alternative, descriptions, inverse_node_pos, proxy_mode: plans.len() == 1, @@ -145,33 +151,17 @@ impl<'a> ExecutionPlan<'a> { } Ok(false) } -} -impl std::fmt::Display for ExecutionPlan<'_> { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - for (i, d) in self.descriptions.iter().enumerate() { - if i > 0 { - writeln!(f, "---[OR]---")?; - } - if let Some(d) = d { - write!(f, "{}", d.debug_string(""))?; - } else { - write!(f, "")?; - } - } - Ok(()) - } -} - -impl Iterator for ExecutionPlan<'_> { - type Item = Result; - - fn next(&mut self) -> Option { + /// Get the next match and the index of the alternative in the original + /// disjunction that produced it. A match that is produced by more than one + /// alternative is only returned once and attributed to the first + /// alternative that produces it. + fn next_with_alternative(&mut self) -> Option> { if self.proxy_mode { // just act as an proxy, but make sure the order is the same as requested in the query self.plans[0] .next() - .map(|n| n.map(|n| self.reorder_match(n))) + .map(|n| n.map(|n| (self.alternative[0], self.reorder_match(n)))) } else { while self.current_plan < self.plans.len() { if let Some(n) = self.plans[self.current_plan].next() { @@ -184,7 +174,8 @@ impl Iterator for ExecutionPlan<'_> { Ok(new_result) => { if new_result { // new result found, break out of while-loop and return the result - return Some(Ok(n)); + let alternative = self.alternative[self.current_plan]; + return Some(Ok((alternative, n))); } } Err(e) => return Some(Err(e)), @@ -202,4 +193,36 @@ impl Iterator for ExecutionPlan<'_> { None } } + + /// Returns an iterator that additionally outputs the index of the + /// alternative in the original disjunction that produced each match. + pub fn matches_with_alternative( + mut self, + ) -> impl Iterator> + 'a { + std::iter::from_fn(move || self.next_with_alternative()) + } +} + +impl std::fmt::Display for ExecutionPlan<'_> { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + for (i, d) in self.descriptions.iter().enumerate() { + if i > 0 { + writeln!(f, "---[OR]---")?; + } + if let Some(d) = d { + write!(f, "{}", d.debug_string(""))?; + } else { + write!(f, "")?; + } + } + Ok(()) + } +} + +impl Iterator for ExecutionPlan<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + self.next_with_alternative().map(|n| n.map(|(_, m)| m)) + } } diff --git a/graphannis/src/annis/types.rs b/graphannis/src/annis/types.rs index eda44a25f..c8ea055b9 100644 --- a/graphannis/src/annis/types.rs +++ b/graphannis/src/annis/types.rs @@ -11,6 +11,27 @@ pub struct CountExtra { pub document_count: u64, } +/// A single result returned from [`CorpusStorage::find_extra`](crate::CorpusStorage::find_extra). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct MatchExtra { + /// The match ID, consisting of the matched node annotation identifiers separated by spaces. + /// + /// This uses the same format as the strings returned from [`CorpusStorage::find`](crate::CorpusStorage::find). + pub match_id: String, + /// The index of the alternative of the query that produced this match. + /// + /// A match that is produced by more than one alternative is only returned once and attributed to the first alternative that produces it. + /// + /// This uses the same numbering as [`QueryAttributeDescription::alternative`]. + /// Therefore, the descriptions returned from [`CorpusStorage::node_descriptions`](crate::CorpusStorage::node_descriptions) + /// can be filtered by this value and by [`QueryAttributeDescription::optional`] being `false` + /// to get the descriptions of the query nodes belonging to this alternative that are part of the output. + /// These are in the same order as the node annotation identifiers in `match_id`, so both can be paired up + /// to get the description of the query node that each matched node belongs to. + pub alternative: usize, +} + /// Definition of the result of a `frequency` query. pub type FrequencyTable = Vec>; diff --git a/graphannis/src/lib.rs b/graphannis/src/lib.rs index fee18146b..7a2775dac 100644 --- a/graphannis/src/lib.rs +++ b/graphannis/src/lib.rs @@ -34,7 +34,7 @@ pub mod corpusstorage { LoadStatus, QueryLanguage, ResultOrder, }; pub use crate::annis::types::{ - CountExtra, FrequencyTable, FrequencyTableRow, QueryAttributeDescription, + CountExtra, FrequencyTable, FrequencyTableRow, MatchExtra, QueryAttributeDescription, }; } From 087cca7ffff9398cb5f07487b20c0482a266703a Mon Sep 17 00:00:00 2001 From: Thomas Krause Date: Mon, 10 Aug 2026 16:44:38 +0200 Subject: [PATCH 2/2] Add copies of the find_ benchmarks but with find_extra The current benchmarks suggest there is a about 6% worse performance for find() queries and these benchmarks for find_extra() allow to compare the two methods and isolate the source of the increased workload. Since find_extra() and find() have roughly the same execution time, the source could be the added usize added as value (which increases the memory stored in the temporary BTreeMap) or some issue with refactoring out some of the work to closures or sub-functions which might not be inlined. --- graphannis/benches/graphannis.rs | 104 ++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/graphannis/benches/graphannis.rs b/graphannis/benches/graphannis.rs index 6fbbe0692..8603e52a1 100644 --- a/graphannis/benches/graphannis.rs +++ b/graphannis/benches/graphannis.rs @@ -74,6 +74,39 @@ fn find_all_nouns_gum(bench: &mut Criterion) { }); } +fn find_extra_all_nouns_gum(bench: &mut Criterion) { + if CORPUS_STORAGE.is_none() { + return; + } + + let cs = CORPUS_STORAGE.as_ref().unwrap(); + + let corpora = cs.list(); + if let Ok(corpora) = corpora { + let corpora: HashSet = corpora.into_iter().map(|c| c.name).collect(); + // ignore if corpus does not exist + if corpora.contains("GUM") { + cs.preload("GUM").unwrap(); + } else { + return; + } + } + + bench.bench_function("find_extra_all_nouns_gum", move |b| { + cs.preload("GUM").unwrap(); + b.iter(|| { + let query = SearchQuery { + corpus_names: &["GUM"], + query: "pos=\"NN\"", + query_language: QueryLanguage::AQL, + timeout: None, + }; + let f = cs.find_extra(query, 0, None, ResultOrder::Normal); + assert!(f.is_ok()); + }) + }); +} + fn find_first_ten_nouns_gum(bench: &mut Criterion) { if CORPUS_STORAGE.is_none() { return; @@ -107,6 +140,39 @@ fn find_first_ten_nouns_gum(bench: &mut Criterion) { }); } +fn find_extra_first_ten_nouns_gum(bench: &mut Criterion) { + if CORPUS_STORAGE.is_none() { + return; + } + + let cs = CORPUS_STORAGE.as_ref().unwrap(); + + let corpora = cs.list(); + if let Ok(corpora) = corpora { + let corpora: HashSet = corpora.into_iter().map(|c| c.name).collect(); + // ignore if corpus does not exist + if corpora.contains("GUM") { + cs.preload("GUM").unwrap(); + } else { + return; + } + } + + bench.bench_function("find_extra_first_ten_nouns_gum", move |b| { + cs.preload("GUM").unwrap(); + b.iter(|| { + let query = SearchQuery { + corpus_names: &["GUM"], + query: "pos=\"NN\"", + query_language: QueryLanguage::AQL, + timeout: None, + }; + let f = cs.find_extra(query, 0, Some(10), ResultOrder::Normal); + assert!(f.is_ok()); + }) + }); +} + fn count_extra_coref_anno(bench: &mut Criterion) { if CORPUS_STORAGE.is_none() { return; @@ -203,6 +269,39 @@ fn find_first_ten_token_gum(bench: &mut Criterion) { }); } +fn find_extra_first_ten_token_gum(bench: &mut Criterion) { + if CORPUS_STORAGE.is_none() { + return; + } + + let cs = CORPUS_STORAGE.as_ref().unwrap(); + + let corpora = cs.list(); + if let Ok(corpora) = corpora { + let corpora: HashSet = corpora.into_iter().map(|c| c.name).collect(); + // ignore if corpus does not exist + if corpora.contains("GUM") { + cs.preload("GUM").unwrap(); + } else { + return; + } + } + + bench.bench_function("find_extra_first_ten_token_gum", move |b| { + cs.preload("GUM").unwrap(); + b.iter(|| { + let query = SearchQuery { + corpus_names: &["GUM"], + query: "tok", + query_language: QueryLanguage::AQL, + timeout: None, + }; + let f = cs.find_extra(query, 0, Some(10), ResultOrder::Normal); + assert!(f.is_ok()); + }) + }); +} + fn deserialize_gum(bench: &mut Criterion) { if CORPUS_STORAGE.is_none() { return; @@ -333,8 +432,11 @@ fn apply_update_ondisk(bench: &mut Criterion) { criterion_group!(name=default; config= Criterion::default().sample_size(50); targets = deserialize_gum, find_first_ten_token_gum, - find_first_ten_nouns_gum, + find_first_ten_nouns_gum, find_all_nouns_gum, + find_extra_first_ten_nouns_gum, + find_extra_first_ten_token_gum, + find_extra_all_nouns_gum, count_extra_coref_anno, count_extra_node); criterion_group!(name=apply_update; config= Criterion::default().sample_size(20); targets = apply_update_inmemory,