From f298865c2edcb0e2837c03182c714a9b758ab52f Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 2 Oct 2026 09:03:54 -0700 Subject: [PATCH 1/4] Improve Hermit Crab performance --- Build/SilVersions.props | 3 ++- Src/LexText/ParserCore/FwXmlTraceManager.cs | 15 ++++++++--- Src/LexText/ParserCore/HCParser.cs | 16 +++++++++--- Src/LexText/ParserCore/ParserReport.cs | 1 + Src/LexText/ParserUI/ParserListener.cs | 25 ++----------------- Src/LexText/ParserUI/ParserParametersDlg.cs | 10 ++++++-- .../HCSynthByGlossLib/HcXmlTraceManager.cs | 19 ++++++++------ 7 files changed, 49 insertions(+), 40 deletions(-) diff --git a/Build/SilVersions.props b/Build/SilVersions.props index 54a3e0701d..72b1f9fa57 100644 --- a/Build/SilVersions.props +++ b/Build/SilVersions.props @@ -17,11 +17,12 @@ 18.0.0-beta0012 6.0.0-beta0065 3.0.1 - 3.9.2 + 3.9.2-master.gc7146cd 1.1.1-beta0001 10.0.0-beta0014 0.9.8 70.1.152 60.0.56 + C:\localnugetpackages diff --git a/Src/LexText/ParserCore/FwXmlTraceManager.cs b/Src/LexText/ParserCore/FwXmlTraceManager.cs index e900730974..cca78071ba 100644 --- a/Src/LexText/ParserCore/FwXmlTraceManager.cs +++ b/Src/LexText/ParserCore/FwXmlTraceManager.cs @@ -69,11 +69,20 @@ public void BeginUnapplyTemplate(AffixTemplate template, Word input) CreateWordElement("Input", input, true))); } - public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied) + public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason) { - ((XElement) output.CurrentTrace).Add(new XElement("TemplateAnalysisTraceOut", + var trace = new XElement( + "TemplateAnalysisTraceOut", CreateHCRuleElement("AffixTemplate", template), - CreateWordElement("Output", unapplied ? output : null, true))); + CreateWordElement("Output", unapplied ? output : null, true) + ); + if (reason == FailureReason.NonPartialRuleProhibitedAfterFinalTemplate) + { + trace.Add( + new XElement("FailureReason", new XAttribute("type", "nonPartialRuleAfterFinalTemplate")) + ); + } + ((XElement)output.CurrentTrace).Add(trace); } public void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output) diff --git a/Src/LexText/ParserCore/HCParser.cs b/Src/LexText/ParserCore/HCParser.cs index 3462f37861..2f1985e8ab 100644 --- a/Src/LexText/ParserCore/HCParser.cs +++ b/Src/LexText/ParserCore/HCParser.cs @@ -34,6 +34,7 @@ public class HCParser : DisposableBase, IParser private bool m_guessRoots; private bool m_mergeAnalyses; private bool m_mergeMSAs = true; + private bool m_makePartialsFinal = false; // the public const strings are for GenerateHCConfigForFLExTrans and HCSynthByGlossLib internal const string CRuleID = "ID"; @@ -164,6 +165,7 @@ private void LoadParser() XElement guessRootsElem = parserParamsElem.Elements("HC").Elements("GuessRoots").FirstOrDefault(); XElement mergeAnalysesElem = parserParamsElem.Elements("HC").Elements("MergeAnalyses").FirstOrDefault(); XElement mergeMSAsElem = parserParamsElem.Elements("HC").Elements("MergeMSAs").FirstOrDefault(); + XElement makePartialsFinalElem = parserParamsElem.Elements("HC").Elements("MakePartialsFinal").FirstOrDefault(); XElement maxRootsElem = parserParamsElem.Elements("HC").Elements("MaxRoots").FirstOrDefault(); XElement maxAlternativesElem = parserParamsElem.Elements("HC").Elements("MaxAlternatives").FirstOrDefault(); if (delReappsElem != null) @@ -174,6 +176,8 @@ private void LoadParser() m_mergeAnalyses = (bool)mergeAnalysesElem; if (mergeMSAsElem != null) m_mergeMSAs = (bool)mergeMSAsElem; + if (makePartialsFinalElem != null) + m_makePartialsFinal = (bool)makePartialsFinalElem; if (maxRootsElem != null) maxStemCount = int.Parse(maxRootsElem.Value); if (maxAlternativesElem != null) @@ -190,10 +194,14 @@ private void LoadParser() } writer.WriteEndElement(); } - m_morpher = new Morpher(m_traceManager, m_language) { DeletionReapplications = delReapps }; - m_morpher.MaxStemCount = maxStemCount; - m_morpher.MergeEquivalentAnalyses = m_mergeAnalyses; - m_morpher.MaxAlternatives = maxAlternatives; + m_morpher = new Morpher(m_traceManager, m_language) + { + DeletionReapplications = delReapps, + MaxStemCount = maxStemCount, + MergeEquivalentAnalyses = m_mergeAnalyses, + MaxAlternatives = maxAlternatives, + AlwaysEnforceFinalTemplates = m_makePartialsFinal, + }; } private XDocument ParseToXml(string form, bool tracing, IEnumerable selectTraceMorphs) diff --git a/Src/LexText/ParserCore/ParserReport.cs b/Src/LexText/ParserCore/ParserReport.cs index cb5bf9310d..7c30eaacf6 100644 --- a/Src/LexText/ParserCore/ParserReport.cs +++ b/Src/LexText/ParserCore/ParserReport.cs @@ -248,6 +248,7 @@ public ParserReport DiffParserReports(ParserReport other) // The following must go after AddParserReport is called. diff.IsDiff = true; + diff.ChangesRecorded = ChangesRecorded || other.ChangesRecorded; diff.ProjectName = DiffNames(ProjectName, other.ProjectName); diff.SourceText = DiffNames(SourceText, other.SourceText); diff.MachineName = DiffNames(MachineName, other.MachineName); diff --git a/Src/LexText/ParserUI/ParserListener.cs b/Src/LexText/ParserUI/ParserListener.cs index a24085f25c..57465ca668 100644 --- a/Src/LexText/ParserUI/ParserListener.cs +++ b/Src/LexText/ParserUI/ParserListener.cs @@ -897,10 +897,8 @@ private static Form CreateInputBox(string title, ref string input) } /// - /// Suppress this parse result if it is an uppercase wordform whose analyses all came from its lowercase version. - /// This only happens in projects that were parsed before we decided that the case of wordforms in analyses - /// should be determined by the case of the word that was parsed rather than the case of the surface form. - /// So, the wordform for "The" should be "the" rather than "The" because "The" is parsed as the determiner "the". + /// Suppress this parse result if it is an uppercase wordform with no analyses + /// that has a lowercase wordform in the same corpus. /// /// /// @@ -923,25 +921,6 @@ private bool SuppressableParseResult(IWfiWordform wordform) { if (m_checkParserResults.ContainsKey(lcWordform)) { - var lcResult = m_checkParserResults[lcWordform]; - // See if lcResult covers wordform's approved analyses. - var userAgent = wordform.Cache.LanguageProject.DefaultUserAgent; - foreach (IWfiAnalysis wfAnalysis in wordform.AnalysesOC) - { - var wfOpinion = wfAnalysis.GetAgentOpinion(userAgent); - if (wfOpinion == Opinions.approves) - { - foreach (ParseAnalysis lcWfAnalysis in lcResult.Analyses) - { - if (!lcWfAnalysis.MatchesIWfiAnalysis(wfAnalysis)) - { - return false; - } - } - } - } - // All approved analyses are covered. - // Suppress the parse results for wordform. return true; } } diff --git a/Src/LexText/ParserUI/ParserParametersDlg.cs b/Src/LexText/ParserUI/ParserParametersDlg.cs index 2979af9a65..c43add1e8a 100644 --- a/Src/LexText/ParserUI/ParserParametersDlg.cs +++ b/Src/LexText/ParserUI/ParserParametersDlg.cs @@ -35,6 +35,7 @@ public class ParserParametersDlg : ParserParametersBase private const string GuessRoots = "GuessRoots"; private const string MergeAnalyses = "MergeAnalyses"; private const string MergeMSAs = "MergeMSAs"; + private const string MakePartialsFinal = "MakePartialsFinal"; private const string Strata = "Strata"; private const string XAmple = "XAmple"; @@ -281,11 +282,13 @@ public void SetDlgInfo(string title, string parserParameters, ILcmOwningSequence PopulateDataGrid(m_dataGrid1, XAmple); PopulateDataGrid(m_dataGrid2, HC); m_dataGrid2.TableStyles[0].GridColumnStyles[2].Width = 130; + m_dataGrid2.TableStyles[0].GridColumnStyles[3].Width = 70; m_dataGrid2.TableStyles[0].GridColumnStyles[4].Width = 160; m_dataGrid2.TableStyles[0].GridColumnStyles[6].Width = 90; - m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 80; + m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 70; m_dataGrid2.TableStyles[0].GridColumnStyles[8].Width = 100; - m_dataGrid2.TableStyles[0].GridColumnStyles[9].Width = 400; + m_dataGrid2.TableStyles[0].GridColumnStyles[9].Width = 100; + m_dataGrid2.TableStyles[0].GridColumnStyles[10].Width = 400; m_compoundRules = compoundRules; if (m_compoundRules?.Count > 0) @@ -322,6 +325,8 @@ private void LoadParserData(DataSet dsParserParameters) hcElem.Add(new XElement(MergeAnalyses, true)); if (hcElem.Element(MergeMSAs) == null) hcElem.Add(new XElement(MergeMSAs, true)); + if (hcElem.Element(MakePartialsFinal) == null) + hcElem.Add(new XElement(MakePartialsFinal, false)); if (hcElem.Element(Strata) == null) hcElem.Add(new XElement(Strata, "")); @@ -377,6 +382,7 @@ private DataTable CreateHCDataTable() tblHC.Columns.Add(GuessRoots, typeof(bool)); tblHC.Columns.Add(MergeAnalyses, typeof(bool)); tblHC.Columns.Add(MergeMSAs, typeof(bool)); + tblHC.Columns.Add(MakePartialsFinal, typeof(bool)); tblHC.Columns.Add(MaxAlternatives, typeof(int)); tblHC.Columns.Add(Strata, typeof(string)); return tblHC; diff --git a/Src/Utilities/HCSynthByGloss/HCSynthByGlossLib/HcXmlTraceManager.cs b/Src/Utilities/HCSynthByGloss/HCSynthByGlossLib/HcXmlTraceManager.cs index 35d5492751..ed77387dd2 100644 --- a/Src/Utilities/HCSynthByGloss/HCSynthByGlossLib/HcXmlTraceManager.cs +++ b/Src/Utilities/HCSynthByGloss/HCSynthByGlossLib/HcXmlTraceManager.cs @@ -97,15 +97,20 @@ public void BeginUnapplyTemplate(AffixTemplate template, Word input) ); } - public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied) + public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason) { - ((XElement)output.CurrentTrace).Add( - new XElement( - "TemplateAnalysisTraceOut", - CreateHCRuleElement("AffixTemplate", template), - CreateWordElement("Output", unapplied ? output : null, true) - ) + var trace = new XElement( + "TemplateAnalysisTraceOut", + CreateHCRuleElement("AffixTemplate", template), + CreateWordElement("Output", unapplied ? output : null, true) ); + if (reason == FailureReason.NonPartialRuleProhibitedAfterFinalTemplate) + { + trace.Add( + new XElement("FailureReason", new XAttribute("type", "nonPartialRuleAfterFinalTemplate")) + ); + } + ((XElement)output.CurrentTrace).Add(trace); } public void MorphologicalRuleUnapplied( From 79e8a4682f8e7597d61e473cdc4c9f1a824e0750 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Tue, 6 Oct 2026 09:10:33 -0700 Subject: [PATCH 2/4] Add transforms and fix bugs --- Build/SilVersions.props | 3 +- Src/LexText/ParserCore/HCParser.cs | 8 +++-- Src/LexText/ParserUI/ParserListener.cs | 2 +- Src/Transforms/Presentation/FormatHCTrace.xsl | 29 ++++++++++++++++-- .../HCSynthByGlossFormatHCTrace.xsl | 30 +++++++++++++++++-- 5 files changed, 62 insertions(+), 10 deletions(-) diff --git a/Build/SilVersions.props b/Build/SilVersions.props index 72b1f9fa57..59c33bbad5 100644 --- a/Build/SilVersions.props +++ b/Build/SilVersions.props @@ -17,12 +17,11 @@ 18.0.0-beta0012 6.0.0-beta0065 3.0.1 - 3.9.2-master.gc7146cd + 3.9.5 1.1.1-beta0001 10.0.0-beta0014 0.9.8 70.1.152 60.0.56 - C:\localnugetpackages diff --git a/Src/LexText/ParserCore/HCParser.cs b/Src/LexText/ParserCore/HCParser.cs index 2f1985e8ab..bef8bb5673 100644 --- a/Src/LexText/ParserCore/HCParser.cs +++ b/Src/LexText/ParserCore/HCParser.cs @@ -35,6 +35,7 @@ public class HCParser : DisposableBase, IParser private bool m_mergeAnalyses; private bool m_mergeMSAs = true; private bool m_makePartialsFinal = false; + private string m_parserParameters; // the public const strings are for GenerateHCConfigForFLExTrans and HCSynthByGlossLib internal const string CRuleID = "ID"; @@ -72,7 +73,9 @@ public bool IsUpToDate() public void Update() { CheckDisposed(); - if (m_changeListener.Reset() || m_forceUpdate) + if (m_changeListener.Reset() + || m_forceUpdate + || m_parserParameters != m_cache.LanguageProject.MorphologicalDataOA.ParserParameters) { LoadParser(); m_forceUpdate = false; @@ -160,7 +163,8 @@ private void LoadParser() using (XmlWriter writer = XmlWriter.Create(loadErrorsFile)) using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance())) { - XElement parserParamsElem = XElement.Parse(m_cache.LanguageProject.MorphologicalDataOA.ParserParameters); + m_parserParameters = m_cache.LanguageProject.MorphologicalDataOA.ParserParameters; + XElement parserParamsElem = XElement.Parse(m_parserParameters); XElement delReappsElem = parserParamsElem.Elements("HC").Elements("DelReapps").FirstOrDefault(); XElement guessRootsElem = parserParamsElem.Elements("HC").Elements("GuessRoots").FirstOrDefault(); XElement mergeAnalysesElem = parserParamsElem.Elements("HC").Elements("MergeAnalyses").FirstOrDefault(); diff --git a/Src/LexText/ParserUI/ParserListener.cs b/Src/LexText/ParserUI/ParserListener.cs index 57465ca668..8838f03dda 100644 --- a/Src/LexText/ParserUI/ParserListener.cs +++ b/Src/LexText/ParserUI/ParserListener.cs @@ -905,7 +905,7 @@ private static Form CreateInputBox(string title, ref string input) private bool SuppressableParseResult(IWfiWordform wordform) { var result = m_checkParserResults[wordform]; - if (result.Analyses.Count > 0) + if (result.Analyses.Count > 0 || !String.IsNullOrEmpty(result.ErrorMessage)) return false; // See if there is a lowercase version of wordform in the parse results. ITsString itsString = wordform.Form.VernacularDefaultWritingSystem; diff --git a/Src/Transforms/Presentation/FormatHCTrace.xsl b/Src/Transforms/Presentation/FormatHCTrace.xsl index 75fd83b298..7a74557479 100644 --- a/Src/Transforms/Presentation/FormatHCTrace.xsl +++ b/Src/Transforms/Presentation/FormatHCTrace.xsl @@ -106,7 +106,8 @@ Main template - + + Y @@ -1244,8 +1245,9 @@ ShowMorph + - + @@ -1351,6 +1353,29 @@ ShowMorph + + + + + color: + + + Template + + + + + + + + + failed. + + + + + + diff --git a/Src/Transforms/Presentation/HCSynthByGlossFormatHCTrace.xsl b/Src/Transforms/Presentation/HCSynthByGlossFormatHCTrace.xsl index f421f07f0a..58e51073a2 100644 --- a/Src/Transforms/Presentation/HCSynthByGlossFormatHCTrace.xsl +++ b/Src/Transforms/Presentation/HCSynthByGlossFormatHCTrace.xsl @@ -115,7 +115,8 @@ Main template - + + Y @@ -1359,8 +1360,8 @@ ShowMorph - - + + @@ -1466,6 +1467,29 @@ ShowMorph + + + + + color: + + + Template + + + + + + + + + failed. + + + + + + From fb642bd6f1a602f2700f5594ec6f0f22f11ee049 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Wed, 7 Oct 2026 08:19:44 -0700 Subject: [PATCH 3/4] Address Devin's review; fix bugs --- Src/LexText/ParserCore/HCParser.cs | 14 ++++---------- Src/LexText/ParserCore/ParserWorker.cs | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Src/LexText/ParserCore/HCParser.cs b/Src/LexText/ParserCore/HCParser.cs index bef8bb5673..f588a0eeef 100644 --- a/Src/LexText/ParserCore/HCParser.cs +++ b/Src/LexText/ParserCore/HCParser.cs @@ -35,7 +35,6 @@ public class HCParser : DisposableBase, IParser private bool m_mergeAnalyses; private bool m_mergeMSAs = true; private bool m_makePartialsFinal = false; - private string m_parserParameters; // the public const strings are for GenerateHCConfigForFLExTrans and HCSynthByGlossLib internal const string CRuleID = "ID"; @@ -73,9 +72,7 @@ public bool IsUpToDate() public void Update() { CheckDisposed(); - if (m_changeListener.Reset() - || m_forceUpdate - || m_parserParameters != m_cache.LanguageProject.MorphologicalDataOA.ParserParameters) + if (m_changeListener.Reset() || m_forceUpdate) { LoadParser(); m_forceUpdate = false; @@ -163,8 +160,7 @@ private void LoadParser() using (XmlWriter writer = XmlWriter.Create(loadErrorsFile)) using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance())) { - m_parserParameters = m_cache.LanguageProject.MorphologicalDataOA.ParserParameters; - XElement parserParamsElem = XElement.Parse(m_parserParameters); + XElement parserParamsElem = XElement.Parse(m_cache.LanguageProject.MorphologicalDataOA.ParserParameters); XElement delReappsElem = parserParamsElem.Elements("HC").Elements("DelReapps").FirstOrDefault(); XElement guessRootsElem = parserParamsElem.Elements("HC").Elements("GuessRoots").FirstOrDefault(); XElement mergeAnalysesElem = parserParamsElem.Elements("HC").Elements("MergeAnalyses").FirstOrDefault(); @@ -178,14 +174,12 @@ private void LoadParser() m_guessRoots = (bool) guessRootsElem; if (mergeAnalysesElem != null) m_mergeAnalyses = (bool)mergeAnalysesElem; - if (mergeMSAsElem != null) - m_mergeMSAs = (bool)mergeMSAsElem; - if (makePartialsFinalElem != null) - m_makePartialsFinal = (bool)makePartialsFinalElem; if (maxRootsElem != null) maxStemCount = int.Parse(maxRootsElem.Value); if (maxAlternativesElem != null) maxAlternatives = int.Parse(maxAlternativesElem.Value); + m_mergeMSAs = mergeMSAsElem != null ? (bool)mergeMSAsElem : true; + m_makePartialsFinal = makePartialsFinalElem != null ? (bool)makePartialsFinalElem : false; writer.WriteStartElement("LoadErrors"); if (m_mergeMSAs) { diff --git a/Src/LexText/ParserCore/ParserWorker.cs b/Src/LexText/ParserCore/ParserWorker.cs index dd8260963f..b6c270d6e8 100644 --- a/Src/LexText/ParserCore/ParserWorker.cs +++ b/Src/LexText/ParserCore/ParserWorker.cs @@ -173,7 +173,7 @@ public bool ParseAndUpdateWordform(IWfiWordform wordform, ParserPriority priorit } stopWatch.Stop(); lcResult.ParseTime = stopWatch.ElapsedMilliseconds; - if (lcResult.Analyses.Count > 0 && lcResult.ErrorMessage == null) + if (lcResult.Analyses.Count > 0 || !String.IsNullOrEmpty(lcResult.ErrorMessage)) { // Don't turn lcText into a wordform here. // This avoids a problem with broadcasting PropChanged (cf. LT-22079). From 092b8fa05efd9011f0d7341ba4d748a5922f071b Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 8 Oct 2026 12:53:29 -0700 Subject: [PATCH 4/4] Respond to Jason's comment --- Src/LexText/ParserCore/ParserWorker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/LexText/ParserCore/ParserWorker.cs b/Src/LexText/ParserCore/ParserWorker.cs index b6c270d6e8..70faf66cc9 100644 --- a/Src/LexText/ParserCore/ParserWorker.cs +++ b/Src/LexText/ParserCore/ParserWorker.cs @@ -173,7 +173,7 @@ public bool ParseAndUpdateWordform(IWfiWordform wordform, ParserPriority priorit } stopWatch.Stop(); lcResult.ParseTime = stopWatch.ElapsedMilliseconds; - if (lcResult.Analyses.Count > 0 || !String.IsNullOrEmpty(lcResult.ErrorMessage)) + if (lcResult.Analyses.Count > 0 || (checkParser && !String.IsNullOrEmpty(lcResult.ErrorMessage))) { // Don't turn lcText into a wordform here. // This avoids a problem with broadcasting PropChanged (cf. LT-22079).