Skip to content

LT-22780: Improve Hermit Crab performance #1176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jtmaxwell3 merged 4 commits into from
Oct 8, 2026
Merged

LT-22780: Improve Hermit Crab performance #1176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f298865
Improve Hermit Crab performance
jtmaxwell3 Oct 2, 2026
79e8a46
Add transforms and fix bugs
jtmaxwell3 Oct 6, 2026
fb642bd
Address Devin's review; fix bugs
jtmaxwell3 Oct 7, 2026
092b8fa
Respond to Jason's comment
jtmaxwell3 Oct 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Build/SilVersions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<SilLibPalasoL10nsVersion>18.0.0-beta0012</SilLibPalasoL10nsVersion>
<SilChorusVersion>6.0.0-beta0065</SilChorusVersion>
<SilChorusL10nsVersion>3.0.1</SilChorusL10nsVersion>
<SilMachineVersion>3.9.2</SilMachineVersion>
<SilMachineVersion>3.9.5</SilMachineVersion>
<SilIPCFrameworkVersion>1.1.1-beta0001</SilIPCFrameworkVersion>
<L10NSharpVersion>10.0.0-beta0014</L10NSharpVersion>
<EncodingConvertersCoreVersion>0.9.8</EncodingConvertersCoreVersion>
Expand Down
15 changes: 12 additions & 3 deletions Src/LexText/ParserCore/FwXmlTraceManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,20 @@ public void BeginUnapplyTemplate(AffixTemplate template, Word input)
CreateWordElement("Input", input, true)));
}

public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied)
public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason)
{
((XElement) output.CurrentTrace).Add(new XElement("TemplateAnalysisTraceOut",
var trace = new XElement(
"TemplateAnalysisTraceOut",
CreateHCRuleElement("AffixTemplate", template),
CreateWordElement("Output", unapplied ? output : null, true)));
CreateWordElement("Output", unapplied ? output : null, true)
);
if (reason == FailureReason.NonPartialRuleProhibitedAfterFinalTemplate)
{
trace.Add(
new XElement("FailureReason", new XAttribute("type", "nonPartialRuleAfterFinalTemplate"))
);
}
((XElement)output.CurrentTrace).Add(trace);
}

public void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output)
Expand Down
18 changes: 12 additions & 6 deletions Src/LexText/ParserCore/HCParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class HCParser : DisposableBase, IParser
private bool m_guessRoots;
private bool m_mergeAnalyses;
private bool m_mergeMSAs = true;
private bool m_makePartialsFinal = false;

// the public const strings are for GenerateHCConfigForFLExTrans and HCSynthByGlossLib
internal const string CRuleID = "ID";
Expand Down Expand Up @@ -164,6 +165,7 @@ private void LoadParser()
XElement guessRootsElem = parserParamsElem.Elements("HC").Elements("GuessRoots").FirstOrDefault();
XElement mergeAnalysesElem = parserParamsElem.Elements("HC").Elements("MergeAnalyses").FirstOrDefault();
XElement mergeMSAsElem = parserParamsElem.Elements("HC").Elements("MergeMSAs").FirstOrDefault();
XElement makePartialsFinalElem = parserParamsElem.Elements("HC").Elements("MakePartialsFinal").FirstOrDefault();
XElement maxRootsElem = parserParamsElem.Elements("HC").Elements("MaxRoots").FirstOrDefault();
XElement maxAlternativesElem = parserParamsElem.Elements("HC").Elements("MaxAlternatives").FirstOrDefault();
if (delReappsElem != null)
Expand All @@ -172,12 +174,12 @@ private void LoadParser()
m_guessRoots = (bool) guessRootsElem;
if (mergeAnalysesElem != null)
m_mergeAnalyses = (bool)mergeAnalysesElem;
if (mergeMSAsElem != null)
m_mergeMSAs = (bool)mergeMSAsElem;
if (maxRootsElem != null)
maxStemCount = int.Parse(maxRootsElem.Value);
if (maxAlternativesElem != null)
maxAlternatives = int.Parse(maxAlternativesElem.Value);
m_mergeMSAs = mergeMSAsElem != null ? (bool)mergeMSAsElem : true;
m_makePartialsFinal = makePartialsFinalElem != null ? (bool)makePartialsFinalElem : false;
writer.WriteStartElement("LoadErrors");
if (m_mergeMSAs)
{
Expand All @@ -190,10 +192,14 @@ private void LoadParser()
}
writer.WriteEndElement();
}
m_morpher = new Morpher(m_traceManager, m_language) { DeletionReapplications = delReapps };
m_morpher.MaxStemCount = maxStemCount;
m_morpher.MergeEquivalentAnalyses = m_mergeAnalyses;
m_morpher.MaxAlternatives = maxAlternatives;
m_morpher = new Morpher(m_traceManager, m_language)
{
DeletionReapplications = delReapps,
MaxStemCount = maxStemCount,
MergeEquivalentAnalyses = m_mergeAnalyses,
MaxAlternatives = maxAlternatives,
AlwaysEnforceFinalTemplates = m_makePartialsFinal,
};
}

private XDocument ParseToXml(string form, bool tracing, IEnumerable<int> selectTraceMorphs)
Expand Down
1 change: 1 addition & 0 deletions Src/LexText/ParserCore/ParserReport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ public ParserReport DiffParserReports(ParserReport other)

// The following must go after AddParserReport is called.
diff.IsDiff = true;
diff.ChangesRecorded = ChangesRecorded || other.ChangesRecorded;
diff.ProjectName = DiffNames(ProjectName, other.ProjectName);
diff.SourceText = DiffNames(SourceText, other.SourceText);
diff.MachineName = DiffNames(MachineName, other.MachineName);
Expand Down
2 changes: 1 addition & 1 deletion Src/LexText/ParserCore/ParserWorker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public bool ParseAndUpdateWordform(IWfiWordform wordform, ParserPriority priorit
}
stopWatch.Stop();
lcResult.ParseTime = stopWatch.ElapsedMilliseconds;
if (lcResult.Analyses.Count > 0 && lcResult.ErrorMessage == null)
if (lcResult.Analyses.Count > 0 || (checkParser && !String.IsNullOrEmpty(lcResult.ErrorMessage)))

@jasonleenaylor jasonleenaylor Oct 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Under Check Parser, an error-only lowercase result still looks like it creates the lowercase wordform, because ParseFiler.UpdateWordforms calls FindOrCreateWordform before it checks CheckParserUpdatesAnalyses. So even a Check Parser run that is "just a test" can add one. I'm not sure if this is an issue for real world use or not so I'm just pointing it out for you to consider.

{
// Don't turn lcText into a wordform here.
// This avoids a problem with broadcasting PropChanged (cf. LT-22079).
Expand Down
27 changes: 3 additions & 24 deletions Src/LexText/ParserUI/ParserListener.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -897,17 +897,15 @@ private static Form CreateInputBox(string title, ref string input)
}

/// <summary>
/// Suppress this parse result if it is an uppercase wordform whose analyses all came from its lowercase version.
/// This only happens in projects that were parsed before we decided that the case of wordforms in analyses
/// should be determined by the case of the word that was parsed rather than the case of the surface form.
/// So, the wordform for "The" should be "the" rather than "The" because "The" is parsed as the determiner "the".
/// Suppress this parse result if it is an uppercase wordform with no analyses
/// that has a lowercase wordform in the same corpus.
/// </summary>
/// <param name="wordform"></param>
/// <returns></returns>
private bool SuppressableParseResult(IWfiWordform wordform)
{
var result = m_checkParserResults[wordform];
if (result.Analyses.Count > 0)
if (result.Analyses.Count > 0 || !String.IsNullOrEmpty(result.ErrorMessage))
return false;
// See if there is a lowercase version of wordform in the parse results.
ITsString itsString = wordform.Form.VernacularDefaultWritingSystem;
Expand All @@ -923,25 +921,6 @@ private bool SuppressableParseResult(IWfiWordform wordform)
{
if (m_checkParserResults.ContainsKey(lcWordform))
{
var lcResult = m_checkParserResults[lcWordform];
// See if lcResult covers wordform's approved analyses.
var userAgent = wordform.Cache.LanguageProject.DefaultUserAgent;
foreach (IWfiAnalysis wfAnalysis in wordform.AnalysesOC)
{
var wfOpinion = wfAnalysis.GetAgentOpinion(userAgent);
if (wfOpinion == Opinions.approves)
{
foreach (ParseAnalysis lcWfAnalysis in lcResult.Analyses)
{
if (!lcWfAnalysis.MatchesIWfiAnalysis(wfAnalysis))
{
return false;
}
}
}
}
// All approved analyses are covered.
// Suppress the parse results for wordform.
return true;
}
}
Expand Down
10 changes: 8 additions & 2 deletions Src/LexText/ParserUI/ParserParametersDlg.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class ParserParametersDlg : ParserParametersBase
private const string GuessRoots = "GuessRoots";
private const string MergeAnalyses = "MergeAnalyses";
private const string MergeMSAs = "MergeMSAs";
private const string MakePartialsFinal = "MakePartialsFinal";
private const string Strata = "Strata";

private const string XAmple = "XAmple";
Expand Down Expand Up @@ -281,11 +282,13 @@ public void SetDlgInfo(string title, string parserParameters, ILcmOwningSequence
PopulateDataGrid(m_dataGrid1, XAmple);
PopulateDataGrid(m_dataGrid2, HC);
m_dataGrid2.TableStyles[0].GridColumnStyles[2].Width = 130;
m_dataGrid2.TableStyles[0].GridColumnStyles[3].Width = 70;
m_dataGrid2.TableStyles[0].GridColumnStyles[4].Width = 160;
m_dataGrid2.TableStyles[0].GridColumnStyles[6].Width = 90;
m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 80;
m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 70;
m_dataGrid2.TableStyles[0].GridColumnStyles[8].Width = 100;
m_dataGrid2.TableStyles[0].GridColumnStyles[9].Width = 400;
m_dataGrid2.TableStyles[0].GridColumnStyles[9].Width = 100;
m_dataGrid2.TableStyles[0].GridColumnStyles[10].Width = 400;

m_compoundRules = compoundRules;
if (m_compoundRules?.Count > 0)
Expand Down Expand Up @@ -322,6 +325,8 @@ private void LoadParserData(DataSet dsParserParameters)
hcElem.Add(new XElement(MergeAnalyses, true));
if (hcElem.Element(MergeMSAs) == null)
hcElem.Add(new XElement(MergeMSAs, true));
if (hcElem.Element(MakePartialsFinal) == null)
hcElem.Add(new XElement(MakePartialsFinal, false));
if (hcElem.Element(Strata) == null)
hcElem.Add(new XElement(Strata, ""));

Expand Down Expand Up @@ -377,6 +382,7 @@ private DataTable CreateHCDataTable()
tblHC.Columns.Add(GuessRoots, typeof(bool));
tblHC.Columns.Add(MergeAnalyses, typeof(bool));
tblHC.Columns.Add(MergeMSAs, typeof(bool));
tblHC.Columns.Add(MakePartialsFinal, typeof(bool));
tblHC.Columns.Add(MaxAlternatives, typeof(int));
tblHC.Columns.Add(Strata, typeof(string));
return tblHC;
Expand Down
29 changes: 27 additions & 2 deletions Src/Transforms/Presentation/FormatHCTrace.xsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ Main template
<xsl:variable name="synthesisCompoundRules" select="$traceRoot/CompoundingRuleSynthesisTrace[MorphologicalRule/@type = 'compound'] | $traceRoot/MorphologicalRuleSynthesisTrace[MorphologicalRule/@type = 'compound']"/>
<xsl:variable name="synthesizedWords" select="$traceRoot/LexLookupTrace/WordSynthesisTrace"/>
<xsl:variable name="parseCompleteTraces" select="$traceRoot/ParseCompleteTrace"/>
<xsl:variable name="parseNodes" select="$analysisAffixes | $synthesisAffixes | $analysisCompoundRules | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces"/>
<xsl:variable name="templateFailures" select="$traceRoot/TemplateAnalysisTraceOut[FailureReason]"/>
<xsl:variable name="parseNodes" select="$analysisAffixes | $synthesisAffixes | $analysisCompoundRules | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces | $templateFailures"/>
<xsl:if test="$parseNodes">
<xsl:text>Y</xsl:text>
</xsl:if>
Expand Down Expand Up @@ -1244,8 +1245,9 @@ ShowMorph
<xsl:variable name="synthesisCompoundRules" select="$traceRoot/CompoundingRuleSynthesisTrace[MorphologicalRule/@type = 'compound'] | $traceRoot/MorphologicalRuleSynthesisTrace[MorphologicalRule/@type = 'compound']"/>
<xsl:variable name="synthesizedWords" select="$traceRoot/LexLookupTrace/WordSynthesisTrace"/>
<xsl:variable name="parseCompleteTraces" select="$traceRoot/ParseCompleteTrace"/>
<xsl:variable name="templateFailures" select="$traceRoot/TemplateAnalysisTraceOut[FailureReason]"/>

<xsl:variable name="parseNodes" select="$analysisAffixes | $synthesisAffixes | $analysisCompoundRules | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces"/>
<xsl:variable name="parseNodes" select="$analysisAffixes | $synthesisAffixes | $analysisCompoundRules | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces | $templateFailures"/>
<xsl:if test="$parseNodes">
<xsl:for-each select="$parseNodes">
<xsl:variable name="lastTemplateTrace" select="(preceding-sibling::*[name()='TemplateAnalysisTraceIn' or name()='TemplateSynthesisTraceIn' or name()='TemplateAnalysisTraceOut' or name()='TemplateSynthesisTraceOut'])[position() = last()]"/>
Expand Down Expand Up @@ -1351,6 +1353,29 @@ ShowMorph
</td>
</tr>
</xsl:when>
<xsl:when test="name() = 'TemplateAnalysisTraceOut'">
<tr>
<td valign="top">
<xsl:attribute name="style">
<xsl:text>color:</xsl:text>
<xsl:value-of select="$sFailureColor"/>
</xsl:attribute>
<xsl:text>Template </xsl:text>
<xsl:if test="string-length(normalize-space($template)) > 0">
<span>
<xsl:attribute name="style">
<xsl:call-template name="GetAnalysisFont"/>
</xsl:attribute>
<xsl:value-of select="normalize-space($template)"/>
</span>
</xsl:if>
<xsl:text> failed.</xsl:text>
</td>
<td valign="top">
<xsl:call-template name="ShowAnyFailure"/>
</td>
</tr>
</xsl:when>
<xsl:otherwise>
<tr>
<td valign="top">
Expand Down
30 changes: 27 additions & 3 deletions Src/Transforms/Presentation/HCSynthByGlossFormatHCTrace.xsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ Main template
<xsl:variable name="synthesisCompoundRules" select="$traceRoot/MorphologicalRuleSynthesisTrace[MorphologicalRule/@type = 'compound']"/>
<xsl:variable name="synthesizedWords" select="$traceRoot/LexLookupTrace/WordSynthesisTrace"/>
<xsl:variable name="parseCompleteTraces" select="$traceRoot/ParseCompleteTrace"/>
<xsl:variable name="parseNodes" select="$synthesisAffixes | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces"/>
<xsl:variable name="templateFailures" select="$traceRoot/TemplateAnalysisTraceOut[FailureReason]"/>
<xsl:variable name="parseNodes" select="$synthesisAffixes | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces | $templateFailures"/>
<xsl:if test="$parseNodes">
<xsl:text>Y</xsl:text>
</xsl:if>
Expand Down Expand Up @@ -1359,8 +1360,8 @@ ShowMorph
<xsl:variable name="synthesisCompoundRules" select="$traceRoot/MorphologicalRuleSynthesisTrace[MorphologicalRule/@type = 'compound']"/>
<xsl:variable name="synthesizedWords" select="$traceRoot/LexLookupTrace/WordSynthesisTrace"/>
<xsl:variable name="parseCompleteTraces" select="$traceRoot/ParseCompleteTrace"/>

<xsl:variable name="parseNodes" select="$synthesisAffixes | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces"/>
<xsl:variable name="templateFailures" select="$traceRoot/TemplateAnalysisTraceOut[FailureReason]"/>
<xsl:variable name="parseNodes" select="$synthesisAffixes | $synthesisCompoundRules | $synthesizedWords | $parseCompleteTraces | $templateFailures"/>
<xsl:if test="$parseNodes">
<xsl:for-each select="$parseNodes">
<xsl:variable name="lastTemplateTrace" select="(preceding-sibling::*[name()='TemplateSynthesisTraceIn' or name()='TemplateSynthesisTraceOut'])[position() = last()]"/>
Expand Down Expand Up @@ -1466,6 +1467,29 @@ ShowMorph
</td>
</tr>
</xsl:when>
<xsl:when test="name() = 'TemplateAnalysisTraceOut'">
<tr>
<td valign="top">
<xsl:attribute name="style">
<xsl:text>color:</xsl:text>
<xsl:value-of select="$sFailureColor"/>
</xsl:attribute>
<xsl:text>Template </xsl:text>
<xsl:if test="string-length(normalize-space($template)) > 0">
<span>
<xsl:attribute name="style">
<xsl:call-template name="GetAnalysisFont"/>
</xsl:attribute>
<xsl:value-of select="normalize-space($template)"/>
</span>
</xsl:if>
<xsl:text> failed.</xsl:text>
</td>
<td valign="top">
<xsl:call-template name="ShowAnyFailure"/>
</td>
</tr>
</xsl:when>
<xsl:otherwise>
<tr>
<td valign="top">
Expand Down
19 changes: 12 additions & 7 deletions Src/Utilities/HCSynthByGloss/HCSynthByGlossLib/HcXmlTraceManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,15 +97,20 @@ public void BeginUnapplyTemplate(AffixTemplate template, Word input)
);
}

public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied)
public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason)
{
((XElement)output.CurrentTrace).Add(
new XElement(
"TemplateAnalysisTraceOut",
CreateHCRuleElement("AffixTemplate", template),
CreateWordElement("Output", unapplied ? output : null, true)
)
var trace = new XElement(
"TemplateAnalysisTraceOut",
CreateHCRuleElement("AffixTemplate", template),
CreateWordElement("Output", unapplied ? output : null, true)
);
if (reason == FailureReason.NonPartialRuleProhibitedAfterFinalTemplate)
{
trace.Add(
new XElement("FailureReason", new XAttribute("type", "nonPartialRuleAfterFinalTemplate"))
);
}
((XElement)output.CurrentTrace).Add(trace);
}

public void MorphologicalRuleUnapplied(
Expand Down
Loading