Add FLEx-compatible per-writing-system collation to FwLite

CONTEXT-MAP.md

@@ -4,4 +4,4 @@ Domain glossaries for this monorepo. Each `CONTEXT.md` is a glossary only — no
 
 ## Contexts
 
-- [FwLite Commenting](./backend/FwLite/CONTEXT.md) — collaborative comments on dictionary entries, senses, and example sentences
+- [FwLite](./backend/FwLite/CONTEXT.md) — dictionary editing concerns including collation and collaborative comments

backend/Directory.Packages.props

@@ -29,7 +29,8 @@
     <PackageVersion Include="HotChocolate.Types.Analyzers" Version="$(HotChocolateVersion)" />
     <PackageVersion Include="HotChocolate.Types.OffsetPagination" Version="$(HotChocolateVersion)" />
     <PackageVersion Include="Humanizer.Core" Version="3.0.10" />
-    <PackageVersion Include="icu.net" Version="3.0.1" />
+    <PackageVersion Include="icu.net" Version="3.0.3-beta.34" />
+    <PackageVersion Include="Icu4c.Android.Fw.Lib" Version="70.1.182" />
     <PackageVersion Include="linq2db.Extensions" Version="6.4.0" />
     <PackageVersion Include="linq2db.EntityFrameworkCore" Version="10.5.0" />
     <PackageVersion Include="L10NSharp" Version="10.0.0-beta0014" />

backend/FwLite/CONTEXT.md

@@ -1,4 +1,46 @@
-# FwLite Commenting
+# FwLite
+
+## Collation
+
+How strings in a writing system are compared when ordering dictionary content. Imported from FLEx per writing system; stored on `WritingSystem` without LDML at runtime.
+
+**Collation**:
+The rules that govern string comparison for a writing system (ICU tailoring or .NET locale alias). Distinct from list position and from the user action of sorting entries.
+_Avoid_: Sort rules (when meaning collation specifically), custom sorting (as a model term)
+
+**Writing system order**:
+Where a writing system appears in the project's vernacular or analysis list (`WritingSystem.Order`). Not how strings alphabetize.
+_Avoid_: Confusing with collation
+
+**Entry sort**:
+The user-facing action of ordering the entry list (e.g. by headword in a chosen writing system). Uses the selected writing system's collation for headword comparison.
+_Avoid_: Collation (when meaning the user action)
+
+**Collation scope (v1)**:
+Collation governs headword entry sort (`SortField.Headword`) and the alphabetical tie-break when search results fall through to headword order (`SortField.SearchRelevance`). It does not govern search matching (whether a query matches text) or FTS ranking.
+_Avoid_: Applying collation to `ContainsDiacriticMatch`, prefix/contains predicates, or FTS rank
+
+**Imported collation**:
+Collation metadata synced from FLEx onto `WritingSystem`. Two optional fields: compiled ICU rules, or a .NET locale alias for "same as another language." No LDML at FwLite runtime.
+_Avoid_: Storing LDML, simple-rule source text, or import metadata
+
+**Legacy collation fallback**:
+When neither field is set, headword comparison uses the pre-existing FwLite behavior (`CultureInfo` from `WsId`, case-insensitive with lowercase-before-uppercase tie-break). FLEx "Default Ordering" is imported as this fallback too — FwLite does not replicate FLEx's empty-rule ICU default until we choose to.
+_Avoid_: Treating `null` and `""` as different states for `IcuCollationRules`
+
+**Collation compare (imported)**:
+When `IcuCollationRules` or `SystemCollationLocale` is set, use icu.net (`RuleBasedCollator` / locale `Collator`) `Compare` with no legacy case tie-break layered on top. Matches FLEx for custom and other-language modes. Android bundles ICU natives via `Icu4c.Android.Fw.Lib`.
+_Avoid_: Applying case-insensitive or lowercase-first logic on top of imported ICU collation
+
+**Collation import (from FLEx)**:
+Populated when mapping `CoreWritingSystemDefinition` → `WritingSystem` in the FwData bridge. Custom modes: store non-empty compiled ICU rules only. Other-language: store .NET locale tag only. Default ordering (including a system collation for the writing system's own language, which liblcm assigns when none is set): leave both fields null (legacy fallback). Import-only — no write-back to fwdata LDML.
+_Avoid_: Parsing LDML in FwLite; persisting empty `IcuCollationRules` for default ordering
+
+**Collation write-back**:
+Explicitly a no-op. The fwdata `UpdateWritingSystemProxy` overrides collation fields and does not write them to LDML.
+_Avoid_: Assuming bidirectional collation sync
+
+## Commenting
 
 Collaborative discussion attached to dictionary data (entries, senses, example sentences). Comments sync via Harmony; read status is local-only per device (not synced, not per Lexbox user account).
 

backend/FwLite/FwDataMiniLcmBridge.Tests/Collation/WritingSystemCollationExtractorTests.cs

@@ -0,0 +1,52 @@
+using FwDataMiniLcmBridge.Collation;
+using FwDataMiniLcmBridge.LcmUtils;
+using SIL.LCModel.Core.WritingSystems;
+using SIL.WritingSystems;
+
+namespace FwDataMiniLcmBridge.Tests.Collation;
+
+public class WritingSystemCollationExtractorTests
+{
+    // Validating a collation needs SLDR and ICU
+    public WritingSystemCollationExtractorTests() => ProjectLoader.Init();
+
+    private static CoreWritingSystemDefinition Ws(string tag, CollationDefinition collation) =>
+        new(tag) { DefaultCollation = collation };
+
+    [Fact]
+    public void OwnLanguageSystemCollation_IsDefaultOrdering()
+    {
+        WritingSystemCollationExtractor.Extract(Ws("es", new SystemCollationDefinition { LanguageTag = "es" }))
+            .Should().Be(((string?)null, (string?)null));
+    }
+
+    [Fact]
+    public void OtherLanguageSystemCollation_StoresLocale()
+    {
+        WritingSystemCollationExtractor.Extract(Ws("es", new SystemCollationDefinition { LanguageTag = "fr" }))
+            .Should().Be(((string?)null, "fr"));
+    }
+
+    [Fact]
+    public void CustomIcuRules_StoresRules()
+    {
+        WritingSystemCollationExtractor.Extract(Ws("es", new IcuRulesCollationDefinition("standard") { IcuRules = "&b < a" }))
+            .IcuCollationRules.Should().Contain("&b < a");
+    }
+
+    [Fact]
+    public void DefaultOrdering_IsNull()
+    {
+        WritingSystemCollationExtractor.Extract(Ws("es", new IcuRulesCollationDefinition("standard")))
+            .Should().Be(((string?)null, (string?)null));
+    }
+
+    [Fact]
+    public void CustomSimpleRules_StoresCompiledRules()
+    {
+        var (rules, locale) = WritingSystemCollationExtractor.Extract(Ws("es", new SimpleRulesCollationDefinition("standard") { SimpleRules = "b B\na A" }));
+        locale.Should().BeNull();
+        rules.Should().NotBeNullOrEmpty();
+        rules!.IndexOf('b').Should().BeLessThan(rules.IndexOf('a'), "simple rules list b before a");
+    }
+}

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/CustomCollationSortingTests.cs

@@ -0,0 +1,33 @@
+using FwDataMiniLcmBridge.Api;
+using FwDataMiniLcmBridge.LcmUtils;
+using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm.Models;
+using SIL.WritingSystems;
+
+namespace FwDataMiniLcmBridge.Tests.MiniLcmTests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class CustomCollationSortingTests(ProjectLoaderFixture fixture) : CustomCollationSortingTestsBase
+{
+    protected override Task<IMiniLcmApi> NewApi()
+    {
+        return Task.FromResult<IMiniLcmApi>(fixture.NewProjectApi("collation-sorting-test", "en", "en"));
+    }
+
+    protected override async Task CreateWritingSystemWithCollation(WritingSystem writingSystem)
+    {
+        await Api.CreateWritingSystem(writingSystem with { IcuCollationRules = null, SystemCollationLocale = null });
+        var fwDataApi = (FwDataMiniLcmApi)BaseApi;
+        await fwDataApi.Cache.DoUsingNewOrCurrentUOW("Set collation",
+            "Revert collation",
+            () =>
+            {
+                // collation is import-only, so it can't be set through the api
+                var ws = fwDataApi.Cache.ServiceLocator.WritingSystemManager.Get(writingSystem.WsId.Code);
+                ws.DefaultCollation = writingSystem.IcuCollationRules is { } rules
+                    ? new IcuRulesCollationDefinition("standard") { IcuRules = rules }
+                    : new SystemCollationDefinition { LanguageTag = writingSystem.SystemCollationLocale };
+                return ValueTask.CompletedTask;
+            });
+    }
+}

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -2,6 +2,7 @@
 using System.Globalization;
 using System.Text;
 using FwDataMiniLcmBridge.Api.UpdateProxy;
+using FwDataMiniLcmBridge.Collation;
 using FwDataMiniLcmBridge.LcmUtils;
 using FwDataMiniLcmBridge.Media;
 using Gridify;
@@ -19,6 +20,7 @@
 using SIL.LCModel.Core.WritingSystems;
 using SIL.LCModel.DomainServices;
 using SIL.LCModel.Infrastructure;
+using SIL.WritingSystems;
 using CollectionExtensions = SIL.Extensions.CollectionExtensions;
 
 namespace FwDataMiniLcmBridge.Api;
@@ -113,6 +115,7 @@ public Task<WritingSystems> GetWritingSystems()
 
     private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, WritingSystemType type, int index = default)
     {
+        var (icuCollationRules, systemCollationLocale) = WritingSystemCollationExtractor.Extract(ws);
         return new WritingSystem
         {
             Id = Guid.Empty,
@@ -125,7 +128,9 @@ private WritingSystem FromLcmWritingSystem(CoreWritingSystemDefinition ws, Writi
             Name = ws.LanguageName,
             Abbreviation = ws.Abbreviation,
             Font = ws.DefaultFontName,
-            Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? []
+            Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? [],
+            IcuCollationRules = icuCollationRules,
+            SystemCollationLocale = systemCollationLocale,
         };
     }
 
@@ -1015,13 +1020,39 @@ private IEnumerable<ILexEntry> GetFilteredAndSortedEntries(Func<ILexEntry, bool>
     private IEnumerable<ILexEntry> ApplySorting(SortOptions order, IEnumerable<ILexEntry> entries, string? query)
     {
         var sortWs = GetWritingSystemHandle(order.WritingSystem, WritingSystemType.Vernacular);
+        var headwordComparer = GetHeadwordComparer(sortWs);
         var stemSecondaryOrder = MorphTypeRepository.GetObject(MoMorphTypeTags.kguidMorphStem).SecondaryOrder;
         if (order.Field == SortField.SearchRelevance)
         {
-            return entries.ApplyRoughBestMatchOrder(order, sortWs, stemSecondaryOrder, query);
+            return entries.ApplyRoughBestMatchOrder(order, sortWs, headwordComparer, stemSecondaryOrder, query);
         }
 
-        return entries.ApplyHeadwordOrder(order, sortWs, stemSecondaryOrder);
+        return entries.ApplyHeadwordOrder(order, sortWs, headwordComparer, stemSecondaryOrder);
+    }
+
+    /// <summary>
+    /// Sorts with the writing system's own collation, like FLEx's RecordSorter, so fwdata and CRDT order match.
+    /// FLEx swaps in a system collation when the rules are empty or invalid; we do the same without modifying the project.
+    /// https://github.com/sillsdev/FieldWorks/blob/32068b620/Src/Common/Filters/RecordSorter.cs#L2213-L2217
+    /// </summary>
+    private IComparer<string?> GetHeadwordComparer(int wsHandle)
+    {
+        var ws = Cache.ServiceLocator.WritingSystemManager.Get(wsHandle);
+        IComparer<string> collator;
+        try
+        {
+            var collation = ws.DefaultCollation;
+            collator = collation is RulesCollationDefinition rules && (!rules.Validate(out _) || string.IsNullOrEmpty(rules.CollationRules))
+                ? new SystemCollationDefinition { LanguageTag = ws.LanguageTag }.Collator
+                : collation.Collator;
+        }
+        catch (Exception e)
+        {
+            logger.LogWarning(e, "Failed to create collator for writing system {WsId}; using culture sort", ws.Id);
+            collator = StringComparer.CurrentCulture;
+        }
+
+        return Comparer<string?>.Create((x, y) => collator.Compare(x ?? string.Empty, y ?? string.Empty));
     }
 
     public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options = null)

backend/FwLite/FwDataMiniLcmBridge/Api/Sorting.cs

@@ -6,20 +6,20 @@ namespace FwDataMiniLcmBridge.Api;
 
 internal static class Sorting
 {
-    public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder)
+    public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, IComparer<string?> headwordComparer, int stemSecondaryOrder)
     {
         if (order.Ascending)
         {
             return entries
-                .OrderBy(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .OrderBy(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false), headwordComparer)
                 .ThenBy(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
                 .ThenBy(e => e.HomographNumber)
                 .ThenBy(e => e.Id.Guid);
         }
         else
         {
             return entries
-                .OrderByDescending(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false))
+                .OrderByDescending(e => e.LexEntryHeadword(sortWsHandle, applyMorphTokens: false), headwordComparer)
                 .ThenByDescending(e => e.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
                 .ThenByDescending(e => e.HomographNumber)
                 .ThenByDescending(e => e.Id.Guid);
@@ -31,7 +31,7 @@ public static IEnumerable<ILexEntry> ApplyHeadwordOrder(this IEnumerable<ILexEnt
     /// prefix matches (e.g. when searching "tan" then "tanan" is before "matan"), then shorter, then alphabetical.
     /// See also: EntrySearchService.FilterAndRank for the FTS-based equivalent in LcmCrdt.
     /// </summary>
-    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, int stemSecondaryOrder, string? query = null)
+    public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<ILexEntry> entries, SortOptions order, int sortWsHandle, IComparer<string?> headwordComparer, int stemSecondaryOrder, string? query = null)
     {
         var projected = entries.Select(e => (
             Entry: e,
@@ -44,7 +44,7 @@ public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<I
                 .OrderByDescending(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.StartsWithDiacriticMatch(query!) ?? false))
                 .ThenByDescending(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenBy(x => x.Headword?.Length ?? 0)
-                .ThenBy(x => x.Headword)
+                .ThenBy(x => x.Headword, headwordComparer)
                 .ThenBy(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
                 .ThenBy(x => x.Entry.HomographNumber)
                 .ThenBy(x => x.Entry.Id.Guid)
@@ -56,7 +56,7 @@ public static IEnumerable<ILexEntry> ApplyRoughBestMatchOrder(this IEnumerable<I
                 .OrderBy(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.StartsWithDiacriticMatch(query!) ?? false))
                 .ThenBy(x => !string.IsNullOrEmpty(query) && (x.HeadwordWithTokens?.ContainsDiacriticMatch(query!) ?? false))
                 .ThenByDescending(x => x.Headword?.Length ?? 0)
-                .ThenByDescending(x => x.Headword)
+                .ThenByDescending(x => x.Headword, headwordComparer)
                 .ThenByDescending(x => x.Entry.PrimaryMorphType?.SecondaryOrder ?? stemSecondaryOrder)
                 .ThenByDescending(x => x.Entry.HomographNumber)
                 .ThenByDescending(x => x.Entry.Id.Guid)

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateWritingSystemProxy.cs

@@ -47,4 +47,17 @@ public override required string Font
             }
         }
     }
+
+    // Collation is import-only from FLEx LDML; MiniLcm updates do not write back to fwdata.
+    public override string? IcuCollationRules
+    {
+        get => null;
+        set { }
+    }
+
+    public override string? SystemCollationLocale
+    {
+        get => null;
+        set { }
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Collation/WritingSystemCollationExtractor.cs

@@ -0,0 +1,30 @@
+using SIL.LCModel.Core.WritingSystems;
+using SIL.WritingSystems;
+
+namespace FwDataMiniLcmBridge.Collation;
+
+public static class WritingSystemCollationExtractor
+{
+    public static (string? IcuCollationRules, string? SystemCollationLocale) Extract(CoreWritingSystemDefinition ws)
+    {
+        var cd = ws.DefaultCollation;
+        cd.Validate(out _);
+
+        return cd switch
+        {
+            // liblcm/libpalaso default a writing system without collation to its own language's system collation
+            SystemCollationDefinition sys when IsOwnLanguage(sys, ws) => (null, null),
+            SystemCollationDefinition sys => (null, sys.LanguageTag),
+            // Custom ICU and Custom Simple both compile to CollationRules
+            RulesCollationDefinition rules when !string.IsNullOrEmpty(rules.CollationRules)
+                => (rules.CollationRules, null),
+            // FLEx "Default Ordering" is an IcuRulesCollationDefinition with empty rules, i.e. ICU root.
+            // We map it to null (FwLite's .NET culture fallback) rather than storing an empty rule set.
+            // https://github.com/sillsdev/libpalaso/blob/32ca8a0a/SIL.Windows.Forms.WritingSystems/WritingSystemSetupModel.cs#L43-L47
+            _ => (null, null)
+        };
+    }
+
+    private static bool IsOwnLanguage(SystemCollationDefinition sys, CoreWritingSystemDefinition ws) =>
+        string.IsNullOrEmpty(sys.LanguageTag) || string.Equals(sys.LanguageTag, ws.LanguageTag, StringComparison.OrdinalIgnoreCase);
+}

backend/FwLite/FwLiteMaui/FwLiteMaui.csproj

@@ -124,6 +124,8 @@
     <PackageReference Include="Microsoft.Toolkit.Uwp.Notifications" />
   </ItemGroup>
   <ItemGroup Condition="$([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">
+    <!-- ICU natives + data for icu.net collation (Android has no public system ICU). -->
+    <PackageReference Include="Icu4c.Android.Fw.Lib" />
     <!-- Android-only Google Play in-app updates (used by Platforms/Android/AndroidInAppUpdateService.cs).
          Conditioned so it is not pulled into the Windows/iOS/Mac bundles, where it is unused. -->
     <PackageReference Include="Xamarin.Google.Android.Play.App.Update" />

backend/FwLite/FwLiteMaui/MauiProgram.cs

@@ -1,4 +1,5 @@
 using FwLiteShared.Services;
+using LcmCrdt.Culture;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Maui.LifecycleEvents;
@@ -127,6 +128,7 @@ await app.ServiceProvider.GetRequiredService<ITroubleshootingService>()
         holder.App = app;
         var logger = app.Services.GetRequiredService<ILogger<MauiApp>>();
         logger.LogInformation("App started, {Version}", AppVersion.Version);
+        IcuInit.TryInitialize(logger);
         AppDomain.CurrentDomain.UnhandledException += (sender, e) =>
         {
             if (e.ExceptionObject is Exception exception)

backend/FwLite/FwLiteProjectSync.Tests/Snapshots/sena-3_snapshot.2026-06-18.verified.txt

@@ -138311,6 +138311,8 @@
         "DeletedAt": null,
         "Type": 1,
         "Exemplars": [],
+        "IcuCollationRules": null,
+        "SystemCollationLocale": null,
         "Order": 1
       },
       {
@@ -138324,6 +138326,8 @@
         "DeletedAt": null,
         "Type": 1,
         "Exemplars": [],
+        "IcuCollationRules": null,
+        "SystemCollationLocale": null,
         "Order": 2
       }
     ],
@@ -138366,6 +138370,8 @@
           "Y",
           "Z"
         ],
+        "IcuCollationRules": null,
+        "SystemCollationLocale": null,
         "Order": 1
       },
       {
@@ -138379,6 +138385,8 @@
         "DeletedAt": null,
         "Type": 0,
         "Exemplars": [],
+        "IcuCollationRules": null,
+        "SystemCollationLocale": null,
         "Order": 2
       }
     ]