From 776692e015a5c0b08256ab61424c3e7a3149eaea Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 07:48:13 +0000 Subject: [PATCH 01/25] Detect change types added or removed since the last app run Changes of an unknown type are stored as OpaqueChange and skipped when snapshots are made. When an app update adds that type, nothing replayed those changes, so the snapshots stayed wrong. - Add a LocalState key/value table that is never synced, with public Get/Set/RemoveLocalState on DataModel. The harmony: key prefix is a convention only. - Add DataModel.ReconcileConfigChanges. It stores the known change types under harmony:config, compares them with the current config, finds the oldest commit with an added or removed type with a SQLite JSON query, and replays from the checkpoint before it. - MissingConfigBehavior controls databases with no stored config: ReplayAll (default) or AssumeCurrentConfig for apps that always used UnknownChangeHandling.Throw. - When a removed type leaves an entity without new snapshots, its projected row is restored from its current snapshot, or deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony.Tests/ConfigReconcileTests.cs | 295 ++++++++++++++++++ src/SIL.Harmony.Tests/DataModelTestBase.cs | 10 +- .../DbContextTests.VerifyModel.verified.txt | 13 + .../Config/ChangeTypeListBuilder.cs | 8 + src/SIL.Harmony/ConfigReconciliation.cs | 55 ++++ src/SIL.Harmony/DataModel.cs | 84 ++++- src/SIL.Harmony/Db/CrdtRepository.cs | 140 ++++++++- .../Db/EntityConfig/LocalStateEntryConfig.cs | 13 + src/SIL.Harmony/Db/FastProjection.cs | 24 +- src/SIL.Harmony/Db/LocalStateEntry.cs | 12 + 10 files changed, 636 insertions(+), 18 deletions(-) create mode 100644 src/SIL.Harmony.Tests/ConfigReconcileTests.cs create mode 100644 src/SIL.Harmony/ConfigReconciliation.cs create mode 100644 src/SIL.Harmony/Db/EntityConfig/LocalStateEntryConfig.cs create mode 100644 src/SIL.Harmony/Db/LocalStateEntry.cs diff --git a/src/SIL.Harmony.Tests/ConfigReconcileTests.cs b/src/SIL.Harmony.Tests/ConfigReconcileTests.cs new file mode 100644 index 0000000..da38586 --- /dev/null +++ b/src/SIL.Harmony.Tests/ConfigReconcileTests.cs @@ -0,0 +1,295 @@ +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Changes; +using SIL.Harmony.Config; +using SIL.Harmony.Db; +using SIL.Harmony.Sample; +using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Tests; + +/// +/// The database starts out written by an "old app" that doesn't know , +/// and is then opened by a "new app" that does, by forking it with a different config. +/// +public class ConfigReconcileTests() : DataModelTestBase(configure: OldApp) +{ + private const string NoteChangeType = nameof(SetWordNoteChange); + + private static void OldApp(IServiceCollection services) + { + services.Configure(config => + { + config.UnknownChangeHandling = UnknownChangeHandling.Fallback; + config.ChangeTypeListBuilder.Remove(); + }); + } + + private static void NewApp(IServiceCollection services) + { + services.Configure(config => config.UnknownChangeHandling = UnknownChangeHandling.Fallback); + } + + private static readonly JsonSerializerOptions NewAppJsonOptions = CreateNewAppJsonOptions(); + + private static JsonSerializerOptions CreateNewAppJsonOptions() + { + using var services = new ServiceCollection().AddCrdtDataSample(":memory:").BuildServiceProvider(); + return services.GetRequiredService(); + } + + /// a note change as the old app sees it, after it was synced from a newer client + private static IChange OpaqueNote(DataModelTestBase oldApp, Guid wordId, string note) + { + var json = JsonSerializer.Serialize(new SetWordNoteChange(wordId, note), NewAppJsonOptions); + var change = JsonSerializer.Deserialize(json, oldApp.CrdtConfig.JsonSerializerOptions); + return change.Should().BeOfType().Subject; + } + + private static async Task SyncOpaqueNote(DataModelTestBase oldApp, Guid wordId, string note) + { + var commit = await oldApp.WriteNextChange(OpaqueNote(oldApp, wordId, note), add: false); + await oldApp.AddCommitsViaSync([commit]); + return commit; + } + + private static async Task NoteOf(DataModelTestBase app, Guid wordId) + { + var word = await app.DataModel.GetLatest(wordId); + word.Should().NotBeNull(); + var projectedWord = await app.DbContext.Set().AsNoTracking() + .SingleAsync(w => w.Id == wordId, TestContext.Current.CancellationToken); + projectedWord.Note.Should().Be(word.Note, "the projected table should match the snapshot"); + return word.Note; + } + + [Fact] + public async Task AddedChangeTypeIsAppliedAfterReconcile() + { + await DataModel.ReconcileConfigChanges(); + var wordId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + var noteCommit = await SyncOpaqueNote(this, wordId, "a note"); + (await NoteOf(this, wordId)).Should().BeNull("the old app can't apply the note change"); + + await using var newApp = ForkDatabase(NewApp); + (await NoteOf(newApp, wordId)).Should().BeNull("nothing replays the note change until the config change is detected"); + + var result = await newApp.DataModel.ReconcileConfigChanges(); + + result.ConfigChanged.Should().BeTrue(); + result.AddedChangeTypes.Should().Equal(NoteChangeType); + result.RemovedChangeTypes.Should().BeEmpty(); + result.ReplayedFrom.Should().NotBeNull(); + result.ReplayedFrom.Id.Should().Be(noteCommit.Id); + (await NoteOf(newApp, wordId)).Should().Be("a note"); + } + + [Fact] + public async Task ReconcileWithTheSameConfigDoesNothing() + { + (await DataModel.ReconcileConfigChanges()).ConfigChanged.Should().BeTrue("no config was stored yet"); + var wordId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + var snapshotIds = await SnapshotIds(this); + + var result = await DataModel.ReconcileConfigChanges(); + + result.Should().Be(ConfigReconcileResult.Unchanged); + (await SnapshotIds(this)).Should().Equal(snapshotIds); + } + + [Fact] + public async Task NewDatabaseStoresTheConfigWithoutReplaying() + { + var result = await DataModel.ReconcileConfigChanges(); + + result.ConfigChanged.Should().BeTrue(); + result.ReplayedFrom.Should().BeNull(); + (await DataModel.GetLocalState("harmony:config")).Should().NotBeNull(); + (await DataModel.ReconcileConfigChanges()).ConfigChanged.Should().BeFalse(); + } + + [Fact] + public async Task ReplayResumesFromTheCheckpointBeforeTheAddedChange() + { + void SmallCheckpoints(IServiceCollection services) => + services.Configure(config => config.MaxChangesBetweenSnapshotCheckpoints = 4); + await using var oldApp = ForkDatabase(services => + { + OldApp(services); + SmallCheckpoints(services); + }); + await oldApp.DataModel.ReconcileConfigChanges(); + var wordId = Guid.NewGuid(); + await oldApp.WriteNextChange(new NewWordChange(wordId, "text 0")); + for (var i = 1; i < 20; i++) + { + await oldApp.WriteNextChange(new SetWordTextChange(wordId, $"text {i}")); + } + var noteCommit = await SyncOpaqueNote(oldApp, wordId, "a note"); + await oldApp.WriteNextChange(new SetWordTextChange(wordId, "last text")); + + await using var newApp = oldApp.ForkDatabase(services => + { + NewApp(services); + SmallCheckpoints(services); + }); + var checkpoint = await newApp.DbContext.Commits.AsNoTracking() + .Where(c => c.IsSnapshotCheckpoint) + .WhereBefore(noteCommit, inclusive: false) + .DefaultOrderDescending() + .FirstAsync(TestContext.Current.CancellationToken); + var commitIdsUpToCheckpoint = newApp.DbContext.Commits.WhereBefore(checkpoint, inclusive: true).Select(c => c.Id); + var snapshotsUpToCheckpoint = await newApp.DbContext.Snapshots.AsNoTracking() + .Where(s => commitIdsUpToCheckpoint.Contains(s.CommitId)) + .Select(s => s.Id) + .ToArrayAsync(TestContext.Current.CancellationToken); + snapshotsUpToCheckpoint.Should().NotBeEmpty(); + + var result = await newApp.DataModel.ReconcileConfigChanges(); + + result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + (await SnapshotIds(newApp)).Should().Contain(snapshotsUpToCheckpoint, "snapshots before the checkpoint should not be rebuilt"); + var word = await newApp.DataModel.GetLatest(wordId); + word!.Note.Should().Be("a note"); + word.Text.Should().Be("last text"); + + var stateAfterReconcile = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); + await newApp.DataModel.RegenerateSnapshots(); + var stateAfterRegenerate = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); + stateAfterReconcile.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted)) + .Should().BeEquivalentTo(stateAfterRegenerate.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted))); + } + + [Fact] + public async Task RemovedChangeTypeIsUnappliedAfterReconcile() + { + await using var newApp = ForkDatabase(NewApp); + await newApp.DataModel.ReconcileConfigChanges(); + var wordId = Guid.NewGuid(); + await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + var noteCommit = await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); + + await using var oldApp = newApp.ForkDatabase(OldApp); + (await NoteOf(oldApp, wordId)).Should().Be("a note", "the snapshots were made by the new app"); + var downgrade = await oldApp.DataModel.ReconcileConfigChanges(); + + downgrade.AddedChangeTypes.Should().BeEmpty(); + downgrade.RemovedChangeTypes.Should().Equal(NoteChangeType); + downgrade.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); + + await using var upgradedAgain = oldApp.ForkDatabase(NewApp); + var upgrade = await upgradedAgain.DataModel.ReconcileConfigChanges(); + + upgrade.AddedChangeTypes.Should().Equal(NoteChangeType); + (await NoteOf(upgradedAgain, wordId)).Should().Be("a note"); + } + + [Fact] + public async Task RemovedCreateChangeTypeRemovesTheEntity() + { + await using var newApp = ForkDatabase(NewApp); + await newApp.DataModel.ReconcileConfigChanges(); + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); + + await using var oldApp = newApp.ForkDatabase(services => + { + NewApp(services); + services.Configure(config => config.ChangeTypeListBuilder.Remove()); + }); + var result = await oldApp.DataModel.ReconcileConfigChanges(); + + result.RemovedChangeTypes.Should().Equal(nameof(SetTagChange)); + (await oldApp.DataModel.GetLatest(tagId)).Should().BeNull(); + (await oldApp.DbContext.Set().AnyAsync(TestContext.Current.CancellationToken)).Should().BeFalse(); + (await oldApp.DataModel.GetLatest(wordId)).Should().NotBeNull(); + } + + [Fact] + public async Task MissingConfigWithReplayAllReplaysAllOfHistory() + { + var wordId = Guid.NewGuid(); + var firstCommit = await WriteNextChange(new NewWordChange(wordId, "hello")); + await SyncOpaqueNote(this, wordId, "a note"); + + await using var newApp = ForkDatabase(NewApp); + var result = await newApp.DataModel.ReconcileConfigChanges(MissingConfigBehavior.ReplayAll); + + result.ConfigChanged.Should().BeTrue(); + result.AddedChangeTypes.Should().BeEmpty("there was no stored config to compare with"); + result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + (await NoteOf(newApp, wordId)).Should().Be("a note"); + } + + [Fact] + public async Task MissingConfigWithAssumeCurrentConfigDoesNotReplay() + { + var wordId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + var snapshotIds = await SnapshotIds(this); + + var result = await DataModel.ReconcileConfigChanges(MissingConfigBehavior.AssumeCurrentConfig); + + result.ConfigChanged.Should().BeTrue(); + result.ReplayedFrom.Should().BeNull(); + (await SnapshotIds(this)).Should().Equal(snapshotIds); + + //the stored config is used from now on + await using var newApp = ForkDatabase(NewApp); + var upgrade = await newApp.DataModel.ReconcileConfigChanges(MissingConfigBehavior.AssumeCurrentConfig); + upgrade.AddedChangeTypes.Should().Equal(NoteChangeType); + } + + [Fact] + public async Task FindOldestCommitWithChangeTypesUsesCommitOrder() + { + await using var newApp = ForkDatabase(NewApp); + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + var laterNote = await newApp.WriteNextChange(new SetWordNoteChange(wordId, "later")); + var tagCommit = await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); + //added last, but sorts before the other note + var earlierNote = await newApp.WriteChangeBefore(laterNote, new SetWordNoteChange(wordId, "earlier")); + await using var repo = newApp.CreateRepository(); + + (await repo.FindOldestCommitWithChangeTypes([NoteChangeType]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeTypes([nameof(SetTagChange)]))!.Id.Should().Be(tagCommit.Id); + (await repo.FindOldestCommitWithChangeTypes([nameof(SetTagChange), NoteChangeType]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeTypes([nameof(EditExampleChange)])).Should().BeNull(); + (await repo.FindOldestCommitWithChangeTypes([])).Should().BeNull(); + } + + [Fact] + public async Task LocalStateCanBeSetReadAndRemoved() + { + (await DataModel.GetLocalState("app:missing")).Should().BeNull(); + + await DataModel.SetLocalState("app:value", new LocalStateTestValue("first", 1)); + (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("first", 1)); + + await DataModel.SetLocalState("app:value", new LocalStateTestValue("second", 2)); + (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("second", 2)); + + await DataModel.RemoveLocalState("app:value"); + (await DataModel.GetLocalState("app:value")).Should().BeNull(); + await DataModel.RemoveLocalState("app:value"); + } + + private record LocalStateTestValue(string Text, int Number); + + private static async Task SnapshotIds(DataModelTestBase app) + { + return await app.DbContext.Snapshots.AsNoTracking() + .Select(s => s.Id) + .OrderBy(id => id) + .ToArrayAsync(TestContext.Current.CancellationToken); + } +} diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index bf9b905..7519320 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -55,13 +55,19 @@ public DataModelTestBase(SqliteConnection connection, bool alwaysValidate = true } public DataModelTestBase ForkDatabase(bool alwaysValidate = true) + { + //the fork has to be configured like its source, otherwise it replays under different settings + return ForkDatabase(_configure, alwaysValidate); + } + + /// a copy of this database opened with a different config, like a newer or older version of the app + public DataModelTestBase ForkDatabase(Action? configure, bool alwaysValidate = true) { var connection = new SqliteConnection("Data Source=:memory:"); connection.Open(); if (DbContext.Database.GetDbConnection() is not SqliteConnection existingConnection) throw new InvalidOperationException("Database is not SQLite"); existingConnection.BackupDatabase(connection); - //the fork has to be configured like its source, otherwise it replays under different settings - var newTestBase = new DataModelTestBase(connection, alwaysValidate, _configure, _performanceTest); + var newTestBase = new DataModelTestBase(connection, alwaysValidate, configure, _performanceTest); newTestBase.SetCurrentDate(currentDate); return newTestBase; } diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 8b57f68..5a92a73 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -51,6 +51,19 @@ Relational:TableName: ChangeEntities Relational:ViewName: Relational:ViewSchema: + EntityType: LocalStateEntry + Properties: + Key (string) Required PK AfterSave:Throw + Value (string) Required + Keys: + Key PK + Annotations: + Relational:FunctionName: + Relational:Schema: + Relational:SqlQuery: + Relational:TableName: LocalState + Relational:ViewName: + Relational:ViewSchema: EntityType: ObjectSnapshot Properties: Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd diff --git a/src/SIL.Harmony/Config/ChangeTypeListBuilder.cs b/src/SIL.Harmony/Config/ChangeTypeListBuilder.cs index 1fd4b1c..dc58992 100644 --- a/src/SIL.Harmony/Config/ChangeTypeListBuilder.cs +++ b/src/SIL.Harmony/Config/ChangeTypeListBuilder.cs @@ -32,4 +32,12 @@ public ChangeTypeListBuilder Add() where TDerived : IChange, IPolyType _types.Add(new RegisteredChangeType(typeof(TDerived), TDerived.TypeName)); return this; } + + /// for tests that simulate a client which doesn't know about a change type yet + internal ChangeTypeListBuilder Remove() where TDerived : IChange, IPolyType + { + CheckFrozen(); + _types.RemoveAll(t => t.Type == typeof(TDerived)); + return this; + } } diff --git a/src/SIL.Harmony/ConfigReconciliation.cs b/src/SIL.Harmony/ConfigReconciliation.cs new file mode 100644 index 0000000..bfdae3e --- /dev/null +++ b/src/SIL.Harmony/ConfigReconciliation.cs @@ -0,0 +1,55 @@ +using SIL.Harmony.Config; + +namespace SIL.Harmony; + +/// +/// What does when the database has no stored config, +/// for example a database created before config tracking existed. +/// +public enum MissingConfigBehavior +{ + /// + /// Replay from the oldest commit with a change type known to the current config, which is usually all of history. + /// Safe for any database, but expensive for big ones. + /// + ReplayAll, + /// + /// Accept the current config as the config the data was written with: store it and don't replay. + /// Only use this when every client that wrote to the database used , + /// or knew the same change types as the current config. Otherwise snapshots may be missing changes. + /// + AssumeCurrentConfig, +} + +/// false when the stored config matched the current one and nothing was done +/// change types the current config knows that the stored config did not +/// change types the stored config knew that the current config does not +/// the oldest commit snapshots were rebuilt from, null when nothing was replayed +public record ConfigReconcileResult( + bool ConfigChanged, + IReadOnlyList AddedChangeTypes, + IReadOnlyList RemovedChangeTypes, + Commit? ReplayedFrom) +{ + internal static readonly ConfigReconcileResult Unchanged = new(false, [], [], null); +} + +/// +/// The parts of the that affect which snapshots are generated, stored in the local state. +/// +internal record StoredHarmonyConfig(int Version, string[] ChangeTypes) +{ + public const string LocalStateKey = "harmony:config"; + public const int CurrentVersion = 1; + + public static StoredHarmonyConfig From(HarmonyConfig config) + { + return new StoredHarmonyConfig(CurrentVersion, + config.ChangeTypes.Select(t => t.Discriminator).Order(StringComparer.Ordinal).ToArray()); + } + + public bool SameAs(StoredHarmonyConfig other) + { + return Version == other.Version && ChangeTypes.SequenceEqual(other.ChangeTypes, StringComparer.Ordinal); + } +} diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 6f1a3b4..faac042 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -186,10 +186,11 @@ ValueTask ISyncable.ShouldSync() /// Rebuilds every snapshot the window covers by replaying its commits onto the state it resumes from. /// A window resuming from nothing rebuilds all of history. /// - private async Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) + /// the snapshots the replay made + private async Task> UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) { var (checkpoint, commitsToApply) = window; - if (commitsToApply.Count == 0) return; + if (commitsToApply.Count == 0) return []; ISnapshotView baseline; // A database with no checkpoints replays all of history, @@ -210,6 +211,7 @@ private async Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWin await PreloadTouched(baseline, commitsToApply); var (newSnapshots, checkpoints) = await SnapshotWorker.ComputeNewSnapshotsAndCheckpoints(baseline, commitsToApply, _crdtConfig.Value); await repo.AddSnapshots(newSnapshots, checkpoints); + return newSnapshots; } private async Task ValidateCommits(CrdtRepository repo) @@ -245,6 +247,84 @@ public async Task RegenerateSnapshots() await transaction.CommitAsync(); } + /// + /// Detects change types that were added to or removed from the config since the last call, for example after an app update, + /// and rebuilds the snapshots from the oldest commit that uses one of them. + /// Changes of an unknown type are stored as and skipped, so once the type is known + /// they must be replayed. A removed type is the reverse: its changes are applied in the snapshots, but are now opaque. + /// Call this when opening a database, before using it. Querying changes by type is SQLite only. + /// + /// what to do when no config was stored yet, see + public async Task ReconcileConfigChanges( + MissingConfigBehavior missingConfig = MissingConfigBehavior.ReplayAll) + { + await using var repo = await _crdtRepositoryFactory.CreateRepository(); + using var locked = await repo.Lock(); + repo.ClearChangeTracker(); + await using var transaction = await repo.BeginTransactionAsync(); + var currentConfig = StoredHarmonyConfig.From(_crdtConfig.Value); + var storedConfig = await repo.GetLocalState(StoredHarmonyConfig.LocalStateKey); + if (storedConfig is not null && storedConfig.SameAs(currentConfig)) return ConfigReconcileResult.Unchanged; + + string[] added = []; + string[] removed = []; + string[] affected; + if (storedConfig is null) + { + affected = missingConfig == MissingConfigBehavior.ReplayAll ? currentConfig.ChangeTypes : []; + } + else + { + added = currentConfig.ChangeTypes.Except(storedConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); + removed = storedConfig.ChangeTypes.Except(currentConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); + affected = [..added, ..removed]; + } + + var replayFrom = await repo.FindOldestCommitWithChangeTypes(affected); + if (replayFrom is not null) + { + _logger.LogInformation("Change types changed (added: {Added}, removed: {Removed}), replaying from commit {CommitId}", + added, removed, replayFrom.Id); + var window = await repo.ReplayWindowFrom(replayFrom); + var replacedSnapshots = await repo.SnapshotsAfter(window.ResumeFrom); + var newSnapshots = await UpdateSnapshots(repo, window); + //a removed change type can leave entities with fewer snapshots than before, which the replay doesn't project + await repo.ReprojectEntitiesWithoutNewSnapshots(replacedSnapshots, newSnapshots); + } + + await repo.SetLocalState(StoredHarmonyConfig.LocalStateKey, currentConfig); + await transaction.CommitAsync(); + return new ConfigReconcileResult(true, added, removed, replayFrom); + } + + /// + /// Gets a value from the local state, which is stored in this database and never synced. + /// Keys starting with harmony: are used by Harmony itself, apps should use their own prefix. + /// + /// the value, or default when the key doesn't exist + public async Task GetLocalState(string key) + { + return await _crdtRepositoryFactory.Execute(repo => repo.GetLocalState(key)); + } + + /// + /// Adds or replaces a value in the local state, the value is stored as json. + /// + /// + public async Task SetLocalState(string key, T value) + { + await _crdtRepositoryFactory.Execute(repo => repo.SetLocalState(key, value)); + } + + /// + /// Removes a value from the local state, does nothing when the key doesn't exist. + /// + /// + public async Task RemoveLocalState(string key) + { + await _crdtRepositoryFactory.Execute(repo => repo.RemoveLocalState(key)); + } + public async Task GetLatestSnapshotByObjectId(Guid entityId) { await using var repo = await _crdtRepositoryFactory.CreateRepository(); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 5ea28f6..4facb19 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Collections.Frozen; using System.Reflection; +using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; @@ -356,20 +357,22 @@ private bool ShouldNotifyProjectedChanges() HarmonyConfig.DefaultOnProjectedEntitiesChanged); } - private async ValueTask NotifyProjectedChanges(IReadOnlyCollection latest) + private ValueTask NotifyProjectedChanges(IReadOnlyCollection latest) { - var changes = new List(latest.Count); + return NotifyProjectedChanges(latest, removed: []); + } + + private async ValueTask NotifyProjectedChanges(IReadOnlyCollection latest, IReadOnlyCollection removed) + { + var changes = new List(latest.Count + removed.Count); foreach (var snapshot in latest) { - var entity = snapshot.Entity.DbObject; - changes.Add(new ProjectedEntityChange - { - Entity = entity, - EntityId = snapshot.EntityId, - ClrType = entity.GetType(), - Kind = snapshot.EntityIsDeleted ? ProjectedChangeKind.Delete : ProjectedChangeKind.Upsert, - Snapshot = snapshot - }); + changes.Add(ToProjectedChange(snapshot, + snapshot.EntityIsDeleted ? ProjectedChangeKind.Delete : ProjectedChangeKind.Upsert)); + } + foreach (var snapshot in removed) + { + changes.Add(ToProjectedChange(snapshot, ProjectedChangeKind.Delete)); } var batch = new ProjectedEntityBatch { DbContext = _dbContext, Changes = changes }; @@ -378,12 +381,102 @@ private async ValueTask NotifyProjectedChanges(IReadOnlyCollection + /// The snapshots a replay resuming from deletes, so the caller can check which entities + /// the replay didn't make new snapshots for, see . + /// + public async Task SnapshotsAfter(CheckpointCommit? checkpoint) + { + var snapshots = Snapshots; + if (checkpoint is not null) + { + var commitIdsAfter = Commits.WhereAfter(checkpoint.Commit).Select(c => c.Id); + snapshots = snapshots.Where(s => commitIdsAfter.Contains(s.CommitId)); + } + return await snapshots.ToArrayAsync(); + } + + /// + /// A replay only projects the snapshots it makes. Normally a replay makes snapshots for the same entities it deleted + /// snapshots of, but not when a change type is no longer known. The projected rows of those entities go back to their + /// current snapshot, or are deleted when the entity has no snapshot left. + /// + /// from , before the replay + /// the snapshots the replay made + public async Task ReprojectEntitiesWithoutNewSnapshots(IReadOnlyCollection replacedSnapshots, + IReadOnlyCollection newSnapshots) + { + if (!_crdtConfig.Value.EnableProjectedTables) return; + var entitiesWithNewSnapshots = newSnapshots.Select(s => s.EntityId).ToHashSet(); + var currentView = CurrentSnapshotView(); + var current = new List(); + var removed = new List(); + foreach (var replaced in replacedSnapshots.DistinctBy(s => s.EntityId)) + { + if (entitiesWithNewSnapshots.Contains(replaced.EntityId)) continue; + if (await currentView.GetAsync(replaced.EntityId) is { } snapshot) current.Add(snapshot); + else removed.Add(replaced); + } + + await _fastProjection.ReprojectAsync(_dbContext, current, removed); + if (ShouldNotifyProjectedChanges() && current.Count + removed.Count > 0) + await NotifyProjectedChanges(current, removed); + } + /// The window that rebuilds every snapshot: all of history, resuming from nothing. public async Task WholeHistory() { return new ReplayWindow(null, await GetCommitsAfter(null)); } + /// The window that rebuilds every snapshot from onwards. + public async Task ReplayWindowFrom(Commit commit) + { + var resumeFrom = await FindCheckpointBefore(commit); + return new ReplayWindow(resumeFrom, await GetCommitsAfter(resumeFrom)); + } + + /// + /// The oldest commit with a change whose $type is one of . + /// Reads every change, so it's slow, only use it for rare operations. SQLite only. + /// + public async Task FindOldestCommitWithChangeTypes(IReadOnlyCollection changeTypes) + { + if (changeTypes.Count == 0) return null; + if (!_dbContext.Database.IsSqlite()) + throw new NotSupportedException( + $"Querying changes by type is only supported on SQLite, not {_dbContext.Database.ProviderName}"); + //the key must be quoted in the path, because $ has a special meaning in json paths + var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; + //one json array parameter, so the parameter count doesn't depend on how many types there are + var changeTypesJson = JsonSerializer.Serialize(changeTypes); + var commitIds = await _dbContext.Database.SqlQuery($""" + SELECT c.Id AS Value FROM Commits c + WHERE EXISTS ( + SELECT 1 FROM ChangeEntities ce + WHERE ce.CommitId = c.Id + AND json_extract(ce.Change, {typeDiscriminatorPath}) IN (SELECT value FROM json_each({changeTypesJson})) + ) + ORDER BY c.DateTime, c.Counter, c.Id + LIMIT 1 + """).ToListAsync(); + if (commitIds is not [var commitId]) return null; + return await Commits.SingleAsync(c => c.Id == commitId); + } + /// public Task AddCommit(Commit commit) => AddCommits([commit]); @@ -428,6 +521,31 @@ private void UpdateCommitHashes(SortedSet commits, Commit? parentCommit) } + private DbSet LocalState => _dbContext.Set(); + + public async Task GetLocalState(string key) + { + var value = await LocalState.AsNoTracking() + .Where(e => e.Key == key) + .Select(e => e.Value) + .SingleOrDefaultAsync(); + return value is null ? default : JsonSerializer.Deserialize(value); + } + + public async Task SetLocalState(string key, T value) + { + var json = JsonSerializer.Serialize(value); + var entry = await LocalState.SingleOrDefaultAsync(e => e.Key == key); + if (entry is null) LocalState.Add(new LocalStateEntry { Key = key, Value = json }); + else entry.Value = json; + await _dbContext.SaveChangesAsync(); + } + + public async Task RemoveLocalState(string key) + { + await LocalState.Where(e => e.Key == key).ExecuteDeleteAsync(); + } + public async Task AddLocalResource(LocalResource localResource) { _dbContext.Set().Add(localResource); diff --git a/src/SIL.Harmony/Db/EntityConfig/LocalStateEntryConfig.cs b/src/SIL.Harmony/Db/EntityConfig/LocalStateEntryConfig.cs new file mode 100644 index 0000000..3577af5 --- /dev/null +++ b/src/SIL.Harmony/Db/EntityConfig/LocalStateEntryConfig.cs @@ -0,0 +1,13 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace SIL.Harmony.Db.EntityConfig; + +public class LocalStateEntryConfig : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("LocalState"); + builder.HasKey(e => e.Key); + } +} diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs index f8ca36b..76c63eb 100644 --- a/src/SIL.Harmony/Db/FastProjection.cs +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -90,10 +90,28 @@ internal static void EnsureSupportedProvider(string? providerName) $"Set {nameof(HarmonyConfig)}.{nameof(HarmonyConfig.EnableProjectedTables)} to false to disable projected tables."); } + /// + /// Writes the projected rows of snapshots that are already persisted, without inserting the snapshots. + /// + /// the current snapshot of each entity to project + /// a snapshot of each entity that no longer has any snapshot, so its projected row is deleted + public async Task ReprojectAsync( + ICrdtDbContext dbContext, + IReadOnlyCollection current, + IReadOnlyCollection removed) + { + if (!_crdtConfig.EnableProjectedTables || current.Count + removed.Count == 0) return; + var removedIds = removed.Select(s => s.Id).ToHashSet(); + var latest = current.Concat(removed).ToDictionary(s => s.EntityId); + await ProjectAsync(dbContext, latest, s => s.EntityIsDeleted || removedIds.Contains(s.Id)); + } + private async Task ProjectAsync( ICrdtDbContext dbContext, - Dictionary latest) + Dictionary latest, + Func? shouldDelete = null) { + shouldDelete ??= static s => s.EntityIsDeleted; EnsureSupportedProvider(dbContext.Database.ProviderName); var connection = dbContext.Database.GetDbConnection(); var transaction = dbContext.Database.CurrentTransaction!.GetDbTransaction(); @@ -109,7 +127,7 @@ private async Task ProjectAsync( // same batch; children before parents (reverse dependency order) to satisfy FK constraints. for (var i = orderedTypes.Count - 1; i >= 0; i--) { - var deleted = byType[orderedTypes[i]].Where(s => s.EntityIsDeleted).ToList(); + var deleted = byType[orderedTypes[i]].Where(shouldDelete).ToList(); if (deleted.Count == 0) continue; var info = GetTableInfo(dbContext, orderedTypes[i], sqlHelper); await DeletePerQueryAsync(connection, transaction, info, deleted); @@ -118,7 +136,7 @@ private async Task ProjectAsync( // upserts: parents first foreach (var type in orderedTypes) { - var live = byType[type].Where(s => !s.EntityIsDeleted).ToList(); + var live = byType[type].Where(s => !shouldDelete(s)).ToList(); if (live.Count == 0) continue; var info = GetTableInfo(dbContext, type, sqlHelper); // within a single type's batch, order rows so a row referenced by another row's diff --git a/src/SIL.Harmony/Db/LocalStateEntry.cs b/src/SIL.Harmony/Db/LocalStateEntry.cs new file mode 100644 index 0000000..0c1d856 --- /dev/null +++ b/src/SIL.Harmony/Db/LocalStateEntry.cs @@ -0,0 +1,12 @@ +namespace SIL.Harmony.Db; + +/// +/// a non CRDT key value pair that is local to this database and is never synced. +/// Keys starting with harmony: are used by Harmony itself, apps should use their own prefix. +/// +public class LocalStateEntry +{ + public required string Key { get; set; } + /// json encoded value + public required string Value { get; set; } +} From abd42ccc8053d65b1bf2595a447b68393c41a87b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 08:32:38 +0000 Subject: [PATCH 02/25] Regenerate all snapshots when a change type is removed A removed change type is rare, so use the existing full regeneration instead of a dedicated step that projects the rows of entities the replay left without new snapshots. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony.Tests/ConfigReconcileTests.cs | 6 +- src/SIL.Harmony/DataModel.cs | 32 +++++--- src/SIL.Harmony/Db/CrdtRepository.cs | 79 +++---------------- src/SIL.Harmony/Db/FastProjection.cs | 24 +----- 4 files changed, 38 insertions(+), 103 deletions(-) diff --git a/src/SIL.Harmony.Tests/ConfigReconcileTests.cs b/src/SIL.Harmony.Tests/ConfigReconcileTests.cs index da38586..eb19ae8 100644 --- a/src/SIL.Harmony.Tests/ConfigReconcileTests.cs +++ b/src/SIL.Harmony.Tests/ConfigReconcileTests.cs @@ -170,8 +170,8 @@ public async Task RemovedChangeTypeIsUnappliedAfterReconcile() await using var newApp = ForkDatabase(NewApp); await newApp.DataModel.ReconcileConfigChanges(); var wordId = Guid.NewGuid(); - await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); - var noteCommit = await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); + var firstCommit = await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); await using var oldApp = newApp.ForkDatabase(OldApp); (await NoteOf(oldApp, wordId)).Should().Be("a note", "the snapshots were made by the new app"); @@ -179,7 +179,7 @@ public async Task RemovedChangeTypeIsUnappliedAfterReconcile() downgrade.AddedChangeTypes.Should().BeEmpty(); downgrade.RemovedChangeTypes.Should().Equal(NoteChangeType); - downgrade.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + downgrade.ReplayedFrom!.Id.Should().Be(firstCommit.Id, "a removed type regenerates all snapshots"); (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); await using var upgradedAgain = oldApp.ForkDatabase(NewApp); diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index faac042..8c2ae13 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -186,11 +186,10 @@ ValueTask ISyncable.ShouldSync() /// Rebuilds every snapshot the window covers by replaying its commits onto the state it resumes from. /// A window resuming from nothing rebuilds all of history. /// - /// the snapshots the replay made - private async Task> UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) + private async Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) { var (checkpoint, commitsToApply) = window; - if (commitsToApply.Count == 0) return []; + if (commitsToApply.Count == 0) return; ISnapshotView baseline; // A database with no checkpoints replays all of history, @@ -211,7 +210,6 @@ private async Task> UpdateSnapshots(CrdtRepository await PreloadTouched(baseline, commitsToApply); var (newSnapshots, checkpoints) = await SnapshotWorker.ComputeNewSnapshotsAndCheckpoints(baseline, commitsToApply, _crdtConfig.Value); await repo.AddSnapshots(newSnapshots, checkpoints); - return newSnapshots; } private async Task ValidateCommits(CrdtRepository repo) @@ -240,18 +238,24 @@ public async Task RegenerateSnapshots() using var locked = await repo.Lock(); repo.ClearChangeTracker(); await using var transaction = await repo.BeginTransactionAsync(); + await RegenerateSnapshots(repo); + await transaction.CommitAsync(); + } + + private async Task RegenerateSnapshots(CrdtRepository repo) + { var wholeHistory = await repo.WholeHistory(); //Replay does nothing without commits, which would leave snapshots with no history behind them in place if (wholeHistory.Commits.Count == 0) await repo.DeleteSnapshotsAndProjectedTables(); else await UpdateSnapshots(repo, wholeHistory); - await transaction.CommitAsync(); } /// /// Detects change types that were added to or removed from the config since the last call, for example after an app update, /// and rebuilds the snapshots from the oldest commit that uses one of them. /// Changes of an unknown type are stored as and skipped, so once the type is known - /// they must be replayed. A removed type is the reverse: its changes are applied in the snapshots, but are now opaque. + /// they must be replayed. A removed type is the reverse: its changes are applied in the snapshots, but are now opaque, + /// so all snapshots are regenerated. /// Call this when opening a database, before using it. Querying changes by type is SQLite only. /// /// what to do when no config was stored yet, see @@ -285,11 +289,17 @@ public async Task ReconcileConfigChanges( { _logger.LogInformation("Change types changed (added: {Added}, removed: {Removed}), replaying from commit {CommitId}", added, removed, replayFrom.Id); - var window = await repo.ReplayWindowFrom(replayFrom); - var replacedSnapshots = await repo.SnapshotsAfter(window.ResumeFrom); - var newSnapshots = await UpdateSnapshots(repo, window); - //a removed change type can leave entities with fewer snapshots than before, which the replay doesn't project - await repo.ReprojectEntitiesWithoutNewSnapshots(replacedSnapshots, newSnapshots); + if (removed.Length > 0) + { + //rare, so keep it simple: a replay from a checkpoint only projects the snapshots it makes, but a removed + //type can leave entities with fewer snapshots than before, so their projected rows would be stale + await RegenerateSnapshots(repo); + replayFrom = await repo.CurrentCommits().FirstAsync(); + } + else + { + await UpdateSnapshots(repo, await repo.ReplayWindowFrom(replayFrom)); + } } await repo.SetLocalState(StoredHarmonyConfig.LocalStateKey, currentConfig); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 4facb19..b8b97cb 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -357,22 +357,20 @@ private bool ShouldNotifyProjectedChanges() HarmonyConfig.DefaultOnProjectedEntitiesChanged); } - private ValueTask NotifyProjectedChanges(IReadOnlyCollection latest) + private async ValueTask NotifyProjectedChanges(IReadOnlyCollection latest) { - return NotifyProjectedChanges(latest, removed: []); - } - - private async ValueTask NotifyProjectedChanges(IReadOnlyCollection latest, IReadOnlyCollection removed) - { - var changes = new List(latest.Count + removed.Count); + var changes = new List(latest.Count); foreach (var snapshot in latest) { - changes.Add(ToProjectedChange(snapshot, - snapshot.EntityIsDeleted ? ProjectedChangeKind.Delete : ProjectedChangeKind.Upsert)); - } - foreach (var snapshot in removed) - { - changes.Add(ToProjectedChange(snapshot, ProjectedChangeKind.Delete)); + var entity = snapshot.Entity.DbObject; + changes.Add(new ProjectedEntityChange + { + Entity = entity, + EntityId = snapshot.EntityId, + ClrType = entity.GetType(), + Kind = snapshot.EntityIsDeleted ? ProjectedChangeKind.Delete : ProjectedChangeKind.Upsert, + Snapshot = snapshot + }); } var batch = new ProjectedEntityBatch { DbContext = _dbContext, Changes = changes }; @@ -381,61 +379,6 @@ private async ValueTask NotifyProjectedChanges(IReadOnlyCollection - /// The snapshots a replay resuming from deletes, so the caller can check which entities - /// the replay didn't make new snapshots for, see . - /// - public async Task SnapshotsAfter(CheckpointCommit? checkpoint) - { - var snapshots = Snapshots; - if (checkpoint is not null) - { - var commitIdsAfter = Commits.WhereAfter(checkpoint.Commit).Select(c => c.Id); - snapshots = snapshots.Where(s => commitIdsAfter.Contains(s.CommitId)); - } - return await snapshots.ToArrayAsync(); - } - - /// - /// A replay only projects the snapshots it makes. Normally a replay makes snapshots for the same entities it deleted - /// snapshots of, but not when a change type is no longer known. The projected rows of those entities go back to their - /// current snapshot, or are deleted when the entity has no snapshot left. - /// - /// from , before the replay - /// the snapshots the replay made - public async Task ReprojectEntitiesWithoutNewSnapshots(IReadOnlyCollection replacedSnapshots, - IReadOnlyCollection newSnapshots) - { - if (!_crdtConfig.Value.EnableProjectedTables) return; - var entitiesWithNewSnapshots = newSnapshots.Select(s => s.EntityId).ToHashSet(); - var currentView = CurrentSnapshotView(); - var current = new List(); - var removed = new List(); - foreach (var replaced in replacedSnapshots.DistinctBy(s => s.EntityId)) - { - if (entitiesWithNewSnapshots.Contains(replaced.EntityId)) continue; - if (await currentView.GetAsync(replaced.EntityId) is { } snapshot) current.Add(snapshot); - else removed.Add(replaced); - } - - await _fastProjection.ReprojectAsync(_dbContext, current, removed); - if (ShouldNotifyProjectedChanges() && current.Count + removed.Count > 0) - await NotifyProjectedChanges(current, removed); - } - /// The window that rebuilds every snapshot: all of history, resuming from nothing. public async Task WholeHistory() { diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs index 76c63eb..f8ca36b 100644 --- a/src/SIL.Harmony/Db/FastProjection.cs +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -90,28 +90,10 @@ internal static void EnsureSupportedProvider(string? providerName) $"Set {nameof(HarmonyConfig)}.{nameof(HarmonyConfig.EnableProjectedTables)} to false to disable projected tables."); } - /// - /// Writes the projected rows of snapshots that are already persisted, without inserting the snapshots. - /// - /// the current snapshot of each entity to project - /// a snapshot of each entity that no longer has any snapshot, so its projected row is deleted - public async Task ReprojectAsync( - ICrdtDbContext dbContext, - IReadOnlyCollection current, - IReadOnlyCollection removed) - { - if (!_crdtConfig.EnableProjectedTables || current.Count + removed.Count == 0) return; - var removedIds = removed.Select(s => s.Id).ToHashSet(); - var latest = current.Concat(removed).ToDictionary(s => s.EntityId); - await ProjectAsync(dbContext, latest, s => s.EntityIsDeleted || removedIds.Contains(s.Id)); - } - private async Task ProjectAsync( ICrdtDbContext dbContext, - Dictionary latest, - Func? shouldDelete = null) + Dictionary latest) { - shouldDelete ??= static s => s.EntityIsDeleted; EnsureSupportedProvider(dbContext.Database.ProviderName); var connection = dbContext.Database.GetDbConnection(); var transaction = dbContext.Database.CurrentTransaction!.GetDbTransaction(); @@ -127,7 +109,7 @@ private async Task ProjectAsync( // same batch; children before parents (reverse dependency order) to satisfy FK constraints. for (var i = orderedTypes.Count - 1; i >= 0; i--) { - var deleted = byType[orderedTypes[i]].Where(shouldDelete).ToList(); + var deleted = byType[orderedTypes[i]].Where(s => s.EntityIsDeleted).ToList(); if (deleted.Count == 0) continue; var info = GetTableInfo(dbContext, orderedTypes[i], sqlHelper); await DeletePerQueryAsync(connection, transaction, info, deleted); @@ -136,7 +118,7 @@ private async Task ProjectAsync( // upserts: parents first foreach (var type in orderedTypes) { - var live = byType[type].Where(s => !shouldDelete(s)).ToList(); + var live = byType[type].Where(s => !s.EntityIsDeleted).ToList(); if (live.Count == 0) continue; var info = GetTableInfo(dbContext, type, sqlHelper); // within a single type's batch, order rows so a row referenced by another row's From a30c1f7d76f82379f9f6d160ff8cb4cd2ef84e23 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:05:35 +0000 Subject: [PATCH 03/25] Fix spread formatting for dotnet format Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony/DataModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 8c2ae13..1066b68 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -281,7 +281,7 @@ public async Task ReconcileConfigChanges( { added = currentConfig.ChangeTypes.Except(storedConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); removed = storedConfig.ChangeTypes.Except(currentConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); - affected = [..added, ..removed]; + affected = [.. added, .. removed]; } var replayFrom = await repo.FindOldestCommitWithChangeTypes(affected); From 6ecce8fb0ce7739b67bfd376e28e2f7cbb7567c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 10:00:29 +0000 Subject: [PATCH 04/25] Replace config tracking with model versions Apps can change how a change type applies, not only add change types. A change authored by a newer app and applied by an older app must be replayed when the older app upgrades. - Apps declare an ordered list of model versions with HarmonyConfig.ModelVersionBuilder. Each version lists the change types it added or modified, and can be major. - ChangeEntity.Version is set when a change is authored: the number of model versions that list its type. It syncs with the commit and is stored in a new column. - ReconcileModelVersions (on DataModel, implemented by the internal ModelVersionService) stores the applied versions in the local state. On upgrade it replays from the oldest change authored with a version the database did not have yet. A major version or a downgrade regenerates all snapshots. A modified applied version throws. - DescribeModelVersions gives a stable text form for app snapshot tests. - Snapshot replay helpers move from DataModel to SnapshotReplay so the service can use them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- README.md | 24 + .../BenchmarkWorkloadBuilders.cs | 9 +- src/SIL.Harmony.Core/ChangeEntity.cs | 5 + src/SIL.Harmony.Tests/ConfigReconcileTests.cs | 295 ------------ src/SIL.Harmony.Tests/DataModelTestBase.cs | 2 + .../DbContextTests.VerifyModel.verified.txt | 3 + ...onTests.DescribeModelVersions.verified.txt | 3 + src/SIL.Harmony.Tests/ModelVersionTests.cs | 437 ++++++++++++++++++ src/SIL.Harmony/Config/HarmonyConfig.cs | 56 +++ src/SIL.Harmony/Config/ModelVersion.cs | 22 + src/SIL.Harmony/Config/ModelVersionBuilder.cs | 86 ++++ src/SIL.Harmony/ConfigReconciliation.cs | 55 --- src/SIL.Harmony/CrdtKernel.cs | 13 +- src/SIL.Harmony/DataModel.cs | 116 +---- src/SIL.Harmony/Db/CrdtRepository.cs | 15 +- .../Db/EntityConfig/ChangeEntityConfig.cs | 1 + src/SIL.Harmony/ModelVersionReconciliation.cs | 40 ++ src/SIL.Harmony/ModelVersionService.cs | 95 ++++ src/SIL.Harmony/SnapshotReplay.cs | 55 +++ 19 files changed, 872 insertions(+), 460 deletions(-) delete mode 100644 src/SIL.Harmony.Tests/ConfigReconcileTests.cs create mode 100644 src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt create mode 100644 src/SIL.Harmony.Tests/ModelVersionTests.cs create mode 100644 src/SIL.Harmony/Config/ModelVersion.cs create mode 100644 src/SIL.Harmony/Config/ModelVersionBuilder.cs delete mode 100644 src/SIL.Harmony/ConfigReconciliation.cs create mode 100644 src/SIL.Harmony/ModelVersionReconciliation.cs create mode 100644 src/SIL.Harmony/ModelVersionService.cs create mode 100644 src/SIL.Harmony/SnapshotReplay.cs diff --git a/README.md b/README.md index cce1892..932f0a0 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,30 @@ await dataModel.SyncWith(remoteModel); ``` It's that easy. All the heavy lifting is done by the interface which is fairly simple to implement. +### Model versions +Clients on different app versions sync with each other. When a newer app adds a change type, an older app stores those changes as `OpaqueChange` (with `UnknownChangeHandling.Fallback`) and skips them. When a newer app changes how a change type applies, an older app applies those changes with its old code. In both cases the snapshots must be rebuilt when the older app upgrades. + +To make this possible, declare a model version each time a release adds a change type, or changes how one applies. Only add versions at the end, never modify a released version: +```C# +services.AddCrdtData(config => +{ + config.ModelVersionBuilder + .Add("Word notes", v => v.Change()) + //a major version regenerates all snapshots on upgrade + .AddMajor("Reference rewrite", v => v.Change()); +}); +``` +Every change is stored with the version of its type when it was authored: how many model versions list that type. Changes from before model versions existed are version 0. + +Call `ReconcileModelVersions` when you open a database, before you use it. It compares the model versions stored in the database with the current ones, and replays only the changes that were authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. +```C# +await dataModel.ReconcileModelVersions(); +``` +To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. + +> [!NOTE] +> Model versions add a `Version` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. + ## Development ### SemVer commit messages diff --git a/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs b/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs index 15a4475..6b890d0 100644 --- a/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs +++ b/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs @@ -20,7 +20,14 @@ public static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IC }; for (var i = 0; i < changes.Length; i++) { - commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id)); + //the sample declares no model versions, so every change is version 0 + commit.ChangeEntities.Add(new ChangeEntity + { + Change = changes[i], + CommitId = commit.Id, + EntityId = changes[i].EntityId, + Index = i + }); } return commit; } diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index 4668988..c33199a 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -13,4 +13,9 @@ public ChangeEntity() public required Guid CommitId { get; set; } public required Guid EntityId { get; set; } public required TChange Change { get; set; } + /// + /// The version of the change type when the change was authored: how many model versions list its type. + /// Changes from before model versions existed are version 0. + /// + public int Version { get; set; } } diff --git a/src/SIL.Harmony.Tests/ConfigReconcileTests.cs b/src/SIL.Harmony.Tests/ConfigReconcileTests.cs deleted file mode 100644 index eb19ae8..0000000 --- a/src/SIL.Harmony.Tests/ConfigReconcileTests.cs +++ /dev/null @@ -1,295 +0,0 @@ -using System.Text.Json; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using SIL.Harmony.Changes; -using SIL.Harmony.Config; -using SIL.Harmony.Db; -using SIL.Harmony.Sample; -using SIL.Harmony.Sample.Changes; -using SIL.Harmony.Sample.Models; - -namespace SIL.Harmony.Tests; - -/// -/// The database starts out written by an "old app" that doesn't know , -/// and is then opened by a "new app" that does, by forking it with a different config. -/// -public class ConfigReconcileTests() : DataModelTestBase(configure: OldApp) -{ - private const string NoteChangeType = nameof(SetWordNoteChange); - - private static void OldApp(IServiceCollection services) - { - services.Configure(config => - { - config.UnknownChangeHandling = UnknownChangeHandling.Fallback; - config.ChangeTypeListBuilder.Remove(); - }); - } - - private static void NewApp(IServiceCollection services) - { - services.Configure(config => config.UnknownChangeHandling = UnknownChangeHandling.Fallback); - } - - private static readonly JsonSerializerOptions NewAppJsonOptions = CreateNewAppJsonOptions(); - - private static JsonSerializerOptions CreateNewAppJsonOptions() - { - using var services = new ServiceCollection().AddCrdtDataSample(":memory:").BuildServiceProvider(); - return services.GetRequiredService(); - } - - /// a note change as the old app sees it, after it was synced from a newer client - private static IChange OpaqueNote(DataModelTestBase oldApp, Guid wordId, string note) - { - var json = JsonSerializer.Serialize(new SetWordNoteChange(wordId, note), NewAppJsonOptions); - var change = JsonSerializer.Deserialize(json, oldApp.CrdtConfig.JsonSerializerOptions); - return change.Should().BeOfType().Subject; - } - - private static async Task SyncOpaqueNote(DataModelTestBase oldApp, Guid wordId, string note) - { - var commit = await oldApp.WriteNextChange(OpaqueNote(oldApp, wordId, note), add: false); - await oldApp.AddCommitsViaSync([commit]); - return commit; - } - - private static async Task NoteOf(DataModelTestBase app, Guid wordId) - { - var word = await app.DataModel.GetLatest(wordId); - word.Should().NotBeNull(); - var projectedWord = await app.DbContext.Set().AsNoTracking() - .SingleAsync(w => w.Id == wordId, TestContext.Current.CancellationToken); - projectedWord.Note.Should().Be(word.Note, "the projected table should match the snapshot"); - return word.Note; - } - - [Fact] - public async Task AddedChangeTypeIsAppliedAfterReconcile() - { - await DataModel.ReconcileConfigChanges(); - var wordId = Guid.NewGuid(); - await WriteNextChange(new NewWordChange(wordId, "hello")); - var noteCommit = await SyncOpaqueNote(this, wordId, "a note"); - (await NoteOf(this, wordId)).Should().BeNull("the old app can't apply the note change"); - - await using var newApp = ForkDatabase(NewApp); - (await NoteOf(newApp, wordId)).Should().BeNull("nothing replays the note change until the config change is detected"); - - var result = await newApp.DataModel.ReconcileConfigChanges(); - - result.ConfigChanged.Should().BeTrue(); - result.AddedChangeTypes.Should().Equal(NoteChangeType); - result.RemovedChangeTypes.Should().BeEmpty(); - result.ReplayedFrom.Should().NotBeNull(); - result.ReplayedFrom.Id.Should().Be(noteCommit.Id); - (await NoteOf(newApp, wordId)).Should().Be("a note"); - } - - [Fact] - public async Task ReconcileWithTheSameConfigDoesNothing() - { - (await DataModel.ReconcileConfigChanges()).ConfigChanged.Should().BeTrue("no config was stored yet"); - var wordId = Guid.NewGuid(); - await WriteNextChange(new NewWordChange(wordId, "hello")); - var snapshotIds = await SnapshotIds(this); - - var result = await DataModel.ReconcileConfigChanges(); - - result.Should().Be(ConfigReconcileResult.Unchanged); - (await SnapshotIds(this)).Should().Equal(snapshotIds); - } - - [Fact] - public async Task NewDatabaseStoresTheConfigWithoutReplaying() - { - var result = await DataModel.ReconcileConfigChanges(); - - result.ConfigChanged.Should().BeTrue(); - result.ReplayedFrom.Should().BeNull(); - (await DataModel.GetLocalState("harmony:config")).Should().NotBeNull(); - (await DataModel.ReconcileConfigChanges()).ConfigChanged.Should().BeFalse(); - } - - [Fact] - public async Task ReplayResumesFromTheCheckpointBeforeTheAddedChange() - { - void SmallCheckpoints(IServiceCollection services) => - services.Configure(config => config.MaxChangesBetweenSnapshotCheckpoints = 4); - await using var oldApp = ForkDatabase(services => - { - OldApp(services); - SmallCheckpoints(services); - }); - await oldApp.DataModel.ReconcileConfigChanges(); - var wordId = Guid.NewGuid(); - await oldApp.WriteNextChange(new NewWordChange(wordId, "text 0")); - for (var i = 1; i < 20; i++) - { - await oldApp.WriteNextChange(new SetWordTextChange(wordId, $"text {i}")); - } - var noteCommit = await SyncOpaqueNote(oldApp, wordId, "a note"); - await oldApp.WriteNextChange(new SetWordTextChange(wordId, "last text")); - - await using var newApp = oldApp.ForkDatabase(services => - { - NewApp(services); - SmallCheckpoints(services); - }); - var checkpoint = await newApp.DbContext.Commits.AsNoTracking() - .Where(c => c.IsSnapshotCheckpoint) - .WhereBefore(noteCommit, inclusive: false) - .DefaultOrderDescending() - .FirstAsync(TestContext.Current.CancellationToken); - var commitIdsUpToCheckpoint = newApp.DbContext.Commits.WhereBefore(checkpoint, inclusive: true).Select(c => c.Id); - var snapshotsUpToCheckpoint = await newApp.DbContext.Snapshots.AsNoTracking() - .Where(s => commitIdsUpToCheckpoint.Contains(s.CommitId)) - .Select(s => s.Id) - .ToArrayAsync(TestContext.Current.CancellationToken); - snapshotsUpToCheckpoint.Should().NotBeEmpty(); - - var result = await newApp.DataModel.ReconcileConfigChanges(); - - result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); - (await SnapshotIds(newApp)).Should().Contain(snapshotsUpToCheckpoint, "snapshots before the checkpoint should not be rebuilt"); - var word = await newApp.DataModel.GetLatest(wordId); - word!.Note.Should().Be("a note"); - word.Text.Should().Be("last text"); - - var stateAfterReconcile = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); - await newApp.DataModel.RegenerateSnapshots(); - var stateAfterRegenerate = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); - stateAfterReconcile.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted)) - .Should().BeEquivalentTo(stateAfterRegenerate.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted))); - } - - [Fact] - public async Task RemovedChangeTypeIsUnappliedAfterReconcile() - { - await using var newApp = ForkDatabase(NewApp); - await newApp.DataModel.ReconcileConfigChanges(); - var wordId = Guid.NewGuid(); - var firstCommit = await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); - await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); - - await using var oldApp = newApp.ForkDatabase(OldApp); - (await NoteOf(oldApp, wordId)).Should().Be("a note", "the snapshots were made by the new app"); - var downgrade = await oldApp.DataModel.ReconcileConfigChanges(); - - downgrade.AddedChangeTypes.Should().BeEmpty(); - downgrade.RemovedChangeTypes.Should().Equal(NoteChangeType); - downgrade.ReplayedFrom!.Id.Should().Be(firstCommit.Id, "a removed type regenerates all snapshots"); - (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); - - await using var upgradedAgain = oldApp.ForkDatabase(NewApp); - var upgrade = await upgradedAgain.DataModel.ReconcileConfigChanges(); - - upgrade.AddedChangeTypes.Should().Equal(NoteChangeType); - (await NoteOf(upgradedAgain, wordId)).Should().Be("a note"); - } - - [Fact] - public async Task RemovedCreateChangeTypeRemovesTheEntity() - { - await using var newApp = ForkDatabase(NewApp); - await newApp.DataModel.ReconcileConfigChanges(); - var wordId = Guid.NewGuid(); - var tagId = Guid.NewGuid(); - await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); - await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); - - await using var oldApp = newApp.ForkDatabase(services => - { - NewApp(services); - services.Configure(config => config.ChangeTypeListBuilder.Remove()); - }); - var result = await oldApp.DataModel.ReconcileConfigChanges(); - - result.RemovedChangeTypes.Should().Equal(nameof(SetTagChange)); - (await oldApp.DataModel.GetLatest(tagId)).Should().BeNull(); - (await oldApp.DbContext.Set().AnyAsync(TestContext.Current.CancellationToken)).Should().BeFalse(); - (await oldApp.DataModel.GetLatest(wordId)).Should().NotBeNull(); - } - - [Fact] - public async Task MissingConfigWithReplayAllReplaysAllOfHistory() - { - var wordId = Guid.NewGuid(); - var firstCommit = await WriteNextChange(new NewWordChange(wordId, "hello")); - await SyncOpaqueNote(this, wordId, "a note"); - - await using var newApp = ForkDatabase(NewApp); - var result = await newApp.DataModel.ReconcileConfigChanges(MissingConfigBehavior.ReplayAll); - - result.ConfigChanged.Should().BeTrue(); - result.AddedChangeTypes.Should().BeEmpty("there was no stored config to compare with"); - result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); - (await NoteOf(newApp, wordId)).Should().Be("a note"); - } - - [Fact] - public async Task MissingConfigWithAssumeCurrentConfigDoesNotReplay() - { - var wordId = Guid.NewGuid(); - await WriteNextChange(new NewWordChange(wordId, "hello")); - var snapshotIds = await SnapshotIds(this); - - var result = await DataModel.ReconcileConfigChanges(MissingConfigBehavior.AssumeCurrentConfig); - - result.ConfigChanged.Should().BeTrue(); - result.ReplayedFrom.Should().BeNull(); - (await SnapshotIds(this)).Should().Equal(snapshotIds); - - //the stored config is used from now on - await using var newApp = ForkDatabase(NewApp); - var upgrade = await newApp.DataModel.ReconcileConfigChanges(MissingConfigBehavior.AssumeCurrentConfig); - upgrade.AddedChangeTypes.Should().Equal(NoteChangeType); - } - - [Fact] - public async Task FindOldestCommitWithChangeTypesUsesCommitOrder() - { - await using var newApp = ForkDatabase(NewApp); - var wordId = Guid.NewGuid(); - var tagId = Guid.NewGuid(); - await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); - var laterNote = await newApp.WriteNextChange(new SetWordNoteChange(wordId, "later")); - var tagCommit = await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); - //added last, but sorts before the other note - var earlierNote = await newApp.WriteChangeBefore(laterNote, new SetWordNoteChange(wordId, "earlier")); - await using var repo = newApp.CreateRepository(); - - (await repo.FindOldestCommitWithChangeTypes([NoteChangeType]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeTypes([nameof(SetTagChange)]))!.Id.Should().Be(tagCommit.Id); - (await repo.FindOldestCommitWithChangeTypes([nameof(SetTagChange), NoteChangeType]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeTypes([nameof(EditExampleChange)])).Should().BeNull(); - (await repo.FindOldestCommitWithChangeTypes([])).Should().BeNull(); - } - - [Fact] - public async Task LocalStateCanBeSetReadAndRemoved() - { - (await DataModel.GetLocalState("app:missing")).Should().BeNull(); - - await DataModel.SetLocalState("app:value", new LocalStateTestValue("first", 1)); - (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("first", 1)); - - await DataModel.SetLocalState("app:value", new LocalStateTestValue("second", 2)); - (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("second", 2)); - - await DataModel.RemoveLocalState("app:value"); - (await DataModel.GetLocalState("app:value")).Should().BeNull(); - await DataModel.RemoveLocalState("app:value"); - } - - private record LocalStateTestValue(string Text, int Number); - - private static async Task SnapshotIds(DataModelTestBase app) - { - return await app.DbContext.Snapshots.AsNoTracking() - .Select(s => s.Id) - .OrderBy(id => id) - .ToArrayAsync(TestContext.Current.CancellationToken); - } -} diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index 7519320..d4176c2 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -86,6 +86,8 @@ internal CrdtRepository CreateRepository() => internal HarmonyConfig CrdtConfig => _services.GetRequiredService>().Value; + internal ModelVersionService ModelVersionService => _services.GetRequiredService(); + private static int _instanceCount = 0; private DateTimeOffset currentDate = new(new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddHours(_instanceCount++)); public DateTimeOffset NextDate() => currentDate = currentDate.AddDays(1); diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 5a92a73..fb1ce1d 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -40,6 +40,9 @@ Annotations: Relational:ColumnType: jsonb EntityId (Guid) Required + Version (int) Required ValueGenerated.OnAdd + Annotations: + Relational:DefaultValue: 0 Keys: CommitId, Index PK Foreign keys: diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt new file mode 100644 index 0000000..d4b6216 --- /dev/null +++ b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt @@ -0,0 +1,3 @@ +1 Notes: SetWordNoteChange +2 major Rewrite: +3 Tags and notes: SetTagChange, SetWordNoteChange diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs new file mode 100644 index 0000000..ffc8b1d --- /dev/null +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -0,0 +1,437 @@ +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using SIL.Harmony.Changes; +using SIL.Harmony.Config; +using SIL.Harmony.Db; +using SIL.Harmony.Sample; +using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Tests; + +/// +/// The database starts out written by an "old app" that has no model versions, +/// and is then opened by a "new app" with model versions, by forking it with a different config. +/// +public class ModelVersionTests() : DataModelTestBase(configure: OldApp) +{ + private const string NoteChangeType = nameof(SetWordNoteChange); + + /// knows , but applies it the old way + private static void OldApp(IServiceCollection services) + { + services.Configure(config => config.UnknownChangeHandling = UnknownChangeHandling.Fallback); + } + + /// doesn't know at all + private static void OldAppWithoutNotes(IServiceCollection services) + { + OldApp(services); + services.Configure(config => config.ChangeTypeListBuilder.Remove()); + } + + private static void NewApp(IServiceCollection services) + { + OldApp(services); + services.Configure(config => + config.ModelVersionBuilder.Add("Notes", v => v.Change())); + } + + private static Action AppWith(Action versions) + { + return services => + { + OldApp(services); + services.Configure(config => versions(config.ModelVersionBuilder)); + }; + } + + private static readonly JsonSerializerOptions NewAppJsonOptions = CreateNewAppJsonOptions(); + + private static JsonSerializerOptions CreateNewAppJsonOptions() + { + using var services = new ServiceCollection().AddCrdtDataSample(":memory:").BuildServiceProvider(); + return services.GetRequiredService(); + } + + /// a note change as an app that doesn't know the type sees it, after it was synced from a newer client + private static IChange OpaqueNote(DataModelTestBase oldApp, Guid wordId, string note) + { + var json = JsonSerializer.Serialize(new SetWordNoteChange(wordId, note), NewAppJsonOptions); + var change = JsonSerializer.Deserialize(json, oldApp.CrdtConfig.JsonSerializerOptions); + return change.Should().BeOfType().Subject; + } + + /// a commit authored by another client, with the change version that client's app gave it + private static async Task SyncFromOtherClient(DataModelTestBase app, IChange change, int version) + { + var commit = new Commit + { + ClientId = Guid.NewGuid(), + HybridDateTime = new HybridDateTime(app.NextDate(), 0), + }; + commit.ChangeEntities.Add(new ChangeEntity + { + Change = change, + CommitId = commit.Id, + EntityId = change.EntityId, + Index = 0, + Version = version + }); + await app.AddCommitsViaSync([commit]); + return commit; + } + + private static async Task NoteOf(DataModelTestBase app, Guid wordId) + { + var word = await app.DataModel.GetLatest(wordId); + word.Should().NotBeNull(); + var projectedWord = await app.DbContext.Set().AsNoTracking() + .SingleAsync(w => w.Id == wordId, TestContext.Current.CancellationToken); + projectedWord.Note.Should().Be(word.Note, "the projected table should match the snapshot"); + return word.Note; + } + + private static async Task SnapshotIds(DataModelTestBase app) + { + return await app.DbContext.Snapshots.AsNoTracking() + .Select(s => s.Id) + .OrderBy(id => id) + .ToArrayAsync(TestContext.Current.CancellationToken); + } + + private static async Task ChangeVersions(DataModelTestBase app, Commit commit) + { + return await app.DbContext.Set>().AsNoTracking() + .Where(ce => ce.CommitId == commit.Id) + .OrderBy(ce => ce.Index) + .Select(ce => ce.Version) + .ToArrayAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task AuthoredChangesGetTheVersionOfTheirType() + { + await using var app = ForkDatabase(AppWith(versions => versions + .Add("Notes", v => v.Change()) + .Add("Notes and tags", v => v.Change().Change()))); + var wordId = Guid.NewGuid(); + + var commit = await app.WriteNextChange([ + new NewWordChange(wordId, "hello"), + new SetTagChange(Guid.NewGuid(), "tag"), + new SetWordNoteChange(wordId, "note") + ]); + + commit.ChangeEntities.Select(ce => ce.Version).Should().Equal(0, 1, 2); + (await ChangeVersions(app, commit)).Should().Equal(0, 1, 2); + var json = JsonSerializer.Serialize(commit, app.CrdtConfig.JsonSerializerOptions); + JsonSerializer.Deserialize(json, app.CrdtConfig.JsonSerializerOptions)! + .ChangeEntities.Select(ce => ce.Version).Should().Equal(0, 1, 2); + } + + [Fact] + public async Task UnknownTypeIsAppliedAfterUpgrade() + { + await using var oldApp = ForkDatabase(OldAppWithoutNotes); + await oldApp.ModelVersionService.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + await oldApp.WriteNextChange(new NewWordChange(wordId, "hello")); + var noteCommit = await SyncFromOtherClient(oldApp, OpaqueNote(oldApp, wordId, "a note"), version: 1); + (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); + + await using var newApp = oldApp.ForkDatabase(NewApp); + var result = await newApp.ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(0, 1, false, result.ReplayedFrom)); + result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + (await NoteOf(newApp, wordId)).Should().Be("a note"); + } + + [Fact] + public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() + { + await DataModel.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + var oldNote = await WriteNextChange(new SetWordNoteChange(wordId, "old note")); + var newerNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "newer note"), version: 1); + (await ChangeVersions(this, oldNote)).Should().Equal(0); + + await using var newApp = ForkDatabase(NewApp); + var result = await newApp.ModelVersionService.ReconcileModelVersions(); + + result.ReplayedFrom!.Id.Should().Be(newerNote.Id, + "the old note was authored and applied by the old app, so it doesn't need a replay"); + result.FullRegeneration.Should().BeFalse(); + (await NoteOf(newApp, wordId)).Should().Be("newer note"); + } + + [Fact] + public async Task ReplayResumesFromTheCheckpointBeforeTheChange() + { + static void SmallCheckpoints(IServiceCollection services) => + services.Configure(config => config.MaxChangesBetweenSnapshotCheckpoints = 4); + await using var oldApp = ForkDatabase(services => + { + OldAppWithoutNotes(services); + SmallCheckpoints(services); + }); + await oldApp.ModelVersionService.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + await oldApp.WriteNextChange(new NewWordChange(wordId, "text 0")); + for (var i = 1; i < 20; i++) + { + await oldApp.WriteNextChange(new SetWordTextChange(wordId, $"text {i}")); + } + var noteCommit = await SyncFromOtherClient(oldApp, OpaqueNote(oldApp, wordId, "a note"), version: 1); + await oldApp.WriteNextChange(new SetWordTextChange(wordId, "last text")); + + await using var newApp = oldApp.ForkDatabase(services => + { + NewApp(services); + SmallCheckpoints(services); + }); + var checkpoint = await newApp.DbContext.Commits.AsNoTracking() + .Where(c => c.IsSnapshotCheckpoint) + .WhereBefore(noteCommit, inclusive: false) + .DefaultOrderDescending() + .FirstAsync(TestContext.Current.CancellationToken); + var commitIdsUpToCheckpoint = newApp.DbContext.Commits.WhereBefore(checkpoint, inclusive: true).Select(c => c.Id); + var snapshotsUpToCheckpoint = await newApp.DbContext.Snapshots.AsNoTracking() + .Where(s => commitIdsUpToCheckpoint.Contains(s.CommitId)) + .Select(s => s.Id) + .ToArrayAsync(TestContext.Current.CancellationToken); + snapshotsUpToCheckpoint.Should().NotBeEmpty(); + + var result = await newApp.ModelVersionService.ReconcileModelVersions(); + + result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + (await SnapshotIds(newApp)).Should().Contain(snapshotsUpToCheckpoint, "snapshots before the checkpoint should not be rebuilt"); + var word = await newApp.DataModel.GetLatest(wordId); + word!.Note.Should().Be("a note"); + word.Text.Should().Be("last text"); + + var stateAfterReconcile = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); + await newApp.DataModel.RegenerateSnapshots(); + var stateAfterRegenerate = await newApp.DataModel.GetProjectSnapshot(includeDeleted: true); + stateAfterReconcile.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted)) + .Should().BeEquivalentTo(stateAfterRegenerate.Snapshots.Values.Select(s => (s.EntityId, s.CommitId, s.EntityIsDeleted))); + } + + [Fact] + public async Task MajorVersionRegeneratesAllSnapshots() + { + await DataModel.ReconcileModelVersions(); + var firstCommit = await WriteNextChange(new NewWordChange(Guid.NewGuid(), "hello")); + await WriteNextChange(new NewWordChange(Guid.NewGuid(), "world")); + + await using var newApp = ForkDatabase(AppWith(versions => versions.AddMajor("Rewrite"))); + var result = await newApp.ModelVersionService.ReconcileModelVersions(); + + result.FullRegeneration.Should().BeTrue(); + result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + result.CurrentVersion.Should().Be(1); + } + + [Fact] + public async Task SameVersionDoesNothing() + { + await using var app = ForkDatabase(NewApp); + (await app.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeTrue("no version was stored yet"); + await app.WriteNextChange(new NewWordChange(Guid.NewGuid(), "hello")); + var snapshotIds = await SnapshotIds(app); + + var result = await app.ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(1, 1, false, null)); + result.Changed.Should().BeFalse(); + (await SnapshotIds(app)).Should().Equal(snapshotIds); + } + + [Fact] + public async Task NoStoredVersionIsVersionZero() + { + var wordId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + await WriteNextChange(new SetWordNoteChange(wordId, "a note")); + + await using var newApp = ForkDatabase(NewApp); + var snapshotIds = await SnapshotIds(newApp); + var result = await newApp.ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(0, 1, false, null), + "all changes are version 0, so the old app applied them the way they were authored"); + (await SnapshotIds(newApp)).Should().Equal(snapshotIds); + (await newApp.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeFalse(); + } + + [Fact] + public async Task NewDatabaseStoresTheVersionWithoutReplaying() + { + await using var app = ForkDatabase(NewApp); + + var result = await app.ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(0, 1, false, null)); + (await app.DataModel.GetLocalState("harmony:modelVersions")).Should().NotBeNull(); + } + + [Fact] + public async Task DowngradeRegeneratesAllSnapshots() + { + await using var newApp = ForkDatabase(NewApp); + await newApp.ModelVersionService.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + var firstCommit = await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); + + await using var oldApp = newApp.ForkDatabase(OldApp); + var result = await oldApp.ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(1, 0, true, result.ReplayedFrom)); + result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + (await NoteOf(oldApp, wordId)).Should().Be("a note"); + } + + [Fact] + public async Task DowngradeToAnAppWithoutAChangeTypeRemovesItsEntities() + { + await using var newApp = ForkDatabase(AppWith(versions => versions.Add("Tags", v => v.Change()))); + await newApp.ModelVersionService.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); + await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); + + await using var oldApp = newApp.ForkDatabase(services => + { + OldApp(services); + services.Configure(config => config.ChangeTypeListBuilder.Remove()); + }); + var result = await oldApp.ModelVersionService.ReconcileModelVersions(); + + result.FullRegeneration.Should().BeTrue(); + (await oldApp.DataModel.GetLatest(tagId)).Should().BeNull(); + (await oldApp.DbContext.Set().AnyAsync(TestContext.Current.CancellationToken)).Should().BeFalse(); + (await oldApp.DataModel.GetLatest(wordId)).Should().NotBeNull(); + } + + [Fact] + public async Task ModifiedAppliedVersionThrows() + { + await using var newApp = ForkDatabase(NewApp); + await newApp.ModelVersionService.ReconcileModelVersions(); + + await using var otherTypes = newApp.ForkDatabase(AppWith(versions => versions.Add("Notes", v => v.Change()))); + var act = () => otherTypes.ModelVersionService.ReconcileModelVersions(); + await act.Should().ThrowAsync().WithMessage("Model version 1 (Notes) was modified*"); + + await using var nowMajor = newApp.ForkDatabase(AppWith(versions => versions.AddMajor("Notes", v => v.Change()))); + act = () => nowMajor.ModelVersionService.ReconcileModelVersions(); + await act.Should().ThrowAsync(); + + await using var renamed = newApp.ForkDatabase(AppWith(versions => versions.Add("Word notes", v => v.Change()))); + (await renamed.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeFalse("names are not stored"); + } + + [Fact] + public async Task DescribeModelVersions() + { + await using var app = ForkDatabase(AppWith(versions => versions + .Add("Notes", v => v.Change()) + .AddMajor("Rewrite") + .Add("Tags and notes", v => v.Change().Change()))); + + await Verify(app.CrdtConfig.DescribeModelVersions()); + } + + [Fact] + public void ModelVersionWithAnUnregisteredChangeTypeFailsValidation() + { + using var services = new ServiceCollection() + .AddCrdtDataSample(":memory:") + .Configure(config => + { + config.ChangeTypeListBuilder.Remove(); + config.ModelVersionBuilder.Add("Notes", v => v.Change()); + }) + .BuildServiceProvider(); + + var act = () => services.GetRequiredService>().Value; + + act.Should().Throw(); + } + + [Fact] + public void ChangeVersionsToReplayAreTheVersionsBeforeTheUpgrade() + { + ModelVersion[] versions = + [ + new("A", false, [NoteChangeType]), + new("B", false, [nameof(SetTagChange), NoteChangeType]), + new("C", false, [nameof(SetTagChange)]), + ]; + + ModelVersionService.ChangeVersionsToReplay(1, versions) + .Should().BeEquivalentTo([(nameof(SetTagChange), 0), (NoteChangeType, 1)]); + ModelVersionService.ChangeVersionsToReplay(0, versions) + .Should().BeEquivalentTo([(NoteChangeType, 0), (nameof(SetTagChange), 0)]); + ModelVersionService.ChangeVersionsToReplay(3, versions).Should().BeEmpty(); + } + + [Fact] + public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() + { + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + await WriteNextChange(new NewWordChange(wordId, "hello")); + var versionZeroNote = await WriteNextChange(new SetWordNoteChange(wordId, "version 0")); + var laterNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "later"), version: 1); + var tagCommit = await SyncFromOtherClient(this, new SetTagChange(tagId, "tag"), version: 2); + //added last, but sorts before the other version 1 note + var earlierNote = await WriteChangeBefore(laterNote, new SetWordNoteChange(wordId, "earlier"), add: false); + earlierNote.ChangeEntities.Single().Version = 1; + await AddCommitsViaSync([earlierNote]); + await using var repo = CreateRepository(); + + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 0)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, -1)]))!.Id.Should().Be(versionZeroNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 1)])).Should().BeNull(); + (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 1)]))!.Id.Should().Be(tagCommit.Id); + (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 1), (NoteChangeType, 0)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(nameof(EditExampleChange), 0)])).Should().BeNull(); + (await repo.FindOldestCommitWithChangeVersions([])).Should().BeNull(); + } + + [Fact] + public async Task DataModelReconcilePassesThroughToTheService() + { + await using var app = ForkDatabase(NewApp); + + var result = await app.DataModel.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(0, 1, false, null)); + (await app.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeFalse(); + } + + [Fact] + public async Task LocalStateCanBeSetReadAndRemoved() + { + (await DataModel.GetLocalState("app:missing")).Should().BeNull(); + + await DataModel.SetLocalState("app:value", new LocalStateTestValue("first", 1)); + (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("first", 1)); + + await DataModel.SetLocalState("app:value", new LocalStateTestValue("second", 2)); + (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("second", 2)); + + await DataModel.RemoveLocalState("app:value"); + (await DataModel.GetLocalState("app:value")).Should().BeNull(); + await DataModel.RemoveLocalState("app:value"); + } + + private record LocalStateTestValue(string Text, int Number); +} diff --git a/src/SIL.Harmony/Config/HarmonyConfig.cs b/src/SIL.Harmony/Config/HarmonyConfig.cs index 6c6fc89..739214d 100644 --- a/src/SIL.Harmony/Config/HarmonyConfig.cs +++ b/src/SIL.Harmony/Config/HarmonyConfig.cs @@ -1,4 +1,6 @@ using System.Collections.Concurrent; +using System.Collections.Frozen; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Microsoft.EntityFrameworkCore; @@ -46,6 +48,14 @@ public class HarmonyConfig public UnknownChangeHandling UnknownChangeHandling { get; set; } = UnknownChangeHandling.Throw; public ChangeTypeListBuilder ChangeTypeListBuilder { get; } = new(); public IReadOnlyList ChangeTypes => ChangeTypeListBuilder.Types; + /// + /// The ordered list of model versions, see . + /// + public ModelVersionBuilder ModelVersionBuilder { get; } = new(); + public IReadOnlyList ModelVersions => ModelVersionBuilder.Versions; + /// the number of model versions, 0 when the app declares none + public int CurrentModelVersion => ModelVersions.Count; + private readonly Lazy> _lazyChangeVersions; public ObjectTypeListBuilder ObjectTypeListBuilder { get; } = new(); public IEnumerable ObjectTypes => ObjectTypeListBuilder.AdapterProviders.SelectMany(p => p.GetRegistrations().Select(r => r.ObjectDbType)); public JsonSerializerOptions JsonSerializerOptions => _lazyJsonSerializerOptions.Value; @@ -66,6 +76,7 @@ public class HarmonyConfig public HarmonyConfig() { _lazyChangeDiscriminatorMaps = new Lazy(BuildChangeDiscriminatorMaps); + _lazyChangeVersions = new Lazy>(BuildChangeVersions); _lazyJsonSerializerOptions = new Lazy(CreateJsonSerializerOptions); } @@ -115,6 +126,51 @@ private ChangeDiscriminatorMaps BuildChangeDiscriminatorMaps() return new ChangeDiscriminatorMaps(knownChanges, discriminators); } + private FrozenDictionary BuildChangeVersions() + { + ModelVersionBuilder.Freeze(); + return ModelVersions.SelectMany(v => v.ChangeTypes) + .CountBy(t => t, StringComparer.Ordinal) + .ToFrozenDictionary(StringComparer.Ordinal); + } + + /// + /// The version a change of this type is authored with: how many model versions list the type. + /// + /// the change type's discriminator, its $type + public int ChangeVersion(string discriminator) + { + return _lazyChangeVersions.Value.GetValueOrDefault(discriminator); + } + + internal int ChangeVersion(IChange change) + { + var discriminator = change is OpaqueChange opaque + ? opaque.TypeName + : _lazyChangeDiscriminatorMaps.Value.ByType.GetValueOrDefault(change.GetType()); + return discriminator is null ? 0 : ChangeVersion(discriminator); + } + + /// + /// A stable text form of , one line per version. + /// Use it in a snapshot test so a released version can't be modified by accident: the snapshot may only grow at the end. + /// + public string DescribeModelVersions() + { + var builder = new StringBuilder(); + for (var i = 0; i < ModelVersions.Count; i++) + { + var version = ModelVersions[i]; + builder.Append(i + 1) + .Append(version.Major ? " major " : " ") + .Append(version.Name) + .Append(": ") + .AppendJoin(", ", version.ChangeTypes) + .Append('\n'); + } + return builder.ToString(); + } + private sealed record ChangeDiscriminatorMaps( IReadOnlyDictionary ByDiscriminator, IReadOnlyDictionary ByType); diff --git a/src/SIL.Harmony/Config/ModelVersion.cs b/src/SIL.Harmony/Config/ModelVersion.cs new file mode 100644 index 0000000..8d9e03e --- /dev/null +++ b/src/SIL.Harmony/Config/ModelVersion.cs @@ -0,0 +1,22 @@ +namespace SIL.Harmony.Config; + +/// +/// One step in the ordered list of model versions an app declares on . +/// Released versions must never be modified, only new versions added at the end. +/// +/// describes the version for people, it is not stored in the database +/// when true, upgrading to this version regenerates all snapshots +/// discriminators of the change types this version added or modified, sorted +public sealed record ModelVersion(string Name, bool Major, IReadOnlyList ChangeTypes) +{ + public bool Equals(ModelVersion? other) + { + return other is not null && Name == other.Name && Major == other.Major + && ChangeTypes.SequenceEqual(other.ChangeTypes, StringComparer.Ordinal); + } + + public override int GetHashCode() + { + return HashCode.Combine(Name, Major, ChangeTypes.Count); + } +} diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs new file mode 100644 index 0000000..6ca07c1 --- /dev/null +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -0,0 +1,86 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Config; + +/// +/// Declares the ordered list of model versions. Add a version when a release adds a change type, or changes how +/// an existing change type applies. Changes authored by a newer app are then replayed when an older app upgrades, +/// see . +/// Released versions must never be modified, use in a snapshot test to make sure. +/// +public class ModelVersionBuilder +{ + private bool _frozen; + private readonly List _versions = []; + private readonly HashSet _changeTypes = []; + + public IReadOnlyList Versions => _versions.AsReadOnly(); + + /// change types added with , which must be registered + internal IReadOnlyCollection ChangeTypes => _changeTypes; + + /// + /// Adds a version. Changes of the listed types that were authored with this version or later + /// are replayed when upgrading to it. + /// + public ModelVersionBuilder Add(string name, Action changeTypes) + { + return Add(name, false, changeTypes); + } + + /// + /// Adds a version that regenerates all snapshots when upgrading to it. + /// The listed change types still get a new version number for the changes authored with them. + /// + public ModelVersionBuilder AddMajor(string name, Action? changeTypes = null) + { + return Add(name, true, changeTypes); + } + + private ModelVersionBuilder Add(string name, bool major, Action? configure) + { + CheckFrozen(); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var changeTypes = new ModelVersionChangeTypes(); + configure?.Invoke(changeTypes); + _changeTypes.UnionWith(changeTypes.Types); + _versions.Add(new ModelVersion(name, major, + changeTypes.Discriminators.Order(StringComparer.Ordinal).ToArray())); + return this; + } + + internal void Freeze() + { + _frozen = true; + } + + private void CheckFrozen() + { + if (_frozen) throw new InvalidOperationException($"{nameof(ModelVersionBuilder)} is frozen"); + } +} + +public class ModelVersionChangeTypes +{ + internal HashSet Discriminators { get; } = new(StringComparer.Ordinal); + internal List Types { get; } = []; + + /// the change type was added or modified in this version + public ModelVersionChangeTypes Change() where T : IChange, IPolyType + { + if (Discriminators.Add(T.TypeName)) Types.Add(typeof(T)); + return this; + } + + /// + /// The change type with this discriminator was added or modified in this version. + /// Only for change types that no longer exist in the code, use otherwise. + /// + public ModelVersionChangeTypes Change(string discriminator) + { + ArgumentException.ThrowIfNullOrWhiteSpace(discriminator); + Discriminators.Add(discriminator); + return this; + } +} diff --git a/src/SIL.Harmony/ConfigReconciliation.cs b/src/SIL.Harmony/ConfigReconciliation.cs deleted file mode 100644 index bfdae3e..0000000 --- a/src/SIL.Harmony/ConfigReconciliation.cs +++ /dev/null @@ -1,55 +0,0 @@ -using SIL.Harmony.Config; - -namespace SIL.Harmony; - -/// -/// What does when the database has no stored config, -/// for example a database created before config tracking existed. -/// -public enum MissingConfigBehavior -{ - /// - /// Replay from the oldest commit with a change type known to the current config, which is usually all of history. - /// Safe for any database, but expensive for big ones. - /// - ReplayAll, - /// - /// Accept the current config as the config the data was written with: store it and don't replay. - /// Only use this when every client that wrote to the database used , - /// or knew the same change types as the current config. Otherwise snapshots may be missing changes. - /// - AssumeCurrentConfig, -} - -/// false when the stored config matched the current one and nothing was done -/// change types the current config knows that the stored config did not -/// change types the stored config knew that the current config does not -/// the oldest commit snapshots were rebuilt from, null when nothing was replayed -public record ConfigReconcileResult( - bool ConfigChanged, - IReadOnlyList AddedChangeTypes, - IReadOnlyList RemovedChangeTypes, - Commit? ReplayedFrom) -{ - internal static readonly ConfigReconcileResult Unchanged = new(false, [], [], null); -} - -/// -/// The parts of the that affect which snapshots are generated, stored in the local state. -/// -internal record StoredHarmonyConfig(int Version, string[] ChangeTypes) -{ - public const string LocalStateKey = "harmony:config"; - public const int CurrentVersion = 1; - - public static StoredHarmonyConfig From(HarmonyConfig config) - { - return new StoredHarmonyConfig(CurrentVersion, - config.ChangeTypes.Select(t => t.Discriminator).Order(StringComparer.Ordinal).ToArray()); - } - - public bool SameAs(StoredHarmonyConfig other) - { - return Version == other.Version && ChangeTypes.SequenceEqual(other.ChangeTypes, StringComparer.Ordinal); - } -} diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index ce7b2bf..df41b4c 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -60,19 +60,28 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service .Validate( config => config.MaxChangesBetweenSnapshotCheckpoints >= 1, $"{nameof(HarmonyConfig.MaxChangesBetweenSnapshotCheckpoints)} must be at least 1") - .PostConfigure(crdtConfig => crdtConfig.ObjectTypeListBuilder.Freeze()); + .PostConfigure(crdtConfig => + { + crdtConfig.ObjectTypeListBuilder.Freeze(); + crdtConfig.ModelVersionBuilder.Freeze(); + }) + .Validate( + config => config.ModelVersionBuilder.ChangeTypes.All(t => config.ChangeTypes.Any(ct => ct.Type == t)), + "Every change type in a model version must be registered in the ChangeTypeListBuilder"); services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); services.AddSingleton(); services.AddScoped(); + services.AddScoped(); //must use factory method because DataModel constructor is internal services.AddScoped(provider => new DataModel( provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService>(), - provider.GetRequiredService>() + provider.GetRequiredService>(), + provider.GetRequiredService() )); return services; } diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 1066b68..127302a 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -24,19 +24,22 @@ public class DataModel : ISyncable, IAsyncDisposable private readonly IHybridDateTimeProvider _timeProvider; private readonly IOptions _crdtConfig; private readonly ILogger _logger; + private readonly ModelVersionService _modelVersionService; //constructor must be internal because CrdtRepository is internal internal DataModel(CrdtRepositoryFactory crdtRepositoryFactory, JsonSerializerOptions serializerOptions, IHybridDateTimeProvider timeProvider, IOptions crdtConfig, - ILogger logger) + ILogger logger, + ModelVersionService modelVersionService) { _crdtRepositoryFactory = crdtRepositoryFactory; _serializerOptions = serializerOptions; _timeProvider = timeProvider; _crdtConfig = crdtConfig; _logger = logger; + _modelVersionService = modelVersionService; } @@ -98,7 +101,7 @@ private Commit NewCommit(Guid clientId, CommitMetadata? commitMetadata, IEnumera HybridDateTime = _timeProvider.GetDateTime(), Metadata = commitMetadata ?? new() }; - commit.ChangeEntities.AddRange(changes.Select((c, i) => ToChangeEntity(c, i, commit.Id))); + commit.ChangeEntities.AddRange(changes.Select((c, i) => ToChangeEntity(c, i, commit.Id, _crdtConfig.Value))); return commit; } @@ -119,14 +122,15 @@ private async Task Add(Commit commit) if (transaction is not null) await transaction.CommitAsync(); } - internal static ChangeEntity ToChangeEntity(IChange change, int index, Guid commitId) + internal static ChangeEntity ToChangeEntity(IChange change, int index, Guid commitId, HarmonyConfig config) { return new ChangeEntity() { Change = change, CommitId = commitId, EntityId = change.EntityId, - Index = index + Index = index, + Version = config.ChangeVersion(change) }; } @@ -182,34 +186,9 @@ ValueTask ISyncable.ShouldSync() return ValueTask.FromResult(true); } - /// - /// Rebuilds every snapshot the window covers by replaying its commits onto the state it resumes from. - /// A window resuming from nothing rebuilds all of history. - /// - private async Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) + private Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window) { - var (checkpoint, commitsToApply) = window; - if (commitsToApply.Count == 0) return; - - ISnapshotView baseline; - // A database with no checkpoints replays all of history, - // which is what we want, because it will trigger creating checkpoints - if (checkpoint is null) - { - await repo.DeleteSnapshotsAndProjectedTables(); - //the delete left the table empty, so there's nothing to query - baseline = EmptySnapshotView.Instance; - } - else - { - await repo.DeleteSnapshotsAfter(checkpoint.Commit); - //the current table is the state at the checkpoint, because the delete above just made it so - baseline = repo.CurrentSnapshotView(); - } - - await PreloadTouched(baseline, commitsToApply); - var (newSnapshots, checkpoints) = await SnapshotWorker.ComputeNewSnapshotsAndCheckpoints(baseline, commitsToApply, _crdtConfig.Value); - await repo.AddSnapshots(newSnapshots, checkpoints); + return SnapshotReplay.UpdateSnapshots(repo, window, _crdtConfig.Value); } private async Task ValidateCommits(CrdtRepository repo) @@ -238,73 +217,14 @@ public async Task RegenerateSnapshots() using var locked = await repo.Lock(); repo.ClearChangeTracker(); await using var transaction = await repo.BeginTransactionAsync(); - await RegenerateSnapshots(repo); + await SnapshotReplay.RegenerateAll(repo, _crdtConfig.Value); await transaction.CommitAsync(); } - private async Task RegenerateSnapshots(CrdtRepository repo) + /// + public Task ReconcileModelVersions() { - var wholeHistory = await repo.WholeHistory(); - //Replay does nothing without commits, which would leave snapshots with no history behind them in place - if (wholeHistory.Commits.Count == 0) await repo.DeleteSnapshotsAndProjectedTables(); - else await UpdateSnapshots(repo, wholeHistory); - } - - /// - /// Detects change types that were added to or removed from the config since the last call, for example after an app update, - /// and rebuilds the snapshots from the oldest commit that uses one of them. - /// Changes of an unknown type are stored as and skipped, so once the type is known - /// they must be replayed. A removed type is the reverse: its changes are applied in the snapshots, but are now opaque, - /// so all snapshots are regenerated. - /// Call this when opening a database, before using it. Querying changes by type is SQLite only. - /// - /// what to do when no config was stored yet, see - public async Task ReconcileConfigChanges( - MissingConfigBehavior missingConfig = MissingConfigBehavior.ReplayAll) - { - await using var repo = await _crdtRepositoryFactory.CreateRepository(); - using var locked = await repo.Lock(); - repo.ClearChangeTracker(); - await using var transaction = await repo.BeginTransactionAsync(); - var currentConfig = StoredHarmonyConfig.From(_crdtConfig.Value); - var storedConfig = await repo.GetLocalState(StoredHarmonyConfig.LocalStateKey); - if (storedConfig is not null && storedConfig.SameAs(currentConfig)) return ConfigReconcileResult.Unchanged; - - string[] added = []; - string[] removed = []; - string[] affected; - if (storedConfig is null) - { - affected = missingConfig == MissingConfigBehavior.ReplayAll ? currentConfig.ChangeTypes : []; - } - else - { - added = currentConfig.ChangeTypes.Except(storedConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); - removed = storedConfig.ChangeTypes.Except(currentConfig.ChangeTypes, StringComparer.Ordinal).ToArray(); - affected = [.. added, .. removed]; - } - - var replayFrom = await repo.FindOldestCommitWithChangeTypes(affected); - if (replayFrom is not null) - { - _logger.LogInformation("Change types changed (added: {Added}, removed: {Removed}), replaying from commit {CommitId}", - added, removed, replayFrom.Id); - if (removed.Length > 0) - { - //rare, so keep it simple: a replay from a checkpoint only projects the snapshots it makes, but a removed - //type can leave entities with fewer snapshots than before, so their projected rows would be stale - await RegenerateSnapshots(repo); - replayFrom = await repo.CurrentCommits().FirstAsync(); - } - else - { - await UpdateSnapshots(repo, await repo.ReplayWindowFrom(replayFrom)); - } - } - - await repo.SetLocalState(StoredHarmonyConfig.LocalStateKey, currentConfig); - await transaction.CommitAsync(); - return new ConfigReconcileResult(true, added, removed, replayFrom); + return _modelVersionService.ReconcileModelVersions(); } /// @@ -468,7 +388,7 @@ private async Task GetAtCommit(Commit commit, Guid entityId, CrdtRepositor //we don't have a persisted snapshot in the correct state, so rebuild it var (baseline, commitsToReplay) = await ResumeFromCheckpoint(commit, repo); - await PreloadTouched(baseline, commitsToReplay); + await SnapshotReplay.PreloadTouched(baseline, commitsToReplay, _crdtConfig.Value); return await (await SnapshotWorker.ReplayCommits(baseline, commitsToReplay, _crdtConfig.Value)).GetAsync(entityId); } @@ -486,12 +406,6 @@ private async Task GetAtCommit(Commit commit, Guid entityId, CrdtRepositor } /// one query for every entity the commits touch beats a lookup per entity only for large batches - private async Task PreloadTouched(ISnapshotView baseline, IEnumerable commits) - { - var entityIds = commits.SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId)).ToHashSet(); - if (entityIds.Count > _crdtConfig.Value.PrefetchSnapshotsBreakpoint) await baseline.PreloadAsync(entityIds); - } - public async Task GetSyncState() { await using var repo = await _crdtRepositoryFactory.CreateRepository(); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index b8b97cb..7bc7a00 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -393,25 +393,28 @@ public async Task ReplayWindowFrom(Commit commit) } /// - /// The oldest commit with a change whose $type is one of . + /// The oldest commit with a change whose $type is one of + /// and whose is higher than the version given for that type. /// Reads every change, so it's slow, only use it for rare operations. SQLite only. /// - public async Task FindOldestCommitWithChangeTypes(IReadOnlyCollection changeTypes) + public async Task FindOldestCommitWithChangeVersions( + IReadOnlyCollection<(string ChangeType, int AfterVersion)> changeVersions) { - if (changeTypes.Count == 0) return null; + if (changeVersions.Count == 0) return null; if (!_dbContext.Database.IsSqlite()) throw new NotSupportedException( $"Querying changes by type is only supported on SQLite, not {_dbContext.Database.ProviderName}"); //the key must be quoted in the path, because $ has a special meaning in json paths var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; //one json array parameter, so the parameter count doesn't depend on how many types there are - var changeTypesJson = JsonSerializer.Serialize(changeTypes); + var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.AfterVersion })); var commitIds = await _dbContext.Database.SqlQuery($""" SELECT c.Id AS Value FROM Commits c WHERE EXISTS ( - SELECT 1 FROM ChangeEntities ce + SELECT 1 FROM ChangeEntities ce, json_each({changeVersionsJson}) cv WHERE ce.CommitId = c.Id - AND json_extract(ce.Change, {typeDiscriminatorPath}) IN (SELECT value FROM json_each({changeTypesJson})) + AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(cv.value, '$.t') + AND ce.Version > json_extract(cv.value, '$.v') ) ORDER BY c.DateTime, c.Counter, c.Id LIMIT 1 diff --git a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs index 306c854..b8aa259 100644 --- a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs +++ b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs @@ -12,6 +12,7 @@ public void Configure(EntityTypeBuilder> builder) { builder.ToTable("ChangeEntities"); builder.HasKey(c => new { c.CommitId, c.Index }); + builder.Property(c => c.Version).HasDefaultValue(0); builder.Property(c => c.Change) .HasColumnType("jsonb") .HasConversion( diff --git a/src/SIL.Harmony/ModelVersionReconciliation.cs b/src/SIL.Harmony/ModelVersionReconciliation.cs new file mode 100644 index 0000000..43e6706 --- /dev/null +++ b/src/SIL.Harmony/ModelVersionReconciliation.cs @@ -0,0 +1,40 @@ +using SIL.Harmony.Config; + +namespace SIL.Harmony; + +/// the model version stored in the database before the call, 0 when none was stored +/// the model version of the current config, which is now stored +/// true when all snapshots were regenerated, because of a major version or a downgrade +/// the oldest commit snapshots were rebuilt from, null when nothing was replayed +public record ModelVersionReconcileResult( + int StoredVersion, + int CurrentVersion, + bool FullRegeneration, + Commit? ReplayedFrom) +{ + public bool Changed => StoredVersion != CurrentVersion; +} + +/// +/// The model versions that were applied to the snapshots of a database, stored in the local state. +/// Names aren't stored, so renaming a version doesn't count as a modification. +/// +internal record StoredModelVersions(int Version, StoredModelVersion[] Versions) +{ + public const string LocalStateKey = "harmony:modelVersions"; + public static readonly StoredModelVersions None = new(0, []); + + public static StoredModelVersions From(IReadOnlyList versions) + { + return new StoredModelVersions(versions.Count, + versions.Select(v => new StoredModelVersion(v.Major, v.ChangeTypes.ToArray())).ToArray()); + } +} + +internal record StoredModelVersion(bool Major, string[] ChangeTypes) +{ + public bool SameAs(ModelVersion version) + { + return Major == version.Major && ChangeTypes.SequenceEqual(version.ChangeTypes, StringComparer.Ordinal); + } +} diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs new file mode 100644 index 0000000..7398a35 --- /dev/null +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -0,0 +1,95 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SIL.Harmony.Changes; +using SIL.Harmony.Config; +using SIL.Harmony.Db; + +namespace SIL.Harmony; + +internal class ModelVersionService( + CrdtRepositoryFactory crdtRepositoryFactory, + IOptions crdtConfig, + ILogger logger) +{ + private HarmonyConfig Config => crdtConfig.Value; + + /// + /// Brings the snapshots up to date with the model versions of the current config, see . + /// Changes authored by a newer app are applied with the old code by an older app, or skipped as + /// when the older app doesn't know their type. After an upgrade those changes are replayed, + /// from the oldest commit with a change authored with a version this database didn't have yet. + /// A major version, or a downgrade, regenerates all snapshots. + /// Call this when opening a database, before using it. Querying changes by type is SQLite only. + /// + /// a model version that was already applied to this database was modified + public async Task ReconcileModelVersions() + { + await using var repo = await crdtRepositoryFactory.CreateRepository(); + using var locked = await repo.Lock(); + repo.ClearChangeTracker(); + await using var transaction = await repo.BeginTransactionAsync(); + var stored = await repo.GetLocalState(StoredModelVersions.LocalStateKey) + ?? StoredModelVersions.None; + var current = Config.ModelVersions; + EnsureAppliedVersionsUnchanged(stored, current); + if (stored.Version == current.Count) + return new ModelVersionReconcileResult(stored.Version, current.Count, false, null); + + var fullRegeneration = false; + Commit? replayFrom; + if (current.Count < stored.Version || current.Skip(stored.Version).Any(v => v.Major)) + { + //a downgrade is rare and the snapshots were made by newer code, so start over + fullRegeneration = true; + replayFrom = await repo.CurrentCommits().FirstOrDefaultAsync(); + await SnapshotReplay.RegenerateAll(repo, Config); + } + else + { + replayFrom = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(stored.Version, current)); + if (replayFrom is not null) + { + await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(replayFrom), Config); + } + } + + if (replayFrom is not null) + { + logger.LogInformation( + "Model version changed from {StoredVersion} to {CurrentVersion}, replayed from commit {CommitId}, full regeneration: {FullRegeneration}", + stored.Version, current.Count, replayFrom.Id, fullRegeneration); + } + + await repo.SetLocalState(StoredModelVersions.LocalStateKey, StoredModelVersions.From(current)); + await transaction.CommitAsync(); + return new ModelVersionReconcileResult(stored.Version, current.Count, fullRegeneration, replayFrom); + } + + private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, IReadOnlyList current) + { + var applied = Math.Min(stored.Versions.Length, current.Count); + for (var i = 0; i < applied; i++) + { + if (stored.Versions[i].SameAs(current[i])) continue; + throw new InvalidOperationException( + $"Model version {i + 1} ({current[i].Name}) was modified after it was applied to this database. " + + "Released model versions must not change, add a new version instead."); + } + } + + /// + /// For each change type in the versions after , the version the database had for it. + /// A change of that type with a higher version was authored by a newer app than the one that applied it. + /// + internal static (string ChangeType, int AfterVersion)[] ChangeVersionsToReplay(int storedVersion, + IReadOnlyList versions) + { + var appliedVersions = versions.Take(storedVersion).ToArray(); + return versions.Skip(storedVersion) + .SelectMany(v => v.ChangeTypes) + .Distinct(StringComparer.Ordinal) + .Select(type => (type, appliedVersions.Count(v => v.ChangeTypes.Contains(type, StringComparer.Ordinal)))) + .ToArray(); + } +} diff --git a/src/SIL.Harmony/SnapshotReplay.cs b/src/SIL.Harmony/SnapshotReplay.cs new file mode 100644 index 0000000..9e0232c --- /dev/null +++ b/src/SIL.Harmony/SnapshotReplay.cs @@ -0,0 +1,55 @@ +using SIL.Harmony.Config; +using SIL.Harmony.Db; + +namespace SIL.Harmony; + +/// +/// Rebuilds and persists snapshots by replaying commits. Callers hold the repository lock and a transaction. +/// +internal static class SnapshotReplay +{ + /// + /// Rebuilds every snapshot the window covers by replaying its commits onto the state it resumes from. + /// A window resuming from nothing rebuilds all of history. + /// + public static async Task UpdateSnapshots(CrdtRepository repo, CrdtRepository.ReplayWindow window, HarmonyConfig config) + { + var (checkpoint, commitsToApply) = window; + if (commitsToApply.Count == 0) return; + + ISnapshotView baseline; + // A database with no checkpoints replays all of history, + // which is what we want, because it will trigger creating checkpoints + if (checkpoint is null) + { + await repo.DeleteSnapshotsAndProjectedTables(); + //the delete left the table empty, so there's nothing to query + baseline = EmptySnapshotView.Instance; + } + else + { + await repo.DeleteSnapshotsAfter(checkpoint.Commit); + //the current table is the state at the checkpoint, because the delete above just made it so + baseline = repo.CurrentSnapshotView(); + } + + await PreloadTouched(baseline, commitsToApply, config); + var (newSnapshots, checkpoints) = await SnapshotWorker.ComputeNewSnapshotsAndCheckpoints(baseline, commitsToApply, config); + await repo.AddSnapshots(newSnapshots, checkpoints); + } + + /// Rebuilds every snapshot from all of history. + public static async Task RegenerateAll(CrdtRepository repo, HarmonyConfig config) + { + var wholeHistory = await repo.WholeHistory(); + //Replay does nothing without commits, which would leave snapshots with no history behind them in place + if (wholeHistory.Commits.Count == 0) await repo.DeleteSnapshotsAndProjectedTables(); + else await UpdateSnapshots(repo, wholeHistory, config); + } + + public static async Task PreloadTouched(ISnapshotView baseline, IEnumerable commits, HarmonyConfig config) + { + var entityIds = commits.SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId)).ToHashSet(); + if (entityIds.Count > config.PrefetchSnapshotsBreakpoint) await baseline.PreloadAsync(entityIds); + } +} From e0bd846b75a29781623f6671effa3e644f33f773 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 04:29:06 +0000 Subject: [PATCH 05/25] Rename AfterVersion to StoredChangeVersion The value is the change version a type had at the stored model version, not a version that comes after something. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony/Db/CrdtRepository.cs | 6 +++--- src/SIL.Harmony/ModelVersionService.cs | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 7bc7a00..6a7a95b 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -394,11 +394,11 @@ public async Task ReplayWindowFrom(Commit commit) /// /// The oldest commit with a change whose $type is one of - /// and whose is higher than the version given for that type. + /// and whose is higher than the stored change version given for that type. /// Reads every change, so it's slow, only use it for rare operations. SQLite only. /// public async Task FindOldestCommitWithChangeVersions( - IReadOnlyCollection<(string ChangeType, int AfterVersion)> changeVersions) + IReadOnlyCollection<(string ChangeType, int StoredChangeVersion)> changeVersions) { if (changeVersions.Count == 0) return null; if (!_dbContext.Database.IsSqlite()) @@ -407,7 +407,7 @@ public async Task ReplayWindowFrom(Commit commit) //the key must be quoted in the path, because $ has a special meaning in json paths var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; //one json array parameter, so the parameter count doesn't depend on how many types there are - var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.AfterVersion })); + var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.StoredChangeVersion })); var commitIds = await _dbContext.Database.SqlQuery($""" SELECT c.Id AS Value FROM Commits c WHERE EXISTS ( diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index 7398a35..aac8265 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -79,10 +79,11 @@ private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, I } /// - /// For each change type in the versions after , the version the database had for it. - /// A change of that type with a higher version was authored by a newer app than the one that applied it. + /// For each change type in the versions after , the change version that type had + /// at the stored model version. A change of that type with a higher version was authored by a newer app than the + /// app that applied it. /// - internal static (string ChangeType, int AfterVersion)[] ChangeVersionsToReplay(int storedVersion, + internal static (string ChangeType, int StoredChangeVersion)[] ChangeVersionsToReplay(int storedVersion, IReadOnlyList versions) { var appliedVersions = versions.Take(storedVersion).ToArray(); From e46f01e0cce97ef63aca5d29a397ec030b0c8fe0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 05:05:10 +0000 Subject: [PATCH 06/25] Check that a full regeneration recreates every snapshot Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony.Tests/ModelVersionTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index ffc8b1d..f1a8cc3 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -102,6 +102,15 @@ private static async Task SnapshotIds(DataModelTestBase app) .ToArrayAsync(TestContext.Current.CancellationToken); } + /// a regenerated snapshot is a new row with a new id, so none of the old ids may be left + private static async Task ShouldBeRegenerated(DataModelTestBase app, Guid[] snapshotIdsBefore) + { + snapshotIdsBefore.Should().NotBeEmpty(); + var snapshotIdsAfter = await SnapshotIds(app); + snapshotIdsAfter.Should().HaveSameCount(snapshotIdsBefore); + snapshotIdsAfter.Should().NotIntersectWith(snapshotIdsBefore, "every snapshot should be recreated"); + } + private static async Task ChangeVersions(DataModelTestBase app, Commit commit) { return await app.DbContext.Set>().AsNoTracking() @@ -229,11 +238,13 @@ public async Task MajorVersionRegeneratesAllSnapshots() await WriteNextChange(new NewWordChange(Guid.NewGuid(), "world")); await using var newApp = ForkDatabase(AppWith(versions => versions.AddMajor("Rewrite"))); + var snapshotIds = await SnapshotIds(newApp); var result = await newApp.ModelVersionService.ReconcileModelVersions(); result.FullRegeneration.Should().BeTrue(); result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); result.CurrentVersion.Should().Be(1); + await ShouldBeRegenerated(newApp, snapshotIds); } [Fact] @@ -289,10 +300,12 @@ public async Task DowngradeRegeneratesAllSnapshots() await newApp.WriteNextChange(new SetWordNoteChange(wordId, "a note")); await using var oldApp = newApp.ForkDatabase(OldApp); + var snapshotIds = await SnapshotIds(oldApp); var result = await oldApp.ModelVersionService.ReconcileModelVersions(); result.Should().Be(new ModelVersionReconcileResult(1, 0, true, result.ReplayedFrom)); result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + await ShouldBeRegenerated(oldApp, snapshotIds); (await NoteOf(oldApp, wordId)).Should().Be("a note"); } @@ -311,9 +324,11 @@ public async Task DowngradeToAnAppWithoutAChangeTypeRemovesItsEntities() OldApp(services); services.Configure(config => config.ChangeTypeListBuilder.Remove()); }); + var snapshotIds = await SnapshotIds(oldApp); var result = await oldApp.ModelVersionService.ReconcileModelVersions(); result.FullRegeneration.Should().BeTrue(); + (await SnapshotIds(oldApp)).Should().NotIntersectWith(snapshotIds, "every snapshot should be recreated"); (await oldApp.DataModel.GetLatest(tagId)).Should().BeNull(); (await oldApp.DbContext.Set().AnyAsync(TestContext.Current.CancellationToken)).Should().BeFalse(); (await oldApp.DataModel.GetLatest(wordId)).Should().NotBeNull(); From 7ff7094944b89c413655f5cdc469499314255027 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 05:08:20 +0000 Subject: [PATCH 07/25] Fix setting local state again after it was removed RemoveLocalState used ExecuteDelete, which left the entry tracked when the db context is shared between calls. Setting the same key again then failed with an EF tracking exception. Use a tracked removal instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PUZStknRU1T6iEdRUFTp95 --- src/SIL.Harmony.Tests/ModelVersionTests.cs | 4 ++++ src/SIL.Harmony/Db/CrdtRepository.cs | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index f1a8cc3..5605353 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -446,6 +446,10 @@ public async Task LocalStateCanBeSetReadAndRemoved() await DataModel.RemoveLocalState("app:value"); (await DataModel.GetLocalState("app:value")).Should().BeNull(); await DataModel.RemoveLocalState("app:value"); + + //the test base shares one db context between calls, so a removed entry must not stay tracked + await DataModel.SetLocalState("app:value", new LocalStateTestValue("third", 3)); + (await DataModel.GetLocalState("app:value")).Should().Be(new LocalStateTestValue("third", 3)); } private record LocalStateTestValue(string Text, int Number); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 6a7a95b..dab0c98 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -489,7 +489,12 @@ public async Task SetLocalState(string key, T value) public async Task RemoveLocalState(string key) { - await LocalState.Where(e => e.Key == key).ExecuteDeleteAsync(); + //a tracked removal, not ExecuteDelete: the db context can be shared between calls, and ExecuteDelete would leave + //a tracked entry behind, so setting the key again would fail + var entry = await LocalState.FindAsync(key); + if (entry is null) return; + LocalState.Remove(entry); + await _dbContext.SaveChangesAsync(); } public async Task AddLocalResource(LocalResource localResource) From b7ff51124c23c38fdeb2655c96345ad2f8085cd2 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 8 Oct 2026 16:45:31 +0700 Subject: [PATCH 08/25] Make model and change versions explicit, add invalidateFrom A released model version that forgot to list a change type it modified can't be fixed: its apps author that type with the previous change version, the same number older apps used, so a later version that lists the type never replays them. Model versions are now numbered explicitly (1, 2, 3...) and each listed change type gives its version (one more than the last time it was listed). Change(version, invalidateFrom) replays changes of that type with invalidateFrom or higher on upgrade; it defaults to the version, so only changes from newer apps are replayed. To fix a forgotten type, pass the version its apps authored it with. The numbers and invalidateFrom are stored with the applied versions, so changing either on a released version throws on reconcile. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 +- ...onTests.DescribeModelVersions.verified.txt | 5 +- src/SIL.Harmony.Tests/ModelVersionTests.cs | 156 +++++++++++++++--- src/SIL.Harmony/Config/HarmonyConfig.cs | 16 +- src/SIL.Harmony/Config/ModelVersion.cs | 20 ++- src/SIL.Harmony/Config/ModelVersionBuilder.cs | 83 +++++++--- src/SIL.Harmony/Db/CrdtRepository.cs | 8 +- src/SIL.Harmony/ModelVersionReconciliation.cs | 6 +- src/SIL.Harmony/ModelVersionService.cs | 16 +- 9 files changed, 243 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 932f0a0..ebc98c8 100644 --- a/README.md +++ b/README.md @@ -181,22 +181,30 @@ It's that easy. All the heavy lifting is done by the interface which is fairly s ### Model versions Clients on different app versions sync with each other. When a newer app adds a change type, an older app stores those changes as `OpaqueChange` (with `UnknownChangeHandling.Fallback`) and skips them. When a newer app changes how a change type applies, an older app applies those changes with its old code. In both cases the snapshots must be rebuilt when the older app upgrades. -To make this possible, declare a model version each time a release adds a change type, or changes how one applies. Only add versions at the end, never modify a released version: +To make this possible, declare a model version each time a release adds a change type, or changes how one applies. Number the versions 1, 2, 3 and so on. Each version lists the change types it added or modified, with the type's new version number. A type's version starts at 1 and goes up by one each time it is listed. Only add versions at the end, never modify a released version: ```C# services.AddCrdtData(config => { config.ModelVersionBuilder - .Add("Word notes", v => v.Change()) + .Add(1, "Word notes", v => v.Change(1)) //a major version regenerates all snapshots on upgrade - .AddMajor("Reference rewrite", v => v.Change()); + .AddMajor(2, "Reference rewrite", v => v.Change(1)); }); ``` -Every change is stored with the version of its type when it was authored: how many model versions list that type. Changes from before model versions existed are version 0. +Every change is stored with the version of its type when it was authored: the version the latest model version that lists the type gave it. Changes from before model versions existed, or of types no model version lists, are version 0. -Call `ReconcileModelVersions` when you open a database, before you use it. It compares the model versions stored in the database with the current ones, and replays only the changes that were authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. +Call `ReconcileModelVersions` when you open a database, before you use it. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it replays the changes of that type with the listed version or higher, which are the changes authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. ```C# await dataModel.ReconcileModelVersions(); ``` +If a released version forgot to list a change type that it added or modified, its apps authored those changes with the type's previous version number, the same as the apps before them. To fix it, list the type in a new version with `invalidateFrom` set to that previous number. Upgrading to the new version then replays every change of the type with that number or higher: +```C# +config.ModelVersionBuilder + .Add(1, "Word notes", v => v.Change(1)) + //forgot to list SetTagChange, so its changes are still authored as version 0 + .Add(2, "Tags", v => v.Change(2)) + .Add(3, "Fix tags", v => v.Change(1, invalidateFrom: 0)); +``` To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. > [!NOTE] diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt index d4b6216..a92619e 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt +++ b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt @@ -1,3 +1,4 @@ -1 Notes: SetWordNoteChange +1 Notes: SetWordNoteChange 1 2 major Rewrite: -3 Tags and notes: SetTagChange, SetWordNoteChange +3 Tags and notes: SetTagChange 1, SetWordNoteChange 2 +4 Fix tags: SetTagChange 2 invalidates from 1, SetWordNoteChange 3 diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 0f8121e..1fbef5d 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -36,7 +36,7 @@ private static void NewApp(IServiceCollection services) { OldApp(services); services.Configure(config => - config.ModelVersionBuilder.Add("Notes", v => v.Change())); + config.ModelVersionBuilder.Add(1, "Notes", v => v.Change(1))); } private static Action AppWith(Action versions) @@ -113,8 +113,8 @@ private static async Task ChangeVersions(DataModelTestBase app, Commit co public async Task AuthoredChangesGetTheVersionOfTheirType() { await using var app = ForkDatabase(AppWith(versions => versions - .Add("Notes", v => v.Change()) - .Add("Notes and tags", v => v.Change().Change()))); + .Add(1, "Notes", v => v.Change(1)) + .Add(2, "Notes and tags", v => v.Change(2).Change(1)))); var wordId = Guid.NewGuid(); var commit = await app.WriteNextChange([ @@ -167,6 +167,54 @@ public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() (await NoteOf(newApp, wordId)).Should().Be("newer note"); } + /// the app that added notes was released, but forgot to list them in its model version + private static void AppThatForgotNotes(IServiceCollection services) + { + OldApp(services); + services.Configure(config => + config.ModelVersionBuilder.Add(1, "Tags", v => v.Change(1))); + } + + private static void AppThatFixedForgottenNotes(IServiceCollection services) + { + OldApp(services); + services.Configure(config => config.ModelVersionBuilder + .Add(1, "Tags", v => v.Change(1)) + .Add(2, "Forgotten notes", v => v.Change(1, invalidateFrom: 0))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedToTheAppThatForgot) + { + await using var oldApp = ForkDatabase(OldAppWithoutNotes); + await oldApp.ModelVersionService.ReconcileModelVersions(); + var wordId = Guid.NewGuid(); + await oldApp.WriteNextChange(new NewWordChange(wordId, "hello")); + await using var appThatForgot = oldApp.ForkDatabase(AppThatForgotNotes); + var forgottenVersion = appThatForgot.CrdtConfig.ChangeVersion(NoteChangeType); + forgottenVersion.Should().Be(0, "the app that forgot notes authors them like the old app did"); + var noteCommit = await SyncFromOtherClient(oldApp, OpaqueNote(oldApp, wordId, "a note"), forgottenVersion); + (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); + + var beforeFix = oldApp; + await using var upgraded = upgradedToTheAppThatForgot ? oldApp.ForkDatabase(AppThatForgotNotes) : null; + if (upgraded is not null) + { + await upgraded.ModelVersionService.ReconcileModelVersions(); + (await NoteOf(upgraded, wordId)).Should().BeNull("the app that forgot notes doesn't know to replay them"); + beforeFix = upgraded; + } + + await using var fixedApp = beforeFix.ForkDatabase(AppThatFixedForgottenNotes); + var result = await fixedApp.ModelVersionService.ReconcileModelVersions(); + + (result.ReplayedFrom?.Id).Should().Be(noteCommit.Id); + (await NoteOf(fixedApp, wordId)).Should().Be("a note"); + fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(1, "notes authored after the fix can be told apart"); + } + [Fact] public async Task ReplayResumesFromTheCheckpointBeforeTheChange() { @@ -226,7 +274,7 @@ public async Task MajorVersionRegeneratesAllSnapshots() var firstCommit = await WriteNextChange(new NewWordChange(Guid.NewGuid(), "hello")); await WriteNextChange(new NewWordChange(Guid.NewGuid(), "world")); - await using var newApp = ForkDatabase(AppWith(versions => versions.AddMajor("Rewrite"))); + await using var newApp = ForkDatabase(AppWith(versions => versions.AddMajor(1, "Rewrite"))); var snapshotIds = await SnapshotIds(newApp); var result = await newApp.ModelVersionService.ReconcileModelVersions(); @@ -301,7 +349,7 @@ public async Task DowngradeRegeneratesAllSnapshots() [Fact] public async Task DowngradeToAnAppWithoutAChangeTypeRemovesItsEntities() { - await using var newApp = ForkDatabase(AppWith(versions => versions.Add("Tags", v => v.Change()))); + await using var newApp = ForkDatabase(AppWith(versions => versions.Add(1, "Tags", v => v.Change(1)))); await newApp.ModelVersionService.ReconcileModelVersions(); var wordId = Guid.NewGuid(); var tagId = Guid.NewGuid(); @@ -329,15 +377,19 @@ public async Task ModifiedAppliedVersionThrows() await using var newApp = ForkDatabase(NewApp); await newApp.ModelVersionService.ReconcileModelVersions(); - await using var otherTypes = newApp.ForkDatabase(AppWith(versions => versions.Add("Notes", v => v.Change()))); + await using var otherTypes = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change(1)))); var act = () => otherTypes.ModelVersionService.ReconcileModelVersions(); await act.Should().ThrowAsync().WithMessage("Model version 1 (Notes) was modified*"); - await using var nowMajor = newApp.ForkDatabase(AppWith(versions => versions.AddMajor("Notes", v => v.Change()))); + await using var nowMajor = newApp.ForkDatabase(AppWith(versions => versions.AddMajor(1, "Notes", v => v.Change(1)))); act = () => nowMajor.ModelVersionService.ReconcileModelVersions(); await act.Should().ThrowAsync(); - await using var renamed = newApp.ForkDatabase(AppWith(versions => versions.Add("Word notes", v => v.Change()))); + await using var nowInvalidates = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change(1, invalidateFrom: 0)))); + act = () => nowInvalidates.ModelVersionService.ReconcileModelVersions(); + await act.Should().ThrowAsync(); + + await using var renamed = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Word notes", v => v.Change(1)))); (await renamed.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeFalse("names are not stored"); } @@ -345,9 +397,10 @@ public async Task ModifiedAppliedVersionThrows() public async Task DescribeModelVersions() { await using var app = ForkDatabase(AppWith(versions => versions - .Add("Notes", v => v.Change()) - .AddMajor("Rewrite") - .Add("Tags and notes", v => v.Change().Change()))); + .Add(1, "Notes", v => v.Change(1)) + .AddMajor(2, "Rewrite") + .Add(3, "Tags and notes", v => v.Change(2).Change(1)) + .Add(4, "Fix tags", v => v.Change(2, invalidateFrom: 1).Change(3)))); await Verify(app.CrdtConfig.DescribeModelVersions()); } @@ -360,7 +413,7 @@ public void ModelVersionWithAnUnregisteredChangeTypeFailsValidation() .Configure(config => { config.ChangeTypeListBuilder.Remove(); - config.ModelVersionBuilder.Add("Notes", v => v.Change()); + config.ModelVersionBuilder.Add(1, "Notes", v => v.Change(1)); }) .BuildServiceProvider(); @@ -370,22 +423,71 @@ public void ModelVersionWithAnUnregisteredChangeTypeFailsValidation() } [Fact] - public void ChangeVersionsToReplayAreTheVersionsBeforeTheUpgrade() + public void ChangeVersionsToReplayStartAtTheFirstNewVersion() { - ModelVersion[] versions = - [ - new("A", false, [NoteChangeType]), - new("B", false, [nameof(SetTagChange), NoteChangeType]), - new("C", false, [nameof(SetTagChange)]), - ]; + var versions = new ModelVersionBuilder() + .Add(1, "A", v => v.Change(1)) + .Add(2, "B", v => v.Change(1).Change(2)) + .Add(3, "C", v => v.Change(2)) + .Versions; ModelVersionService.ChangeVersionsToReplay(1, versions) - .Should().BeEquivalentTo([(nameof(SetTagChange), 0), (NoteChangeType, 1)]); + .Should().BeEquivalentTo([(nameof(SetTagChange), 1), (NoteChangeType, 2)]); ModelVersionService.ChangeVersionsToReplay(0, versions) - .Should().BeEquivalentTo([(NoteChangeType, 0), (nameof(SetTagChange), 0)]); + .Should().BeEquivalentTo([(NoteChangeType, 1), (nameof(SetTagChange), 1)]); ModelVersionService.ChangeVersionsToReplay(3, versions).Should().BeEmpty(); } + [Fact] + public void ChangeVersionsToReplayStartAtTheInvalidatedVersion() + { + var versions = new ModelVersionBuilder() + .Add(1, "A", v => v.Change(1)) + //forgot to list tags + .Add(2, "B", v => v.Change(2)) + .Add(3, "C", v => v.Change(1, invalidateFrom: 0).Change(3)) + .Add(4, "D", v => v.Change(2)) + .Versions; + + ModelVersionService.ChangeVersionsToReplay(0, versions) + .Should().BeEquivalentTo([(NoteChangeType, 1), (nameof(SetTagChange), 0)]); + ModelVersionService.ChangeVersionsToReplay(2, versions) + .Should().BeEquivalentTo([(NoteChangeType, 3), (nameof(SetTagChange), 0)], + "the apps with versions A and B authored tags with version 0"); + ModelVersionService.ChangeVersionsToReplay(3, versions) + .Should().BeEquivalentTo([(nameof(SetTagChange), 2)], "the fix was already applied"); + } + + [Fact] + public void ModelVersionsMustBeNumberedInOrder() + { + var builder = new ModelVersionBuilder().Add(1, "A", v => v.Change(1)); + + builder.Invoking(b => b.Add(3, "C", _ => { })).Should().Throw(); + builder.Invoking(b => b.AddMajor(1, "A again")).Should().Throw(); + builder.Versions.Should().ContainSingle(); + } + + [Fact] + public void ChangeVersionsMustBeValid() + { + new ModelVersionBuilder().Invoking(b => b.Add(1, "A", v => v.Change(2))) + .Should().Throw().WithMessage("*must be 1, one more than its previous version 0"); + var builder = new ModelVersionBuilder().Add(1, "A", v => v.Change(1).Change(1)); + + builder.Invoking(b => b.Add(2, "B", v => v.Change(1))) + .Should().Throw().WithMessage("*must be 2, one more than its previous version 1"); + builder.Invoking(b => b.Add(2, "B", v => v.Change(3))) + .Should().Throw().WithMessage("*must be 2, one more than its previous version 1"); + builder.Invoking(b => b.Add(2, "B", v => v.Change(0))).Should().Throw(); + builder.Invoking(b => b.Add(2, "B", v => v.Change(2, invalidateFrom: 3))).Should().Throw(); + builder.Invoking(b => b.Add(2, "B", v => v.Change(2, invalidateFrom: -1))).Should().Throw(); + builder.Invoking(b => b.Add(2, "B", v => v.Change(2).Change(2))) + .Should().Throw().WithMessage("*more than once*"); + builder.Versions.Should().ContainSingle("a version that failed validation isn't added"); + builder.Add(2, "B", v => v.Change(2)).Versions.Should().HaveCount(2); + } + [Fact] public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() { @@ -401,12 +503,12 @@ public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() await AddCommitsViaSync([earlierNote]); await using var repo = CreateRepository(); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 0)]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, -1)]))!.Id.Should().Be(versionZeroNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 1)])).Should().BeNull(); - (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 1)]))!.Id.Should().Be(tagCommit.Id); - (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 1), (NoteChangeType, 0)]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(nameof(EditExampleChange), 0)])).Should().BeNull(); + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 0)]))!.Id.Should().Be(versionZeroNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 2)])).Should().BeNull(); + (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 2)]))!.Id.Should().Be(tagCommit.Id); + (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 2), (NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithChangeVersions([(nameof(EditExampleChange), 1)])).Should().BeNull(); (await repo.FindOldestCommitWithChangeVersions([])).Should().BeNull(); } diff --git a/src/SIL.Harmony/Config/HarmonyConfig.cs b/src/SIL.Harmony/Config/HarmonyConfig.cs index 739214d..1902c6b 100644 --- a/src/SIL.Harmony/Config/HarmonyConfig.cs +++ b/src/SIL.Harmony/Config/HarmonyConfig.cs @@ -129,13 +129,12 @@ private ChangeDiscriminatorMaps BuildChangeDiscriminatorMaps() private FrozenDictionary BuildChangeVersions() { ModelVersionBuilder.Freeze(); - return ModelVersions.SelectMany(v => v.ChangeTypes) - .CountBy(t => t, StringComparer.Ordinal) - .ToFrozenDictionary(StringComparer.Ordinal); + return ModelVersionBuilder.ChangeVersions.ToFrozenDictionary(StringComparer.Ordinal); } /// - /// The version a change of this type is authored with: how many model versions list the type. + /// The version a change of this type is authored with: the version the latest model version that lists the type gave it, + /// 0 when none lists it. /// /// the change type's discriminator, its $type public int ChangeVersion(string discriminator) @@ -158,14 +157,15 @@ internal int ChangeVersion(IChange change) public string DescribeModelVersions() { var builder = new StringBuilder(); - for (var i = 0; i < ModelVersions.Count; i++) + foreach (var version in ModelVersions) { - var version = ModelVersions[i]; - builder.Append(i + 1) + builder.Append(version.Number) .Append(version.Major ? " major " : " ") .Append(version.Name) .Append(": ") - .AppendJoin(", ", version.ChangeTypes) + .AppendJoin(", ", version.Changes.Select(c => c.InvalidateFrom == c.Version + ? $"{c.ChangeType} {c.Version}" + : $"{c.ChangeType} {c.Version} invalidates from {c.InvalidateFrom}")) .Append('\n'); } return builder.ToString(); diff --git a/src/SIL.Harmony/Config/ModelVersion.cs b/src/SIL.Harmony/Config/ModelVersion.cs index 8d9e03e..98ff82c 100644 --- a/src/SIL.Harmony/Config/ModelVersion.cs +++ b/src/SIL.Harmony/Config/ModelVersion.cs @@ -4,19 +4,29 @@ namespace SIL.Harmony.Config; /// One step in the ordered list of model versions an app declares on . /// Released versions must never be modified, only new versions added at the end. /// +/// the position of the version in the list, starting at 1 /// describes the version for people, it is not stored in the database /// when true, upgrading to this version regenerates all snapshots -/// discriminators of the change types this version added or modified, sorted -public sealed record ModelVersion(string Name, bool Major, IReadOnlyList ChangeTypes) +/// the change types this version added or modified, sorted by type +public sealed record ModelVersion(int Number, string Name, bool Major, IReadOnlyList Changes) { public bool Equals(ModelVersion? other) { - return other is not null && Name == other.Name && Major == other.Major - && ChangeTypes.SequenceEqual(other.ChangeTypes, StringComparer.Ordinal); + return other is not null && Number == other.Number && Name == other.Name && Major == other.Major + && Changes.SequenceEqual(other.Changes); } public override int GetHashCode() { - return HashCode.Combine(Name, Major, ChangeTypes.Count); + return HashCode.Combine(Number, Name, Major, Changes.Count); } } + +/// a change type that a model version added or modified +/// the discriminator of the change type, its $type +/// the version changes of this type are authored with, from this model version on +/// +/// on upgrade, changes of this type with this version or higher are replayed. +/// Usually , lower when an earlier model version forgot to list the type. +/// +public sealed record ModelVersionChange(string ChangeType, int Version, int InvalidateFrom); diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs index 6ca07c1..d301107 100644 --- a/src/SIL.Harmony/Config/ModelVersionBuilder.cs +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -14,39 +14,61 @@ public class ModelVersionBuilder private bool _frozen; private readonly List _versions = []; private readonly HashSet _changeTypes = []; + private readonly Dictionary _changeVersions = new(StringComparer.Ordinal); public IReadOnlyList Versions => _versions.AsReadOnly(); - /// change types added with , which must be registered + /// change types added with , which must be registered internal IReadOnlyCollection ChangeTypes => _changeTypes; + /// the latest version of each change type listed by a model version + internal IReadOnlyDictionary ChangeVersions => _changeVersions; + /// - /// Adds a version. Changes of the listed types that were authored with this version or later - /// are replayed when upgrading to it. + /// Adds a version. Changes of the listed types are replayed when upgrading to it, + /// see for which ones. /// - public ModelVersionBuilder Add(string name, Action changeTypes) + /// the next number: versions are numbered 1, 2, 3 and so on + public ModelVersionBuilder Add(int number, string name, Action changes) { - return Add(name, false, changeTypes); + return Add(number, name, false, changes); } /// /// Adds a version that regenerates all snapshots when upgrading to it. - /// The listed change types still get a new version number for the changes authored with them. + /// The listed change types are still authored with their new version. /// - public ModelVersionBuilder AddMajor(string name, Action? changeTypes = null) + /// the next number: versions are numbered 1, 2, 3 and so on + public ModelVersionBuilder AddMajor(int number, string name, Action? changes = null) { - return Add(name, true, changeTypes); + return Add(number, name, true, changes); } - private ModelVersionBuilder Add(string name, bool major, Action? configure) + private ModelVersionBuilder Add(int number, string name, bool major, Action? configure) { CheckFrozen(); ArgumentException.ThrowIfNullOrWhiteSpace(name); - var changeTypes = new ModelVersionChangeTypes(); - configure?.Invoke(changeTypes); - _changeTypes.UnionWith(changeTypes.Types); - _versions.Add(new ModelVersion(name, major, - changeTypes.Discriminators.Order(StringComparer.Ordinal).ToArray())); + var expectedNumber = _versions.Count + 1; + if (number != expectedNumber) + throw new ArgumentOutOfRangeException(nameof(number), number, + $"Model version {name} must be number {expectedNumber}, versions are numbered 1, 2, 3 and so on"); + var changes = new ModelVersionChanges(); + configure?.Invoke(changes); + foreach (var change in changes.Changes.Values) + { + var previous = _changeVersions.GetValueOrDefault(change.ChangeType); + if (change.Version != previous + 1) + throw new ArgumentException( + $"{change.ChangeType} version {change.Version} in model version {number} ({name}) must be {previous + 1}, one more than its previous version {previous}"); + } + + foreach (var change in changes.Changes.Values) + { + _changeVersions[change.ChangeType] = change.Version; + } + _changeTypes.UnionWith(changes.Types); + _versions.Add(new ModelVersion(number, name, major, + changes.Changes.Values.OrderBy(c => c.ChangeType, StringComparer.Ordinal).ToArray())); return this; } @@ -61,26 +83,43 @@ private void CheckFrozen() } } -public class ModelVersionChangeTypes +public class ModelVersionChanges { - internal HashSet Discriminators { get; } = new(StringComparer.Ordinal); + internal Dictionary Changes { get; } = new(StringComparer.Ordinal); internal List Types { get; } = []; - /// the change type was added or modified in this version - public ModelVersionChangeTypes Change() where T : IChange, IPolyType + /// + /// The change type was added or modified in this version. + /// + /// + /// the version changes of this type are authored with from now on: one more than its previous version, so 1 the first time it is listed. + /// Changes from before model versions existed are version 0. + /// + /// + /// on upgrade, changes of this type with this version or higher are replayed. Defaults to , + /// so only changes authored by this app or a newer one are replayed. When an earlier, released version added or + /// modified the type but forgot to list it, pass the version its apps authored the type with, to replay those too. + /// + public ModelVersionChanges Change(int version, int? invalidateFrom = null) where T : IChange, IPolyType { - if (Discriminators.Add(T.TypeName)) Types.Add(typeof(T)); + Change(T.TypeName, version, invalidateFrom); + Types.Add(typeof(T)); return this; } /// - /// The change type with this discriminator was added or modified in this version. + /// by discriminator. /// Only for change types that no longer exist in the code, use otherwise. /// - public ModelVersionChangeTypes Change(string discriminator) + public ModelVersionChanges Change(string discriminator, int version, int? invalidateFrom = null) { ArgumentException.ThrowIfNullOrWhiteSpace(discriminator); - Discriminators.Add(discriminator); + ArgumentOutOfRangeException.ThrowIfLessThan(version, 1); + var from = invalidateFrom ?? version; + ArgumentOutOfRangeException.ThrowIfNegative(from, nameof(invalidateFrom)); + ArgumentOutOfRangeException.ThrowIfGreaterThan(from, version, nameof(invalidateFrom)); + if (!Changes.TryAdd(discriminator, new ModelVersionChange(discriminator, version, from))) + throw new ArgumentException($"{discriminator} is listed more than once in the same model version"); return this; } } diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index dab0c98..ce73d41 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -394,11 +394,11 @@ public async Task ReplayWindowFrom(Commit commit) /// /// The oldest commit with a change whose $type is one of - /// and whose is higher than the stored change version given for that type. + /// and whose is at least the version given for that type. /// Reads every change, so it's slow, only use it for rare operations. SQLite only. /// public async Task FindOldestCommitWithChangeVersions( - IReadOnlyCollection<(string ChangeType, int StoredChangeVersion)> changeVersions) + IReadOnlyCollection<(string ChangeType, int FromVersion)> changeVersions) { if (changeVersions.Count == 0) return null; if (!_dbContext.Database.IsSqlite()) @@ -407,14 +407,14 @@ public async Task ReplayWindowFrom(Commit commit) //the key must be quoted in the path, because $ has a special meaning in json paths var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; //one json array parameter, so the parameter count doesn't depend on how many types there are - var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.StoredChangeVersion })); + var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.FromVersion })); var commitIds = await _dbContext.Database.SqlQuery($""" SELECT c.Id AS Value FROM Commits c WHERE EXISTS ( SELECT 1 FROM ChangeEntities ce, json_each({changeVersionsJson}) cv WHERE ce.CommitId = c.Id AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(cv.value, '$.t') - AND ce.Version > json_extract(cv.value, '$.v') + AND ce.Version >= json_extract(cv.value, '$.v') ) ORDER BY c.DateTime, c.Counter, c.Id LIMIT 1 diff --git a/src/SIL.Harmony/ModelVersionReconciliation.cs b/src/SIL.Harmony/ModelVersionReconciliation.cs index 43e6706..a839cd9 100644 --- a/src/SIL.Harmony/ModelVersionReconciliation.cs +++ b/src/SIL.Harmony/ModelVersionReconciliation.cs @@ -27,14 +27,14 @@ internal record StoredModelVersions(int Version, StoredModelVersion[] Versions) public static StoredModelVersions From(IReadOnlyList versions) { return new StoredModelVersions(versions.Count, - versions.Select(v => new StoredModelVersion(v.Major, v.ChangeTypes.ToArray())).ToArray()); + versions.Select(v => new StoredModelVersion(v.Major, v.Changes.ToArray())).ToArray()); } } -internal record StoredModelVersion(bool Major, string[] ChangeTypes) +internal record StoredModelVersion(bool Major, ModelVersionChange[] Changes) { public bool SameAs(ModelVersion version) { - return Major == version.Major && ChangeTypes.SequenceEqual(version.ChangeTypes, StringComparer.Ordinal); + return Major == version.Major && Changes.SequenceEqual(version.Changes); } } diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index aac8265..d0822f2 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -18,7 +18,7 @@ internal class ModelVersionService( /// Brings the snapshots up to date with the model versions of the current config, see . /// Changes authored by a newer app are applied with the old code by an older app, or skipped as /// when the older app doesn't know their type. After an upgrade those changes are replayed, - /// from the oldest commit with a change authored with a version this database didn't have yet. + /// from the oldest commit with a change whose version a new model version invalidates. /// A major version, or a downgrade, regenerates all snapshots. /// Call this when opening a database, before using it. Querying changes by type is SQLite only. /// @@ -79,18 +79,16 @@ private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, I } /// - /// For each change type in the versions after , the change version that type had - /// at the stored model version. A change of that type with a higher version was authored by a newer app than the - /// app that applied it. + /// For each change type in the versions after , the lowest change version to replay: + /// the lowest those versions give the type. /// - internal static (string ChangeType, int StoredChangeVersion)[] ChangeVersionsToReplay(int storedVersion, + internal static (string ChangeType, int FromVersion)[] ChangeVersionsToReplay(int storedVersion, IReadOnlyList versions) { - var appliedVersions = versions.Take(storedVersion).ToArray(); return versions.Skip(storedVersion) - .SelectMany(v => v.ChangeTypes) - .Distinct(StringComparer.Ordinal) - .Select(type => (type, appliedVersions.Count(v => v.ChangeTypes.Contains(type, StringComparer.Ordinal)))) + .SelectMany(v => v.Changes) + .GroupBy(c => c.ChangeType, StringComparer.Ordinal) + .Select(changes => (changes.Key, changes.Min(c => c.InvalidateFrom))) .ToArray(); } } From 8b115de52e9d87f0125aca17cdf9d528f833a878 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 12:24:30 +0200 Subject: [PATCH 09/25] Rename ChangeEntity.Version to ChangeVersion Version alone reads like the model version. The name is the JSON property on synced commits and the column name, so it has to be settled before the first release. It matches HarmonyConfig.ChangeVersion and the "change version" wording the rest of the feature already uses. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/SIL.Harmony.Core/ChangeEntity.cs | 2 +- .../DbContextTests.VerifyModel.verified.txt | 4 ++-- src/SIL.Harmony.Tests/ModelVersionTests.cs | 10 +++++----- src/SIL.Harmony/DataModel.cs | 2 +- src/SIL.Harmony/Db/CrdtRepository.cs | 4 ++-- src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ebc98c8..fe75930 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ config.ModelVersionBuilder To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. > [!NOTE] -> Model versions add a `Version` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. +> Model versions add a `ChangeVersion` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. ## Development diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index c33199a..6172eca 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -17,5 +17,5 @@ public ChangeEntity() /// The version of the change type when the change was authored: how many model versions list its type. /// Changes from before model versions existed are version 0. /// - public int Version { get; set; } + public int ChangeVersion { get; set; } } diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index fb1ce1d..2a16464 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -39,10 +39,10 @@ Change (IChange) Annotations: Relational:ColumnType: jsonb - EntityId (Guid) Required - Version (int) Required ValueGenerated.OnAdd + ChangeVersion (int) Required ValueGenerated.OnAdd Annotations: Relational:DefaultValue: 0 + EntityId (Guid) Required Keys: CommitId, Index PK Foreign keys: diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 1fbef5d..0857fdc 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -68,7 +68,7 @@ private static IChange OpaqueNote(DataModelTestBase oldApp, Guid wordId, string private static async Task SyncFromOtherClient(DataModelTestBase app, IChange change, int version) { var commit = BuildCommit(Guid.NewGuid(), app.NextDate(), [change]); - commit.ChangeEntities.Single().Version = version; + commit.ChangeEntities.Single().ChangeVersion = version; await app.AddCommitsViaSync([commit]); return commit; } @@ -105,7 +105,7 @@ private static async Task ChangeVersions(DataModelTestBase app, Commit co return await app.DbContext.Set>().AsNoTracking() .Where(ce => ce.CommitId == commit.Id) .OrderBy(ce => ce.Index) - .Select(ce => ce.Version) + .Select(ce => ce.ChangeVersion) .ToArrayAsync(TestContext.Current.CancellationToken); } @@ -123,11 +123,11 @@ public async Task AuthoredChangesGetTheVersionOfTheirType() new SetWordNoteChange(wordId, "note") ]); - commit.ChangeEntities.Select(ce => ce.Version).Should().Equal(0, 1, 2); + commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 1, 2); (await ChangeVersions(app, commit)).Should().Equal(0, 1, 2); var json = JsonSerializer.Serialize(commit, app.CrdtConfig.JsonSerializerOptions); JsonSerializer.Deserialize(json, app.CrdtConfig.JsonSerializerOptions)! - .ChangeEntities.Select(ce => ce.Version).Should().Equal(0, 1, 2); + .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 1, 2); } [Fact] @@ -499,7 +499,7 @@ public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() var tagCommit = await SyncFromOtherClient(this, new SetTagChange(tagId, "tag"), version: 2); //added last, but sorts before the other version 1 note var earlierNote = CommitBefore(laterNote, new SetWordNoteChange(wordId, "earlier")); - earlierNote.ChangeEntities.Single().Version = 1; + earlierNote.ChangeEntities.Single().ChangeVersion = 1; await AddCommitsViaSync([earlierNote]); await using var repo = CreateRepository(); diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 127302a..9362289 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -130,7 +130,7 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, CommitId = commitId, EntityId = change.EntityId, Index = index, - Version = config.ChangeVersion(change) + ChangeVersion = config.ChangeVersion(change) }; } diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index ce73d41..6c52a5c 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -394,7 +394,7 @@ public async Task ReplayWindowFrom(Commit commit) /// /// The oldest commit with a change whose $type is one of - /// and whose is at least the version given for that type. + /// and whose is at least the version given for that type. /// Reads every change, so it's slow, only use it for rare operations. SQLite only. /// public async Task FindOldestCommitWithChangeVersions( @@ -414,7 +414,7 @@ WHERE EXISTS ( SELECT 1 FROM ChangeEntities ce, json_each({changeVersionsJson}) cv WHERE ce.CommitId = c.Id AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(cv.value, '$.t') - AND ce.Version >= json_extract(cv.value, '$.v') + AND ce.ChangeVersion >= json_extract(cv.value, '$.v') ) ORDER BY c.DateTime, c.Counter, c.Id LIMIT 1 diff --git a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs index b8aa259..67e7df7 100644 --- a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs +++ b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs @@ -12,7 +12,7 @@ public void Configure(EntityTypeBuilder> builder) { builder.ToTable("ChangeEntities"); builder.HasKey(c => new { c.CommitId, c.Index }); - builder.Property(c => c.Version).HasDefaultValue(0); + builder.Property(c => c.ChangeVersion).HasDefaultValue(0); builder.Property(c => c.Change) .HasColumnType("jsonb") .HasConversion( From defce9574694a3ab726525bdb593a2981dfc639b Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 15:22:03 +0200 Subject: [PATCH 10/25] Move the PreloadTouched summary back onto PreloadTouched It was left on GetSyncState when PreloadTouched moved to SnapshotReplay. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony/DataModel.cs | 1 - src/SIL.Harmony/SnapshotReplay.cs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 9362289..655afa8 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -405,7 +405,6 @@ private async Task GetAtCommit(Commit commit, Guid entityId, CrdtRepositor return (repo.SnapshotViewAsOf(checkpoint), commitsToReplay); } - /// one query for every entity the commits touch beats a lookup per entity only for large batches public async Task GetSyncState() { await using var repo = await _crdtRepositoryFactory.CreateRepository(); diff --git a/src/SIL.Harmony/SnapshotReplay.cs b/src/SIL.Harmony/SnapshotReplay.cs index 9e0232c..0607ff8 100644 --- a/src/SIL.Harmony/SnapshotReplay.cs +++ b/src/SIL.Harmony/SnapshotReplay.cs @@ -47,6 +47,7 @@ public static async Task RegenerateAll(CrdtRepository repo, HarmonyConfig config else await UpdateSnapshots(repo, wholeHistory, config); } + /// one query for every entity the commits touch beats a lookup per entity only for large batches public static async Task PreloadTouched(ISnapshotView baseline, IEnumerable commits, HarmonyConfig config) { var entityIds = commits.SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId)).ToHashSet(); From d70be42c0a59c527ad98a6fd837645bbc7bf2ce1 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 15:22:20 +0200 Subject: [PATCH 11/25] Compare model versions by number, not list count ReconcileModelVersions compared the stored version with ModelVersions.Count. HarmonyConfig.CurrentModelVersion already names that number. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony/ModelVersionService.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index d0822f2..95f2c89 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -32,13 +32,14 @@ public async Task ReconcileModelVersions() var stored = await repo.GetLocalState(StoredModelVersions.LocalStateKey) ?? StoredModelVersions.None; var current = Config.ModelVersions; + var currentVersion = Config.CurrentModelVersion; EnsureAppliedVersionsUnchanged(stored, current); - if (stored.Version == current.Count) - return new ModelVersionReconcileResult(stored.Version, current.Count, false, null); + if (stored.Version == currentVersion) + return new ModelVersionReconcileResult(stored.Version, currentVersion, false, null); var fullRegeneration = false; Commit? replayFrom; - if (current.Count < stored.Version || current.Skip(stored.Version).Any(v => v.Major)) + if (currentVersion < stored.Version || current.Skip(stored.Version).Any(v => v.Major)) { //a downgrade is rare and the snapshots were made by newer code, so start over fullRegeneration = true; @@ -58,12 +59,12 @@ public async Task ReconcileModelVersions() { logger.LogInformation( "Model version changed from {StoredVersion} to {CurrentVersion}, replayed from commit {CommitId}, full regeneration: {FullRegeneration}", - stored.Version, current.Count, replayFrom.Id, fullRegeneration); + stored.Version, currentVersion, replayFrom.Id, fullRegeneration); } await repo.SetLocalState(StoredModelVersions.LocalStateKey, StoredModelVersions.From(current)); await transaction.CommitAsync(); - return new ModelVersionReconcileResult(stored.Version, current.Count, fullRegeneration, replayFrom); + return new ModelVersionReconcileResult(stored.Version, currentVersion, fullRegeneration, replayFrom); } private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, IReadOnlyList current) From caa7079e9c1485c3a3ac0da8a85566b3e20056df Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 15:22:36 +0200 Subject: [PATCH 12/25] Rename ReplayedFrom to OldestInvalidatedCommit The replay starts at the checkpoint before that commit, not at it. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Tests/ModelVersionTests.cs | 16 ++++++++-------- src/SIL.Harmony/ModelVersionReconciliation.cs | 7 +++++-- src/SIL.Harmony/ModelVersionService.cs | 18 +++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 0857fdc..6e62b1d 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -143,8 +143,8 @@ public async Task UnknownTypeIsAppliedAfterUpgrade() await using var newApp = oldApp.ForkDatabase(NewApp); var result = await newApp.ModelVersionService.ReconcileModelVersions(); - result.Should().Be(new ModelVersionReconcileResult(0, 1, false, result.ReplayedFrom)); - result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + result.Should().Be(new ModelVersionReconcileResult(0, 1, false, result.OldestInvalidatedCommit)); + result.OldestInvalidatedCommit!.Id.Should().Be(noteCommit.Id); (await NoteOf(newApp, wordId)).Should().Be("a note"); } @@ -161,7 +161,7 @@ public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() await using var newApp = ForkDatabase(NewApp); var result = await newApp.ModelVersionService.ReconcileModelVersions(); - result.ReplayedFrom!.Id.Should().Be(newerNote.Id, + result.OldestInvalidatedCommit!.Id.Should().Be(newerNote.Id, "the old note was authored and applied by the old app, so it doesn't need a replay"); result.FullRegeneration.Should().BeFalse(); (await NoteOf(newApp, wordId)).Should().Be("newer note"); @@ -210,7 +210,7 @@ public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedTo await using var fixedApp = beforeFix.ForkDatabase(AppThatFixedForgottenNotes); var result = await fixedApp.ModelVersionService.ReconcileModelVersions(); - (result.ReplayedFrom?.Id).Should().Be(noteCommit.Id); + (result.OldestInvalidatedCommit?.Id).Should().Be(noteCommit.Id); (await NoteOf(fixedApp, wordId)).Should().Be("a note"); fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(1, "notes authored after the fix can be told apart"); } @@ -254,7 +254,7 @@ static void SmallCheckpoints(IServiceCollection services) => var result = await newApp.ModelVersionService.ReconcileModelVersions(); - result.ReplayedFrom!.Id.Should().Be(noteCommit.Id); + result.OldestInvalidatedCommit!.Id.Should().Be(noteCommit.Id); (await SnapshotIds(newApp)).Should().Contain(snapshotsUpToCheckpoint, "snapshots before the checkpoint should not be rebuilt"); var word = await newApp.DataModel.GetLatest(wordId); word!.Note.Should().Be("a note"); @@ -279,7 +279,7 @@ public async Task MajorVersionRegeneratesAllSnapshots() var result = await newApp.ModelVersionService.ReconcileModelVersions(); result.FullRegeneration.Should().BeTrue(); - result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + result.OldestInvalidatedCommit!.Id.Should().Be(firstCommit.Id); result.CurrentVersion.Should().Be(1); await ShouldBeRegenerated(newApp, snapshotIds); } @@ -340,8 +340,8 @@ public async Task DowngradeRegeneratesAllSnapshots() var snapshotIds = await SnapshotIds(oldApp); var result = await oldApp.ModelVersionService.ReconcileModelVersions(); - result.Should().Be(new ModelVersionReconcileResult(1, 0, true, result.ReplayedFrom)); - result.ReplayedFrom!.Id.Should().Be(firstCommit.Id); + result.Should().Be(new ModelVersionReconcileResult(1, 0, true, result.OldestInvalidatedCommit)); + result.OldestInvalidatedCommit!.Id.Should().Be(firstCommit.Id); await ShouldBeRegenerated(oldApp, snapshotIds); (await NoteOf(oldApp, wordId)).Should().Be("a note"); } diff --git a/src/SIL.Harmony/ModelVersionReconciliation.cs b/src/SIL.Harmony/ModelVersionReconciliation.cs index a839cd9..8546e59 100644 --- a/src/SIL.Harmony/ModelVersionReconciliation.cs +++ b/src/SIL.Harmony/ModelVersionReconciliation.cs @@ -5,12 +5,15 @@ namespace SIL.Harmony; /// the model version stored in the database before the call, 0 when none was stored /// the model version of the current config, which is now stored /// true when all snapshots were regenerated, because of a major version or a downgrade -/// the oldest commit snapshots were rebuilt from, null when nothing was replayed +/// +/// the oldest commit with a change the upgrade invalidated, snapshots were rebuilt from the checkpoint before it. +/// The first commit on a full regeneration, null when nothing was replayed +/// public record ModelVersionReconcileResult( int StoredVersion, int CurrentVersion, bool FullRegeneration, - Commit? ReplayedFrom) + Commit? OldestInvalidatedCommit) { public bool Changed => StoredVersion != CurrentVersion; } diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index 95f2c89..a9de5a4 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -38,33 +38,33 @@ public async Task ReconcileModelVersions() return new ModelVersionReconcileResult(stored.Version, currentVersion, false, null); var fullRegeneration = false; - Commit? replayFrom; + Commit? oldestInvalidated; if (currentVersion < stored.Version || current.Skip(stored.Version).Any(v => v.Major)) { //a downgrade is rare and the snapshots were made by newer code, so start over fullRegeneration = true; - replayFrom = await repo.CurrentCommits().FirstOrDefaultAsync(); + oldestInvalidated = await repo.CurrentCommits().FirstOrDefaultAsync(); await SnapshotReplay.RegenerateAll(repo, Config); } else { - replayFrom = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(stored.Version, current)); - if (replayFrom is not null) + oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(stored.Version, current)); + if (oldestInvalidated is not null) { - await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(replayFrom), Config); + await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(oldestInvalidated), Config); } } - if (replayFrom is not null) + if (oldestInvalidated is not null) { logger.LogInformation( - "Model version changed from {StoredVersion} to {CurrentVersion}, replayed from commit {CommitId}, full regeneration: {FullRegeneration}", - stored.Version, currentVersion, replayFrom.Id, fullRegeneration); + "Model version changed from {StoredVersion} to {CurrentVersion}, oldest invalidated commit {CommitId}, full regeneration: {FullRegeneration}", + stored.Version, currentVersion, oldestInvalidated.Id, fullRegeneration); } await repo.SetLocalState(StoredModelVersions.LocalStateKey, StoredModelVersions.From(current)); await transaction.CommitAsync(); - return new ModelVersionReconcileResult(stored.Version, currentVersion, fullRegeneration, replayFrom); + return new ModelVersionReconcileResult(stored.Version, currentVersion, fullRegeneration, oldestInvalidated); } private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, IReadOnlyList current) From 4756e8c01c18c5bd2a5b2feee308672892725770 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 15:23:16 +0200 Subject: [PATCH 13/25] Show the old app applies a newer app's change before upgrading Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Tests/ModelVersionTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 6e62b1d..74b4703 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -157,6 +157,7 @@ public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() var oldNote = await WriteNextChange(new SetWordNoteChange(wordId, "old note")); var newerNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "newer note"), version: 1); (await ChangeVersions(this, oldNote)).Should().Equal(0); + (await NoteOf(this, wordId)).Should().Be("newer note", "the old app applies a newer app's change with its own code"); await using var newApp = ForkDatabase(NewApp); var result = await newApp.ModelVersionService.ReconcileModelVersions(); From 6ba754f34d3029660d17197be4c2be7b7cbd9438 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 13 Oct 2026 15:23:42 +0200 Subject: [PATCH 14/25] Spell out why notes and tags replay from different versions Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Tests/ModelVersionTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 74b4703..956147e 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -451,9 +451,10 @@ public void ChangeVersionsToReplayStartAtTheInvalidatedVersion() .Versions; ModelVersionService.ChangeVersionsToReplay(0, versions) - .Should().BeEquivalentTo([(NoteChangeType, 1), (nameof(SetTagChange), 0)]); + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 0)], + "a database at 0 applied version 0 notes as authored, but C replays all tags because B forgot them"); ModelVersionService.ChangeVersionsToReplay(2, versions) - .Should().BeEquivalentTo([(NoteChangeType, 3), (nameof(SetTagChange), 0)], + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 0)], "the apps with versions A and B authored tags with version 0"); ModelVersionService.ChangeVersionsToReplay(3, versions) .Should().BeEquivalentTo([(nameof(SetTagChange), 2)], "the fix was already applied"); From 5c46e4a62c4fc0590f62c089e5cb41699511428e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 16 Oct 2026 16:50:37 +0700 Subject: [PATCH 15/25] Require model versions listing every change type, from version 0 Forgetting to list a new change type in a model version can't be fixed cleanly: its changes are authored with version 0, so no later version can tell them apart from changes older apps applied. Model versions are now required and number from 0. Version 0 lists every change type that existed before model versions, with change version 0, and can't be major. Options validation fails when no versions are declared, or when a registered change type isn't listed by any version, naming the missing types. A new change type then fails at startup until a new version lists it, starting at change version 1. RemoteResourceChanges lists the types AddCrdtRemoteResources registers. A database with nothing stored is at version 0. The first reconcile stores the versions even when that's version 0, so a later edit to it throws like any other released version. The forgotten-type test is now a forgotten behavior change to an existing type, since a forgotten new type can't happen. Co-Authored-By: Claude Opus 5.5 --- src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 20 +++ .../Adapter/CustomObjectAdapterTests.cs | 3 + ...onTests.DescribeModelVersions.verified.txt | 3 +- src/SIL.Harmony.Tests/ModelVersionTests.cs | 170 +++++++++++++----- .../PersistExtraDataTests.cs | 1 + src/SIL.Harmony/Config/HarmonyConfig.cs | 6 +- src/SIL.Harmony/Config/ModelVersionBuilder.cs | 63 +++++-- .../Config/ModelVersionsValidator.cs | 29 +++ src/SIL.Harmony/CrdtKernel.cs | 1 + src/SIL.Harmony/ModelVersionReconciliation.cs | 4 +- src/SIL.Harmony/ModelVersionService.cs | 32 ++-- 11 files changed, 254 insertions(+), 78 deletions(-) create mode 100644 src/SIL.Harmony/Config/ModelVersionsValidator.cs diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index b934c6a..e8810d0 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -62,6 +62,26 @@ public static void ConfigureSample(HarmonyConfig config) .Add>() .Add>() ; + config.ModelVersionBuilder + .Add(0, "Before model versions", v => v + .Change(0) + .Change(0) + .Change(0) + .Change(0) + .Change(0) + .Change(0) + .Change(0) + .Change(0) + .Change>(0) + .Change(0) + .Change(0) + .Change(0) + .Change>(0) + .Change>(0) + .Change>(0) + .Change>(0) + //registered by AddCrdtRemoteResources + .RemoteResourceChanges(0)); config.ObjectTypeListBuilder.DefaultAdapter() .Add(builder => { diff --git a/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs b/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs index 6e9e3c4..511f943 100644 --- a/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs +++ b/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs @@ -154,6 +154,9 @@ public async Task CanAdaptACustomObject() { config.AlwaysValidateCommits = true; config.ChangeTypeListBuilder.Add().Add(); + config.ModelVersionBuilder.Add(0, "Before model versions", v => v + .Change(0) + .Change(0)); config.ObjectTypeListBuilder .CustomAdapter() .Add(builder => builder.HasKey(o => o.Identifier)) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt index a92619e..82cbb8c 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt +++ b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt @@ -1,4 +1,5 @@ -1 Notes: SetWordNoteChange 1 +0 Before model versions: AddWordImageChange 0, EditExampleChange 0, NewDefinitionChange 0, NewExampleChange 0, NewWordChange 0, SetAntonymReferenceChange 0, SetDefinitionPartOfSpeechChange 0, SetTagChange 0, SetWordNoteChange 0, SetWordTextChange 0, TagWordChange 0, create:pendingUpload 0, create:remote-resource 0, delete:Definition 0, delete:Example 0, delete:RemoteResource 0, delete:Tag 0, delete:Word 0, set:remote-resource-metadata 0, setOrder:Definition 0, uploaded:RemoteResource 0 +1 Notes: SetWordNoteChange 1 2 major Rewrite: 3 Tags and notes: SetTagChange 1, SetWordNoteChange 2 4 Fix tags: SetTagChange 2 invalidates from 1, SetWordNoteChange 3 diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 956147e..d37f9ee 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -5,6 +5,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Config; using SIL.Harmony.Db; +using SIL.Harmony.Entities; using SIL.Harmony.Sample; using SIL.Harmony.Sample.Changes; using SIL.Harmony.Sample.Models; @@ -12,8 +13,8 @@ namespace SIL.Harmony.Tests; /// -/// The database starts out written by an "old app" that has no model versions, -/// and is then opened by a "new app" with model versions, by forking it with a different config. +/// The database starts out written by an "old app" that only has the sample's model version 0, +/// and is then opened by a "new app" with more model versions, by forking it with a different config. /// public class ModelVersionTests() : DataModelTestBase(configure: OldApp) { @@ -25,13 +26,48 @@ private static void OldApp(IServiceCollection services) services.Configure(config => config.UnknownChangeHandling = UnknownChangeHandling.Fallback); } - /// doesn't know at all - private static void OldAppWithoutNotes(IServiceCollection services) + /// replaces the sample's versions with a version 0 that lists every registered change type except + private static void VersionZeroWithout(HarmonyConfig config, params string[] newTypes) + { + config.ModelVersionBuilder.Clear(); + config.ModelVersionBuilder.Add(0, "Before model versions", v => + { + foreach (var changeType in config.ChangeTypes.Where(ct => !newTypes.Contains(ct.Discriminator))) + { + v.Change(changeType.Discriminator, 0); + } + }); + } + + /// an old app that doesn't know at all + private static void WithoutChangeType(IServiceCollection services) where T : IChange, IPolyType { OldApp(services); - services.Configure(config => config.ChangeTypeListBuilder.Remove()); + services.Configure(config => + { + config.ChangeTypeListBuilder.Remove(); + VersionZeroWithout(config); + }); } + /// the app that adds in version 1 + private static void AddingChangeType(IServiceCollection services) where T : IChange, IPolyType + { + OldApp(services); + services.Configure(config => + { + VersionZeroWithout(config, T.TypeName); + config.ModelVersionBuilder.Add(1, "Added " + T.TypeName, v => v.Change(1)); + }); + } + + /// doesn't know at all + private static void OldAppWithoutNotes(IServiceCollection services) => WithoutChangeType(services); + + /// adds , which doesn't know + private static void NewAppAddingNotes(IServiceCollection services) => AddingChangeType(services); + + /// changes how applies private static void NewApp(IServiceCollection services) { OldApp(services); @@ -39,6 +75,7 @@ private static void NewApp(IServiceCollection services) config.ModelVersionBuilder.Add(1, "Notes", v => v.Change(1))); } + /// the sample's version 0, then private static Action AppWith(Action versions) { return services => @@ -140,7 +177,7 @@ public async Task UnknownTypeIsAppliedAfterUpgrade() var noteCommit = await SyncFromOtherClient(oldApp, OpaqueNote(oldApp, wordId, "a note"), version: 1); (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); - await using var newApp = oldApp.ForkDatabase(NewApp); + await using var newApp = oldApp.ForkDatabase(NewAppAddingNotes); var result = await newApp.ModelVersionService.ReconcileModelVersions(); result.Should().Be(new ModelVersionReconcileResult(0, 1, false, result.OldestInvalidatedCommit)); @@ -168,7 +205,7 @@ public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() (await NoteOf(newApp, wordId)).Should().Be("newer note"); } - /// the app that added notes was released, but forgot to list them in its model version + /// the app that changed how notes apply was released, but forgot to list them in its model version private static void AppThatForgotNotes(IServiceCollection services) { OldApp(services); @@ -189,29 +226,26 @@ private static void AppThatFixedForgottenNotes(IServiceCollection services) [InlineData(true)] public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedToTheAppThatForgot) { - await using var oldApp = ForkDatabase(OldAppWithoutNotes); - await oldApp.ModelVersionService.ReconcileModelVersions(); + await DataModel.ReconcileModelVersions(); var wordId = Guid.NewGuid(); - await oldApp.WriteNextChange(new NewWordChange(wordId, "hello")); - await using var appThatForgot = oldApp.ForkDatabase(AppThatForgotNotes); - var forgottenVersion = appThatForgot.CrdtConfig.ChangeVersion(NoteChangeType); - forgottenVersion.Should().Be(0, "the app that forgot notes authors them like the old app did"); - var noteCommit = await SyncFromOtherClient(oldApp, OpaqueNote(oldApp, wordId, "a note"), forgottenVersion); - (await NoteOf(oldApp, wordId)).Should().BeNull("the old app can't apply the note change"); + await WriteNextChange(new NewWordChange(wordId, "hello")); + //the app that forgot notes authors them with version 0, the same as the old app + var forgottenNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "a note"), version: 0); - var beforeFix = oldApp; - await using var upgraded = upgradedToTheAppThatForgot ? oldApp.ForkDatabase(AppThatForgotNotes) : null; + var beforeFix = (DataModelTestBase)this; + await using var upgraded = upgradedToTheAppThatForgot ? ForkDatabase(AppThatForgotNotes) : null; if (upgraded is not null) { - await upgraded.ModelVersionService.ReconcileModelVersions(); - (await NoteOf(upgraded, wordId)).Should().BeNull("the app that forgot notes doesn't know to replay them"); + var forgotResult = await upgraded.ModelVersionService.ReconcileModelVersions(); + forgotResult.OldestInvalidatedCommit.Should().BeNull("the app that forgot notes doesn't know to replay them"); beforeFix = upgraded; } await using var fixedApp = beforeFix.ForkDatabase(AppThatFixedForgottenNotes); var result = await fixedApp.ModelVersionService.ReconcileModelVersions(); - (result.OldestInvalidatedCommit?.Id).Should().Be(noteCommit.Id); + (result.OldestInvalidatedCommit?.Id).Should().Be(forgottenNote.Id); + result.FullRegeneration.Should().BeFalse(); (await NoteOf(fixedApp, wordId)).Should().Be("a note"); fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(1, "notes authored after the fix can be told apart"); } @@ -238,7 +272,7 @@ static void SmallCheckpoints(IServiceCollection services) => await using var newApp = oldApp.ForkDatabase(services => { - NewApp(services); + NewAppAddingNotes(services); SmallCheckpoints(services); }); var checkpoint = await newApp.DbContext.Commits.AsNoTracking() @@ -328,6 +362,18 @@ public async Task NewDatabaseStoresTheVersionWithoutReplaying() (await app.DataModel.GetLocalState("harmony:modelVersions")).Should().NotBeNull(); } + [Fact] + public async Task VersionZeroIsStoredSoItCantBeModifiedLater() + { + var result = await ModelVersionService.ReconcileModelVersions(); + + result.Should().Be(new ModelVersionReconcileResult(0, 0, false, null)); + (await DataModel.GetLocalState("harmony:modelVersions")).Should().NotBeNull(); + await using var withoutTags = ForkDatabase(WithoutChangeType); + var act = () => withoutTags.ModelVersionService.ReconcileModelVersions(); + await act.Should().ThrowAsync().WithMessage("Model version 0 (Before model versions) was modified*"); + } + [Fact] public async Task DowngradeRegeneratesAllSnapshots() { @@ -350,18 +396,14 @@ public async Task DowngradeRegeneratesAllSnapshots() [Fact] public async Task DowngradeToAnAppWithoutAChangeTypeRemovesItsEntities() { - await using var newApp = ForkDatabase(AppWith(versions => versions.Add(1, "Tags", v => v.Change(1)))); + await using var newApp = ForkDatabase(AddingChangeType); await newApp.ModelVersionService.ReconcileModelVersions(); var wordId = Guid.NewGuid(); var tagId = Guid.NewGuid(); await newApp.WriteNextChange(new NewWordChange(wordId, "hello")); await newApp.WriteNextChange(new SetTagChange(tagId, "tag")); - await using var oldApp = newApp.ForkDatabase(services => - { - OldApp(services); - services.Configure(config => config.ChangeTypeListBuilder.Remove()); - }); + await using var oldApp = newApp.ForkDatabase(WithoutChangeType); var snapshotIds = await SnapshotIds(oldApp); var result = await oldApp.ModelVersionService.ReconcileModelVersions(); @@ -423,19 +465,48 @@ public void ModelVersionWithAnUnregisteredChangeTypeFailsValidation() act.Should().Throw(); } + [Fact] + public void NoModelVersionsFailsValidation() + { + using var services = new ServiceCollection() + .AddCrdtDataSample(":memory:") + .Configure(config => config.ModelVersionBuilder.Clear()) + .BuildServiceProvider(); + + var act = () => services.GetRequiredService>().Value; + + act.Should().Throw().WithMessage("Declare model versions*"); + } + + [Fact] + public void ChangeTypeNotListedByAModelVersionFailsValidation() + { + using var services = new ServiceCollection() + .AddCrdtDataSample(":memory:") + .Configure(config => VersionZeroWithout(config, NoteChangeType, nameof(SetTagChange))) + .BuildServiceProvider(); + + var act = () => services.GetRequiredService>().Value; + + act.Should().Throw() + .WithMessage($"*aren't listed by a model version: {nameof(SetTagChange)}, {NoteChangeType}.*"); + } + [Fact] public void ChangeVersionsToReplayStartAtTheFirstNewVersion() { var versions = new ModelVersionBuilder() + .Add(0, "Before", v => v.Change(0).Change(0)) .Add(1, "A", v => v.Change(1)) .Add(2, "B", v => v.Change(1).Change(2)) .Add(3, "C", v => v.Change(2)) .Versions; ModelVersionService.ChangeVersionsToReplay(1, versions) - .Should().BeEquivalentTo([(nameof(SetTagChange), 1), (NoteChangeType, 2)]); + .Should().BeEquivalentTo([(nameof(SetTagChange), 1), (nameof(SetWordNoteChange), 2)]); ModelVersionService.ChangeVersionsToReplay(0, versions) - .Should().BeEquivalentTo([(NoteChangeType, 1), (nameof(SetTagChange), 1)]); + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 1)], + "version 0 is the state before model versions, so its changes were applied as authored"); ModelVersionService.ChangeVersionsToReplay(3, versions).Should().BeEmpty(); } @@ -443,8 +514,9 @@ public void ChangeVersionsToReplayStartAtTheFirstNewVersion() public void ChangeVersionsToReplayStartAtTheInvalidatedVersion() { var versions = new ModelVersionBuilder() + .Add(0, "Before", v => v.Change(0).Change(0)) .Add(1, "A", v => v.Change(1)) - //forgot to list tags + //changed how tags apply, but forgot to list them .Add(2, "B", v => v.Change(2)) .Add(3, "C", v => v.Change(1, invalidateFrom: 0).Change(3)) .Add(4, "D", v => v.Change(2)) @@ -455,39 +527,43 @@ public void ChangeVersionsToReplayStartAtTheInvalidatedVersion() "a database at 0 applied version 0 notes as authored, but C replays all tags because B forgot them"); ModelVersionService.ChangeVersionsToReplay(2, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 0)], - "the apps with versions A and B authored tags with version 0"); + "the apps with versions 0 to 2 authored tags with version 0"); ModelVersionService.ChangeVersionsToReplay(3, versions) .Should().BeEquivalentTo([(nameof(SetTagChange), 2)], "the fix was already applied"); } [Fact] - public void ModelVersionsMustBeNumberedInOrder() + public void ModelVersionsMustBeNumberedInOrderFromZero() { - var builder = new ModelVersionBuilder().Add(1, "A", v => v.Change(1)); + new ModelVersionBuilder().Invoking(b => b.Add(1, "A", _ => { })).Should().Throw(); + new ModelVersionBuilder().Invoking(b => b.AddMajor(0, "Before")) + .Should().Throw().WithMessage("Model version 0 (Before) can't be major*"); + var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change(0)); - builder.Invoking(b => b.Add(3, "C", _ => { })).Should().Throw(); - builder.Invoking(b => b.AddMajor(1, "A again")).Should().Throw(); + builder.Invoking(b => b.Add(2, "B", _ => { })).Should().Throw(); + builder.Invoking(b => b.AddMajor(0, "Before again")).Should().Throw(); builder.Versions.Should().ContainSingle(); } [Fact] public void ChangeVersionsMustBeValid() { - new ModelVersionBuilder().Invoking(b => b.Add(1, "A", v => v.Change(2))) + new ModelVersionBuilder().Invoking(b => b.Add(0, "Before", v => v.Change(1))) + .Should().Throw().WithMessage("*in model version 0 (Before) must be 0*"); + var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change(0).Change(0)); + + builder.Invoking(b => b.Add(1, "A", v => v.Change(0))) + .Should().Throw().WithMessage("*must be 1, one more than its previous version 0"); + builder.Invoking(b => b.Add(1, "A", v => v.Change(2))) .Should().Throw().WithMessage("*must be 1, one more than its previous version 0"); - var builder = new ModelVersionBuilder().Add(1, "A", v => v.Change(1).Change(1)); - - builder.Invoking(b => b.Add(2, "B", v => v.Change(1))) - .Should().Throw().WithMessage("*must be 2, one more than its previous version 1"); - builder.Invoking(b => b.Add(2, "B", v => v.Change(3))) - .Should().Throw().WithMessage("*must be 2, one more than its previous version 1"); - builder.Invoking(b => b.Add(2, "B", v => v.Change(0))).Should().Throw(); - builder.Invoking(b => b.Add(2, "B", v => v.Change(2, invalidateFrom: 3))).Should().Throw(); - builder.Invoking(b => b.Add(2, "B", v => v.Change(2, invalidateFrom: -1))).Should().Throw(); - builder.Invoking(b => b.Add(2, "B", v => v.Change(2).Change(2))) + builder.Invoking(b => b.Add(1, "A", v => v.Change(-1))).Should().Throw(); + builder.Invoking(b => b.Add(1, "A", v => v.Change(1, invalidateFrom: 2))).Should().Throw(); + builder.Invoking(b => b.Add(1, "A", v => v.Change(1, invalidateFrom: -1))).Should().Throw(); + builder.Invoking(b => b.Add(1, "A", v => v.Change(1).Change(1))) .Should().Throw().WithMessage("*more than once*"); builder.Versions.Should().ContainSingle("a version that failed validation isn't added"); - builder.Add(2, "B", v => v.Change(2)).Versions.Should().HaveCount(2); + builder.Add(1, "A", v => v.Change(1).Change(1)) + .Versions.Should().HaveCount(2, "a type new in this version starts at 1"); } [Fact] diff --git a/src/SIL.Harmony.Tests/PersistExtraDataTests.cs b/src/SIL.Harmony.Tests/PersistExtraDataTests.cs index 22e8857..597ab7a 100644 --- a/src/SIL.Harmony.Tests/PersistExtraDataTests.cs +++ b/src/SIL.Harmony.Tests/PersistExtraDataTests.cs @@ -58,6 +58,7 @@ public PersistExtraDataTests() { config.ObjectTypeListBuilder.DefaultAdapter().Add(); config.ChangeTypeListBuilder.Add(); + config.ModelVersionBuilder.Add(1, "Extra data", v => v.Change(1)); config.BeforeSaveObject = (obj, snapshot) => { if (obj is ExtraDataModel extraDataModel) diff --git a/src/SIL.Harmony/Config/HarmonyConfig.cs b/src/SIL.Harmony/Config/HarmonyConfig.cs index 1902c6b..3db39f6 100644 --- a/src/SIL.Harmony/Config/HarmonyConfig.cs +++ b/src/SIL.Harmony/Config/HarmonyConfig.cs @@ -53,8 +53,10 @@ public class HarmonyConfig /// public ModelVersionBuilder ModelVersionBuilder { get; } = new(); public IReadOnlyList ModelVersions => ModelVersionBuilder.Versions; - /// the number of model versions, 0 when the app declares none - public int CurrentModelVersion => ModelVersions.Count; + /// the number of the last model version + public int CurrentModelVersion => ModelVersions.Count == 0 + ? throw new InvalidOperationException("No model versions are declared, see " + nameof(ModelVersionBuilder)) + : ModelVersions[^1].Number; private readonly Lazy> _lazyChangeVersions; public ObjectTypeListBuilder ObjectTypeListBuilder { get; } = new(); public IEnumerable ObjectTypes => ObjectTypeListBuilder.AdapterProviders.SelectMany(p => p.GetRegistrations().Select(r => r.ObjectDbType)); diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs index d301107..9eceed6 100644 --- a/src/SIL.Harmony/Config/ModelVersionBuilder.cs +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -1,12 +1,14 @@ using SIL.Harmony.Changes; using SIL.Harmony.Entities; +using SIL.Harmony.Resource; namespace SIL.Harmony.Config; /// -/// Declares the ordered list of model versions. Add a version when a release adds a change type, or changes how -/// an existing change type applies. Changes authored by a newer app are then replayed when an older app upgrades, -/// see . +/// Declares the ordered list of model versions, which is required. Version 0 lists every change type that existed +/// before model versions, with change version 0. Add a version when a release adds a change type, or changes how +/// an existing change type applies. Every registered change type must be listed by a version. +/// Changes authored by a newer app are then replayed when an older app upgrades, see . /// Released versions must never be modified, use in a snapshot test to make sure. /// public class ModelVersionBuilder @@ -28,17 +30,17 @@ public class ModelVersionBuilder /// Adds a version. Changes of the listed types are replayed when upgrading to it, /// see for which ones. /// - /// the next number: versions are numbered 1, 2, 3 and so on + /// the next number: versions are numbered 0, 1, 2 and so on public ModelVersionBuilder Add(int number, string name, Action changes) { return Add(number, name, false, changes); } /// - /// Adds a version that regenerates all snapshots when upgrading to it. + /// Adds a version that regenerates all snapshots when upgrading to it. Version 0 can't be major. /// The listed change types are still authored with their new version. /// - /// the next number: versions are numbered 1, 2, 3 and so on + /// the next number: versions are numbered 0, 1, 2 and so on public ModelVersionBuilder AddMajor(int number, string name, Action? changes = null) { return Add(number, name, true, changes); @@ -48,14 +50,24 @@ private ModelVersionBuilder Add(int number, string name, bool major, Actionremoves all versions, so a test can declare a different history than the model it starts from + internal void Clear() + { + CheckFrozen(); + _versions.Clear(); + _changeTypes.Clear(); + _changeVersions.Clear(); + } + internal void Freeze() { _frozen = true; @@ -89,16 +110,16 @@ public class ModelVersionChanges internal List Types { get; } = []; /// - /// The change type was added or modified in this version. + /// The change type was added or modified in this version, or existed before model versions when this is version 0. /// /// - /// the version changes of this type are authored with from now on: one more than its previous version, so 1 the first time it is listed. - /// Changes from before model versions existed are version 0. + /// the version changes of this type are authored with from now on. 0 in model version 0, + /// otherwise one more than its previous version, so 1 the first time a new type is listed. /// /// /// on upgrade, changes of this type with this version or higher are replayed. Defaults to , - /// so only changes authored by this app or a newer one are replayed. When an earlier, released version added or - /// modified the type but forgot to list it, pass the version its apps authored the type with, to replay those too. + /// so only changes authored by this app or a newer one are replayed. When an earlier, released version modified + /// how the type applies but forgot to list it, pass the version its apps authored the type with, to replay those too. /// public ModelVersionChanges Change(int version, int? invalidateFrom = null) where T : IChange, IPolyType { @@ -107,6 +128,20 @@ public ModelVersionChanges Change(int version, int? invalidateFrom = null) wh return this; } + /// + /// The change types registered by , all with the same version. + /// List them again only if they change, with their next version. + /// + public ModelVersionChanges RemoteResourceChanges(int version, int? invalidateFrom = null) + where TMetadata : class + { + return Change>(version, invalidateFrom) + .Change>(version, invalidateFrom) + .Change>(version, invalidateFrom) + .Change>(version, invalidateFrom) + .Change>(version, invalidateFrom); + } + /// /// by discriminator. /// Only for change types that no longer exist in the code, use otherwise. @@ -114,7 +149,7 @@ public ModelVersionChanges Change(int version, int? invalidateFrom = null) wh public ModelVersionChanges Change(string discriminator, int version, int? invalidateFrom = null) { ArgumentException.ThrowIfNullOrWhiteSpace(discriminator); - ArgumentOutOfRangeException.ThrowIfLessThan(version, 1); + ArgumentOutOfRangeException.ThrowIfNegative(version); var from = invalidateFrom ?? version; ArgumentOutOfRangeException.ThrowIfNegative(from, nameof(invalidateFrom)); ArgumentOutOfRangeException.ThrowIfGreaterThan(from, version, nameof(invalidateFrom)); diff --git a/src/SIL.Harmony/Config/ModelVersionsValidator.cs b/src/SIL.Harmony/Config/ModelVersionsValidator.cs new file mode 100644 index 0000000..978fc1f --- /dev/null +++ b/src/SIL.Harmony/Config/ModelVersionsValidator.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.Options; + +namespace SIL.Harmony.Config; + +/// +/// Model versions are required, and every registered change type must be listed by one, +/// so a new change type can't be released without a version that makes old apps replay it. +/// +internal class ModelVersionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, HarmonyConfig options) + { + if (options.ModelVersions.Count == 0) + return ValidateOptionsResult.Fail( + $"Declare model versions with {nameof(HarmonyConfig)}.{nameof(HarmonyConfig.ModelVersionBuilder)}, " + + "starting with version 0 listing every change type that existed before model versions"); + var listed = options.ModelVersionBuilder.ChangeVersions; + var unlisted = options.ChangeTypes + .Select(ct => ct.Discriminator) + .Where(discriminator => !listed.ContainsKey(discriminator)) + .Order(StringComparer.Ordinal) + .ToArray(); + if (unlisted.Length > 0) + return ValidateOptionsResult.Fail( + $"These change types aren't listed by a model version: {string.Join(", ", unlisted)}. " + + "Add a new model version that lists them, released versions must not change"); + return ValidateOptionsResult.Success; + } +} diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index df41b4c..7466b86 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -68,6 +68,7 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service .Validate( config => config.ModelVersionBuilder.ChangeTypes.All(t => config.ChangeTypes.Any(ct => ct.Type == t)), "Every change type in a model version must be registered in the ChangeTypeListBuilder"); + services.AddSingleton, ModelVersionsValidator>(); services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); diff --git a/src/SIL.Harmony/ModelVersionReconciliation.cs b/src/SIL.Harmony/ModelVersionReconciliation.cs index 8546e59..bae80bf 100644 --- a/src/SIL.Harmony/ModelVersionReconciliation.cs +++ b/src/SIL.Harmony/ModelVersionReconciliation.cs @@ -25,11 +25,9 @@ public record ModelVersionReconcileResult( internal record StoredModelVersions(int Version, StoredModelVersion[] Versions) { public const string LocalStateKey = "harmony:modelVersions"; - public static readonly StoredModelVersions None = new(0, []); - public static StoredModelVersions From(IReadOnlyList versions) { - return new StoredModelVersions(versions.Count, + return new StoredModelVersions(versions[^1].Number, versions.Select(v => new StoredModelVersion(v.Major, v.Changes.ToArray())).ToArray()); } } diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index a9de5a4..08e5b79 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -29,17 +29,21 @@ public async Task ReconcileModelVersions() using var locked = await repo.Lock(); repo.ClearChangeTracker(); await using var transaction = await repo.BeginTransactionAsync(); - var stored = await repo.GetLocalState(StoredModelVersions.LocalStateKey) - ?? StoredModelVersions.None; + //nothing stored means the snapshots were made before model versions, which is version 0 + var stored = await repo.GetLocalState(StoredModelVersions.LocalStateKey); + var storedVersion = stored?.Version ?? 0; var current = Config.ModelVersions; var currentVersion = Config.CurrentModelVersion; - EnsureAppliedVersionsUnchanged(stored, current); - if (stored.Version == currentVersion) - return new ModelVersionReconcileResult(stored.Version, currentVersion, false, null); + if (stored is not null) + { + EnsureAppliedVersionsUnchanged(stored, current); + if (storedVersion == currentVersion) + return new ModelVersionReconcileResult(storedVersion, currentVersion, false, null); + } var fullRegeneration = false; Commit? oldestInvalidated; - if (currentVersion < stored.Version || current.Skip(stored.Version).Any(v => v.Major)) + if (currentVersion < storedVersion || NewVersions(storedVersion, current).Any(v => v.Major)) { //a downgrade is rare and the snapshots were made by newer code, so start over fullRegeneration = true; @@ -48,7 +52,7 @@ public async Task ReconcileModelVersions() } else { - oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(stored.Version, current)); + oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(storedVersion, current)); if (oldestInvalidated is not null) { await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(oldestInvalidated), Config); @@ -59,12 +63,12 @@ public async Task ReconcileModelVersions() { logger.LogInformation( "Model version changed from {StoredVersion} to {CurrentVersion}, oldest invalidated commit {CommitId}, full regeneration: {FullRegeneration}", - stored.Version, currentVersion, oldestInvalidated.Id, fullRegeneration); + storedVersion, currentVersion, oldestInvalidated.Id, fullRegeneration); } await repo.SetLocalState(StoredModelVersions.LocalStateKey, StoredModelVersions.From(current)); await transaction.CommitAsync(); - return new ModelVersionReconcileResult(stored.Version, currentVersion, fullRegeneration, oldestInvalidated); + return new ModelVersionReconcileResult(storedVersion, currentVersion, fullRegeneration, oldestInvalidated); } private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, IReadOnlyList current) @@ -74,11 +78,17 @@ private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, I { if (stored.Versions[i].SameAs(current[i])) continue; throw new InvalidOperationException( - $"Model version {i + 1} ({current[i].Name}) was modified after it was applied to this database. " + + $"Model version {current[i].Number} ({current[i].Name}) was modified after it was applied to this database. " + "Released model versions must not change, add a new version instead."); } } + /// the versions after , versions are numbered from 0 + private static IEnumerable NewVersions(int storedVersion, IReadOnlyList versions) + { + return versions.Skip(storedVersion + 1); + } + /// /// For each change type in the versions after , the lowest change version to replay: /// the lowest those versions give the type. @@ -86,7 +96,7 @@ private static void EnsureAppliedVersionsUnchanged(StoredModelVersions stored, I internal static (string ChangeType, int FromVersion)[] ChangeVersionsToReplay(int storedVersion, IReadOnlyList versions) { - return versions.Skip(storedVersion) + return NewVersions(storedVersion, versions) .SelectMany(v => v.Changes) .GroupBy(c => c.ChangeType, StringComparer.Ordinal) .Select(changes => (changes.Key, changes.Min(c => c.InvalidateFrom))) From 9a54bc1ad79507ef0e473869ba9f6fedd9aeed83 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 16 Oct 2026 16:54:10 +0700 Subject: [PATCH 16/25] Document required model versions in the README and AGENTS.md Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + README.md | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e35409e..62b6969 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,6 +66,7 @@ Harmony is the substrate other apps sync through, so a bug here corrupts every c - **Hash chain.** `Commit.Hash` chains over `ParentHash` in commit order; `AlwaysValidateCommits` (on in tests) checks it after every write. - **Serialized changes are forever.** Commits written by old clients must still deserialize. Keep every change type's `$type` discriminator and JSON property names stable; a rename or removal needs a migration path. `PeekThenConcreteChangeConverter` owns `IChange` discrimination and needs `$type` first; unknown types become `OpaqueChange` when configured. - **JSON options come from `HarmonyConfig.JsonSerializerOptions`**, never a fresh `JsonSerializerOptions`, so registered change and object types resolve. +- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. To fix a release that changed a type's behavior without listing it, list the type in a new version with `invalidateFrom` set to the change version that release authored. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. ## Review diff --git a/README.md b/README.md index fe75930..2130a10 100644 --- a/README.md +++ b/README.md @@ -181,28 +181,34 @@ It's that easy. All the heavy lifting is done by the interface which is fairly s ### Model versions Clients on different app versions sync with each other. When a newer app adds a change type, an older app stores those changes as `OpaqueChange` (with `UnknownChangeHandling.Fallback`) and skips them. When a newer app changes how a change type applies, an older app applies those changes with its old code. In both cases the snapshots must be rebuilt when the older app upgrades. -To make this possible, declare a model version each time a release adds a change type, or changes how one applies. Number the versions 1, 2, 3 and so on. Each version lists the change types it added or modified, with the type's new version number. A type's version starts at 1 and goes up by one each time it is listed. Only add versions at the end, never modify a released version: +To make this possible, model versions are required. Number them 0, 1, 2 and so on. Version 0 lists every change type that existed before model versions, with change version 0. After that, add a version each time a release adds a change type, or changes how one applies, listing the type with its next change version: 1 for a new type, then one more each time it is listed. Startup fails if no versions are declared, or if a registered change type isn't listed by any version. Only add versions at the end, never modify a released version: ```C# services.AddCrdtData(config => { config.ModelVersionBuilder + .Add(0, "Before model versions", v => v + .Change(0) + .Change(0) + //the change types AddCrdtRemoteResources registers + .RemoteResourceChanges(0)) .Add(1, "Word notes", v => v.Change(1)) //a major version regenerates all snapshots on upgrade - .AddMajor(2, "Reference rewrite", v => v.Change(1)); + .AddMajor(2, "Reference rewrite", v => v.Change(1)); }); ``` -Every change is stored with the version of its type when it was authored: the version the latest model version that lists the type gave it. Changes from before model versions existed, or of types no model version lists, are version 0. +Every change is stored with the version of its type when it was authored: the version the latest model version that lists the type gave it. -Call `ReconcileModelVersions` when you open a database, before you use it. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it replays the changes of that type with the listed version or higher, which are the changes authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. +Call `ReconcileModelVersions` when you open a database, before you use it. A database that has no model versions stored yet is at version 0. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it replays the changes of that type with the listed version or higher, which are the changes authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. ```C# await dataModel.ReconcileModelVersions(); ``` -If a released version forgot to list a change type that it added or modified, its apps authored those changes with the type's previous version number, the same as the apps before them. To fix it, list the type in a new version with `invalidateFrom` set to that previous number. Upgrading to the new version then replays every change of the type with that number or higher: +If a released version changed how a change type applies but forgot to list it, its apps authored those changes with the type's previous version number, the same as the apps before them. To fix it, list the type in a new version with `invalidateFrom` set to that previous number. Upgrading to the new version then replays every change of the type with that number or higher: ```C# config.ModelVersionBuilder + .Add(0, "Before model versions", v => v.Change(0).Change(0)) .Add(1, "Word notes", v => v.Change(1)) - //forgot to list SetTagChange, so its changes are still authored as version 0 - .Add(2, "Tags", v => v.Change(2)) + //changed how SetTagChange applies but forgot to list it, so tags are still authored as version 0 + .Add(2, "Notes again", v => v.Change(2)) .Add(3, "Fix tags", v => v.Change(1, invalidateFrom: 0)); ``` To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. From 5bcc223ef6687f2b2d6e3ff6c0cc5ffe77bbdde2 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 16 Oct 2026 16:06:49 +0200 Subject: [PATCH 17/25] Say what invalidateFrom is for, fix two stale doc comments Listing a type only invalidates changes authored at its new version. invalidateFrom also covers a behavior change that must apply to older changes, not just a release that forgot to list the type, so say so. ChangeEntity.ChangeVersion isn't a count of model versions since version 0 lists every type, and model versions number from 0, not 1. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- README.md | 4 ++-- src/SIL.Harmony.Core/ChangeEntity.cs | 4 ++-- src/SIL.Harmony/Config/ModelVersion.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 62b6969..f1dd47c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ Harmony is the substrate other apps sync through, so a bug here corrupts every c - **Hash chain.** `Commit.Hash` chains over `ParentHash` in commit order; `AlwaysValidateCommits` (on in tests) checks it after every write. - **Serialized changes are forever.** Commits written by old clients must still deserialize. Keep every change type's `$type` discriminator and JSON property names stable; a rename or removal needs a migration path. `PeekThenConcreteChangeConverter` owns `IChange` discrimination and needs `$type` first; unknown types become `OpaqueChange` when configured. - **JSON options come from `HarmonyConfig.JsonSerializerOptions`**, never a fresh `JsonSerializerOptions`, so registered change and object types resolve. -- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. To fix a release that changed a type's behavior without listing it, list the type in a new version with `invalidateFrom` set to the change version that release authored. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. +- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Listing a type invalidates the changes authored with its new change version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. ## Review diff --git a/README.md b/README.md index 2130a10..d1cb85c 100644 --- a/README.md +++ b/README.md @@ -198,11 +198,11 @@ services.AddCrdtData(config => ``` Every change is stored with the version of its type when it was authored: the version the latest model version that lists the type gave it. -Call `ReconcileModelVersions` when you open a database, before you use it. A database that has no model versions stored yet is at version 0. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it replays the changes of that type with the listed version or higher, which are the changes authored by a newer app than the one that applied them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. +Call `ReconcileModelVersions` when you open a database, before you use it. A database that has no model versions stored yet is at version 0. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it invalidates the changes of that type with the listed version or higher, the ones authored by a newer app than the one that applied them, and rebuilds the snapshots from the checkpoint before the oldest of them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. ```C# await dataModel.ReconcileModelVersions(); ``` -If a released version changed how a change type applies but forgot to list it, its apps authored those changes with the type's previous version number, the same as the apps before them. To fix it, list the type in a new version with `invalidateFrom` set to that previous number. Upgrading to the new version then replays every change of the type with that number or higher: +Listing a type invalidates only the changes authored with its new change version or later, the ones from apps on the new version. When the new behavior must also apply to older changes, pass `invalidateFrom` with the change version to invalidate from. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the previous number: ```C# config.ModelVersionBuilder .Add(0, "Before model versions", v => v.Change(0).Change(0)) diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index 6172eca..1085a78 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -14,8 +14,8 @@ public ChangeEntity() public required Guid EntityId { get; set; } public required TChange Change { get; set; } /// - /// The version of the change type when the change was authored: how many model versions list its type. - /// Changes from before model versions existed are version 0. + /// The version of the change type when the change was authored: the version the latest model version that lists + /// its type gave it. Changes from before model versions existed are version 0. /// public int ChangeVersion { get; set; } } diff --git a/src/SIL.Harmony/Config/ModelVersion.cs b/src/SIL.Harmony/Config/ModelVersion.cs index 98ff82c..9ca41d0 100644 --- a/src/SIL.Harmony/Config/ModelVersion.cs +++ b/src/SIL.Harmony/Config/ModelVersion.cs @@ -4,7 +4,7 @@ namespace SIL.Harmony.Config; /// One step in the ordered list of model versions an app declares on . /// Released versions must never be modified, only new versions added at the end. /// -/// the position of the version in the list, starting at 1 +/// the position of the version in the list, starting at 0 /// describes the version for people, it is not stored in the database /// when true, upgrading to this version regenerates all snapshots /// the change types this version added or modified, sorted by type From 9866fb44647424f03160a7581dd975cea31a560d Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 16 Oct 2026 16:27:25 +0200 Subject: [PATCH 18/25] Require RemoteResourceChanges when remote resources are enabled Listing Harmony's own change types by hand hides them from any future Harmony versioning (sillsdev/harmony#141), so options validation requires the helper, naming the call to add. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- src/SIL.Harmony.Tests/ModelVersionTests.cs | 28 ++++++++++++++++--- src/SIL.Harmony/Config/ModelVersionBuilder.cs | 11 +++++++- .../Config/ModelVersionsValidator.cs | 6 ++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f1dd47c..e937e43 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ Harmony is the substrate other apps sync through, so a bug here corrupts every c - **Hash chain.** `Commit.Hash` chains over `ParentHash` in commit order; `AlwaysValidateCommits` (on in tests) checks it after every write. - **Serialized changes are forever.** Commits written by old clients must still deserialize. Keep every change type's `$type` discriminator and JSON property names stable; a rename or removal needs a migration path. `PeekThenConcreteChangeConverter` owns `IChange` discrimination and needs `$type` first; unknown types become `OpaqueChange` when configured. - **JSON options come from `HarmonyConfig.JsonSerializerOptions`**, never a fresh `JsonSerializerOptions`, so registered change and object types resolve. -- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Listing a type invalidates the changes authored with its new change version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. +- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Harmony's own change types aren't versioned yet: changing how one applies needs sillsdev/harmony#141 first. Listing a type invalidates the changes authored with its new change version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. ## Review diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index d37f9ee..5dfd16b 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -32,10 +32,9 @@ private static void VersionZeroWithout(HarmonyConfig config, params string[] new config.ModelVersionBuilder.Clear(); config.ModelVersionBuilder.Add(0, "Before model versions", v => { - foreach (var changeType in config.ChangeTypes.Where(ct => !newTypes.Contains(ct.Discriminator))) - { - v.Change(changeType.Discriminator, 0); - } + v.RemoteResourceChanges(0); + var others = config.ChangeTypes.Select(ct => ct.Discriminator).Except(v.Changes.Keys).Except(newTypes); + foreach (var discriminator in others) v.Change(discriminator, 0); }); } @@ -566,6 +565,27 @@ public void ChangeVersionsMustBeValid() .Versions.Should().HaveCount(2, "a type new in this version starts at 1"); } + [Fact] + public void RemoteResourceChangesMustBeListed() + { + using var services = new ServiceCollection() + .AddCrdtDataSample(":memory:") + .Configure(config => + { + config.ModelVersionBuilder.Clear(); + config.ModelVersionBuilder.Add(0, "Before model versions", v => + { + foreach (var changeType in config.ChangeTypes) v.Change(changeType.Discriminator, 0); + }); + }) + .BuildServiceProvider(); + + var act = () => services.GetRequiredService>().Value; + + act.Should().Throw() + .WithMessage("Call RemoteResourceChanges in a model version*"); + } + [Fact] public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() { diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs index 9eceed6..9e28416 100644 --- a/src/SIL.Harmony/Config/ModelVersionBuilder.cs +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -17,6 +17,7 @@ public class ModelVersionBuilder private readonly List _versions = []; private readonly HashSet _changeTypes = []; private readonly Dictionary _changeVersions = new(StringComparer.Ordinal); + private bool _remoteResourcesListed; public IReadOnlyList Versions => _versions.AsReadOnly(); @@ -26,6 +27,9 @@ public class ModelVersionBuilder /// the latest version of each change type listed by a model version internal IReadOnlyDictionary ChangeVersions => _changeVersions; + /// whether a version called + internal bool RemoteResourcesListed => _remoteResourcesListed; + /// /// Adds a version. Changes of the listed types are replayed when upgrading to it, /// see for which ones. @@ -58,6 +62,7 @@ private ModelVersionBuilder Add(int number, string name, bool major, Action Changes { get; } = new(StringComparer.Ordinal); internal List Types { get; } = []; + internal bool RemoteResourcesListed { get; private set; } /// /// The change type was added or modified in this version, or existed before model versions when this is version 0. @@ -130,11 +137,13 @@ public ModelVersionChanges Change(int version, int? invalidateFrom = null) wh /// /// The change types registered by , all with the same version. - /// List them again only if they change, with their next version. + /// Options validation requires it when remote resources are enabled. Harmony doesn't version these types yet, + /// see https://github.com/sillsdev/harmony/issues/141. /// public ModelVersionChanges RemoteResourceChanges(int version, int? invalidateFrom = null) where TMetadata : class { + RemoteResourcesListed = true; return Change>(version, invalidateFrom) .Change>(version, invalidateFrom) .Change>(version, invalidateFrom) diff --git a/src/SIL.Harmony/Config/ModelVersionsValidator.cs b/src/SIL.Harmony/Config/ModelVersionsValidator.cs index 978fc1f..3424312 100644 --- a/src/SIL.Harmony/Config/ModelVersionsValidator.cs +++ b/src/SIL.Harmony/Config/ModelVersionsValidator.cs @@ -5,6 +5,8 @@ namespace SIL.Harmony.Config; /// /// Model versions are required, and every registered change type must be listed by one, /// so a new change type can't be released without a version that makes old apps replay it. +/// Harmony's own change types must be listed through , +/// so a Harmony version can be slotted in there later, see https://github.com/sillsdev/harmony/issues/141. /// internal class ModelVersionsValidator : IValidateOptions { @@ -14,6 +16,10 @@ public ValidateOptionsResult Validate(string? name, HarmonyConfig options) return ValidateOptionsResult.Fail( $"Declare model versions with {nameof(HarmonyConfig)}.{nameof(HarmonyConfig.ModelVersionBuilder)}, " + "starting with version 0 listing every change type that existed before model versions"); + if (options.RemoteResourcesEnabled && !options.ModelVersionBuilder.RemoteResourcesListed) + return ValidateOptionsResult.Fail( + $"Call {nameof(ModelVersionChanges.RemoteResourceChanges)}<{options.RemoteResourceMetadataType!.Name}> in a model version, " + + "it lists the change types AddCrdtRemoteResources registers"); var listed = options.ModelVersionBuilder.ChangeVersions; var unlisted = options.ChangeTypes .Select(ct => ct.Discriminator) From 6cb451b57c03f2896fb6f4b6cd3f937a1676852f Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 16 Oct 2026 16:28:18 +0200 Subject: [PATCH 19/25] Make the change version the model version number A change type's version was a per-type counter, one more each time a model version listed it, beside the model version numbers themselves. Two number lines for one history meant numbers to type on every Change() call, a "one more than the previous" rule to guard them, and an ambiguous invalidateFrom: "SetTagChange 2 invalidates from 1" named the type's first revision, which a reader had to trace back to the model version that assigned it. A change is now stored with the number of the latest model version that lists its type. Change() takes no number, invalidateFrom is a model version, and DescribeModelVersions prints type names. For a type, the old counter was a strictly increasing function of the model versions listing it, so replay decisions are unchanged: the service, the stored versions and the query only drop the Version field. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- README.md | 28 ++--- src/SIL.Harmony.Core/ChangeEntity.cs | 4 +- src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 34 ++--- .../Adapter/CustomObjectAdapterTests.cs | 4 +- ...onTests.DescribeModelVersions.verified.txt | 8 +- src/SIL.Harmony.Tests/ModelVersionTests.cs | 118 +++++++++--------- .../PersistExtraDataTests.cs | 2 +- src/SIL.Harmony/Config/HarmonyConfig.cs | 8 +- src/SIL.Harmony/Config/ModelVersion.cs | 13 +- src/SIL.Harmony/Config/ModelVersionBuilder.cs | 83 ++++++------ src/SIL.Harmony/ModelVersionService.cs | 6 +- 12 files changed, 150 insertions(+), 160 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e937e43..034437c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ Harmony is the substrate other apps sync through, so a bug here corrupts every c - **Hash chain.** `Commit.Hash` chains over `ParentHash` in commit order; `AlwaysValidateCommits` (on in tests) checks it after every write. - **Serialized changes are forever.** Commits written by old clients must still deserialize. Keep every change type's `$type` discriminator and JSON property names stable; a rename or removal needs a migration path. `PeekThenConcreteChangeConverter` owns `IChange` discrimination and needs `$type` first; unknown types become `OpaqueChange` when configured. - **JSON options come from `HarmonyConfig.JsonSerializerOptions`**, never a fresh `JsonSerializerOptions`, so registered change and object types resolve. -- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type at its next change version (1 for a new type); options validation fails while a registered type is unlisted. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Harmony's own change types aren't versioned yet: changing how one applies needs sillsdev/harmony#141 first. Listing a type invalidates the changes authored with its new change version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. +- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type; a change is stored with the number of the latest version listing its type, and options validation fails while a registered type is unlisted. Listing a type invalidates changes authored at the new version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Harmony's own change types aren't versioned yet: changing how one applies needs sillsdev/harmony#141 first. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. ## Review diff --git a/README.md b/README.md index d1cb85c..33a4b95 100644 --- a/README.md +++ b/README.md @@ -179,37 +179,35 @@ await dataModel.SyncWith(remoteModel); It's that easy. All the heavy lifting is done by the interface which is fairly simple to implement. ### Model versions -Clients on different app versions sync with each other. When a newer app adds a change type, an older app stores those changes as `OpaqueChange` (with `UnknownChangeHandling.Fallback`) and skips them. When a newer app changes how a change type applies, an older app applies those changes with its old code. In both cases the snapshots must be rebuilt when the older app upgrades. +Clients on different app versions sync with each other, so an older app receives changes a newer app authored. It skips the ones whose type it doesn't know (stored as `OpaqueChange` with `UnknownChangeHandling.Fallback`) and applies the rest with its old code. When the older app upgrades, those changes must be replayed. -To make this possible, model versions are required. Number them 0, 1, 2 and so on. Version 0 lists every change type that existed before model versions, with change version 0. After that, add a version each time a release adds a change type, or changes how one applies, listing the type with its next change version: 1 for a new type, then one more each time it is listed. Startup fails if no versions are declared, or if a registered change type isn't listed by any version. Only add versions at the end, never modify a released version: +Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. A change is stored with the number of the latest version that lists its type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared or a registered change type isn't listed. Only add versions at the end, never modify a released one: ```C# services.AddCrdtData(config => { config.ModelVersionBuilder .Add(0, "Before model versions", v => v - .Change(0) - .Change(0) + .Change() + .Change() //the change types AddCrdtRemoteResources registers - .RemoteResourceChanges(0)) - .Add(1, "Word notes", v => v.Change(1)) + .RemoteResourceChanges()) + .Add(1, "Word notes", v => v.Change()) //a major version regenerates all snapshots on upgrade - .AddMajor(2, "Reference rewrite", v => v.Change(1)); + .AddMajor(2, "Reference rewrite", v => v.Change()); }); ``` -Every change is stored with the version of its type when it was authored: the version the latest model version that lists the type gave it. - -Call `ReconcileModelVersions` when you open a database, before you use it. A database that has no model versions stored yet is at version 0. It compares the model versions stored in the database with the current ones. For each change type that a new model version lists, it invalidates the changes of that type with the listed version or higher, the ones authored by a newer app than the one that applied them, and rebuilds the snapshots from the checkpoint before the oldest of them. A major version, or a downgrade, regenerates all snapshots. It throws if a model version that was already applied to the database was modified. Querying changes by type is supported on SQLite only. +Call `ReconcileModelVersions` when you open a database, before you use it. A database with nothing stored is at version 0. For each type listed by a version newer than the stored one, it invalidates the changes authored with that version or later, and rebuilds the snapshots from the checkpoint before the oldest of them. A major version, or a downgrade, regenerates all snapshots. It throws if an applied version was modified. Querying changes by type is supported on SQLite only. ```C# await dataModel.ReconcileModelVersions(); ``` -Listing a type invalidates only the changes authored with its new change version or later, the ones from apps on the new version. When the new behavior must also apply to older changes, pass `invalidateFrom` with the change version to invalidate from. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the previous number: +Listing a type invalidates only the changes authored by apps on the new version or later. When the new behavior must also apply to older changes, pass `invalidateFrom`. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the earlier number: ```C# config.ModelVersionBuilder - .Add(0, "Before model versions", v => v.Change(0).Change(0)) - .Add(1, "Word notes", v => v.Change(1)) + .Add(0, "Before model versions", v => v.Change().Change()) + .Add(1, "Word notes", v => v.Change()) //changed how SetTagChange applies but forgot to list it, so tags are still authored as version 0 - .Add(2, "Notes again", v => v.Change(2)) - .Add(3, "Fix tags", v => v.Change(1, invalidateFrom: 0)); + .Add(2, "Notes again", v => v.Change()) + .Add(3, "Fix tags", v => v.Change(invalidateFrom: 0)); ``` To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index 1085a78..ded7fb5 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -14,8 +14,8 @@ public ChangeEntity() public required Guid EntityId { get; set; } public required TChange Change { get; set; } /// - /// The version of the change type when the change was authored: the version the latest model version that lists - /// its type gave it. Changes from before model versions existed are version 0. + /// The model version the change was authored under: the number of the latest model version that listed its type + /// in the authoring app. Changes from before model versions existed are version 0. /// public int ChangeVersion { get; set; } } diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index e8810d0..210dd38 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -64,24 +64,24 @@ public static void ConfigureSample(HarmonyConfig config) ; config.ModelVersionBuilder .Add(0, "Before model versions", v => v - .Change(0) - .Change(0) - .Change(0) - .Change(0) - .Change(0) - .Change(0) - .Change(0) - .Change(0) - .Change>(0) - .Change(0) - .Change(0) - .Change(0) - .Change>(0) - .Change>(0) - .Change>(0) - .Change>(0) + .Change() + .Change() + .Change() + .Change() + .Change() + .Change() + .Change() + .Change() + .Change>() + .Change() + .Change() + .Change() + .Change>() + .Change>() + .Change>() + .Change>() //registered by AddCrdtRemoteResources - .RemoteResourceChanges(0)); + .RemoteResourceChanges()); config.ObjectTypeListBuilder.DefaultAdapter() .Add(builder => { diff --git a/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs b/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs index 465002d..52e327e 100644 --- a/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs +++ b/src/SIL.Harmony.Tests/Adapter/CustomObjectAdapterTests.cs @@ -156,8 +156,8 @@ public async Task CanAdaptACustomObject() config.AlwaysValidateCommits = true; config.ChangeTypeListBuilder.Add().Add(); config.ModelVersionBuilder.Add(0, "Before model versions", v => v - .Change(0) - .Change(0)); + .Change() + .Change()); config.ObjectTypeListBuilder .CustomAdapter() .Add(builder => builder.HasKey(o => o.Identifier)) diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt index 82cbb8c..bbb8645 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt +++ b/src/SIL.Harmony.Tests/ModelVersionTests.DescribeModelVersions.verified.txt @@ -1,5 +1,5 @@ -0 Before model versions: AddWordImageChange 0, EditExampleChange 0, NewDefinitionChange 0, NewExampleChange 0, NewWordChange 0, SetAntonymReferenceChange 0, SetDefinitionPartOfSpeechChange 0, SetTagChange 0, SetWordNoteChange 0, SetWordTextChange 0, TagWordChange 0, create:pendingUpload 0, create:remote-resource 0, delete:Definition 0, delete:Example 0, delete:RemoteResource 0, delete:Tag 0, delete:Word 0, set:remote-resource-metadata 0, setOrder:Definition 0, uploaded:RemoteResource 0 -1 Notes: SetWordNoteChange 1 +0 Before model versions: AddWordImageChange, EditExampleChange, NewDefinitionChange, NewExampleChange, NewWordChange, SetAntonymReferenceChange, SetDefinitionPartOfSpeechChange, SetTagChange, SetWordNoteChange, SetWordTextChange, TagWordChange, create:pendingUpload, create:remote-resource, delete:Definition, delete:Example, delete:RemoteResource, delete:Tag, delete:Word, set:remote-resource-metadata, setOrder:Definition, uploaded:RemoteResource +1 Notes: SetWordNoteChange 2 major Rewrite: -3 Tags and notes: SetTagChange 1, SetWordNoteChange 2 -4 Fix tags: SetTagChange 2 invalidates from 1, SetWordNoteChange 3 +3 Tags and notes: SetTagChange, SetWordNoteChange +4 Fix tags: SetTagChange invalidates from 1, SetWordNoteChange diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index 5dfd16b..f77b280 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -32,9 +32,11 @@ private static void VersionZeroWithout(HarmonyConfig config, params string[] new config.ModelVersionBuilder.Clear(); config.ModelVersionBuilder.Add(0, "Before model versions", v => { - v.RemoteResourceChanges(0); - var others = config.ChangeTypes.Select(ct => ct.Discriminator).Except(v.Changes.Keys).Except(newTypes); - foreach (var discriminator in others) v.Change(discriminator, 0); + v.RemoteResourceChanges(); + var others = config.ChangeTypes.Select(ct => ct.Discriminator) + .Except(v.Changes.Keys) // exclude the resource change types added above + .Except(newTypes); + foreach (var discriminator in others) v.Change(discriminator); }); } @@ -56,7 +58,7 @@ private static void AddingChangeType(IServiceCollection services) where T : I services.Configure(config => { VersionZeroWithout(config, T.TypeName); - config.ModelVersionBuilder.Add(1, "Added " + T.TypeName, v => v.Change(1)); + config.ModelVersionBuilder.Add(1, "Added " + T.TypeName, v => v.Change()); }); } @@ -71,7 +73,7 @@ private static void NewApp(IServiceCollection services) { OldApp(services); services.Configure(config => - config.ModelVersionBuilder.Add(1, "Notes", v => v.Change(1))); + config.ModelVersionBuilder.Add(1, "Notes", v => v.Change())); } /// the sample's version 0, then @@ -149,8 +151,8 @@ private static async Task ChangeVersions(DataModelTestBase app, Commit co public async Task AuthoredChangesGetTheVersionOfTheirType() { await using var app = ForkDatabase(AppWith(versions => versions - .Add(1, "Notes", v => v.Change(1)) - .Add(2, "Notes and tags", v => v.Change(2).Change(1)))); + .Add(1, "Notes", v => v.Change()) + .Add(2, "Notes and tags", v => v.Change().Change()))); var wordId = Guid.NewGuid(); var commit = await app.WriteNextChange([ @@ -159,11 +161,11 @@ public async Task AuthoredChangesGetTheVersionOfTheirType() new SetWordNoteChange(wordId, "note") ]); - commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 1, 2); - (await ChangeVersions(app, commit)).Should().Equal(0, 1, 2); + commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal([0, 2, 2], "the latest version that lists a type"); + (await ChangeVersions(app, commit)).Should().Equal(0, 2, 2); var json = JsonSerializer.Serialize(commit, app.CrdtConfig.JsonSerializerOptions); JsonSerializer.Deserialize(json, app.CrdtConfig.JsonSerializerOptions)! - .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 1, 2); + .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 2, 2); } [Fact] @@ -209,15 +211,15 @@ private static void AppThatForgotNotes(IServiceCollection services) { OldApp(services); services.Configure(config => - config.ModelVersionBuilder.Add(1, "Tags", v => v.Change(1))); + config.ModelVersionBuilder.Add(1, "Tags", v => v.Change())); } private static void AppThatFixedForgottenNotes(IServiceCollection services) { OldApp(services); services.Configure(config => config.ModelVersionBuilder - .Add(1, "Tags", v => v.Change(1)) - .Add(2, "Forgotten notes", v => v.Change(1, invalidateFrom: 0))); + .Add(1, "Tags", v => v.Change()) + .Add(2, "Forgotten notes", v => v.Change(invalidateFrom: 0))); } [Theory] @@ -246,7 +248,7 @@ public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedTo (result.OldestInvalidatedCommit?.Id).Should().Be(forgottenNote.Id); result.FullRegeneration.Should().BeFalse(); (await NoteOf(fixedApp, wordId)).Should().Be("a note"); - fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(1, "notes authored after the fix can be told apart"); + fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(2, "notes authored after the fix can be told apart"); } [Fact] @@ -419,19 +421,19 @@ public async Task ModifiedAppliedVersionThrows() await using var newApp = ForkDatabase(NewApp); await newApp.ModelVersionService.ReconcileModelVersions(); - await using var otherTypes = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change(1)))); + await using var otherTypes = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change()))); var act = () => otherTypes.ModelVersionService.ReconcileModelVersions(); await act.Should().ThrowAsync().WithMessage("Model version 1 (Notes) was modified*"); - await using var nowMajor = newApp.ForkDatabase(AppWith(versions => versions.AddMajor(1, "Notes", v => v.Change(1)))); + await using var nowMajor = newApp.ForkDatabase(AppWith(versions => versions.AddMajor(1, "Notes", v => v.Change()))); act = () => nowMajor.ModelVersionService.ReconcileModelVersions(); await act.Should().ThrowAsync(); - await using var nowInvalidates = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change(1, invalidateFrom: 0)))); + await using var nowInvalidates = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Notes", v => v.Change(invalidateFrom: 0)))); act = () => nowInvalidates.ModelVersionService.ReconcileModelVersions(); await act.Should().ThrowAsync(); - await using var renamed = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Word notes", v => v.Change(1)))); + await using var renamed = newApp.ForkDatabase(AppWith(versions => versions.Add(1, "Word notes", v => v.Change()))); (await renamed.ModelVersionService.ReconcileModelVersions()).Changed.Should().BeFalse("names are not stored"); } @@ -439,10 +441,10 @@ public async Task ModifiedAppliedVersionThrows() public async Task DescribeModelVersions() { await using var app = ForkDatabase(AppWith(versions => versions - .Add(1, "Notes", v => v.Change(1)) + .Add(1, "Notes", v => v.Change()) .AddMajor(2, "Rewrite") - .Add(3, "Tags and notes", v => v.Change(2).Change(1)) - .Add(4, "Fix tags", v => v.Change(2, invalidateFrom: 1).Change(3)))); + .Add(3, "Tags and notes", v => v.Change().Change()) + .Add(4, "Fix tags", v => v.Change(invalidateFrom: 1).Change()))); await Verify(app.CrdtConfig.DescribeModelVersions()); } @@ -455,7 +457,7 @@ public void ModelVersionWithAnUnregisteredChangeTypeFailsValidation() .Configure(config => { config.ChangeTypeListBuilder.Remove(); - config.ModelVersionBuilder.Add(1, "Notes", v => v.Change(1)); + config.ModelVersionBuilder.Add(1, "Notes", v => v.Change()); }) .BuildServiceProvider(); @@ -492,43 +494,43 @@ public void ChangeTypeNotListedByAModelVersionFailsValidation() } [Fact] - public void ChangeVersionsToReplayStartAtTheFirstNewVersion() + public void ChangeVersionsToInvalidateStartAtTheFirstNewVersion() { var versions = new ModelVersionBuilder() - .Add(0, "Before", v => v.Change(0).Change(0)) - .Add(1, "A", v => v.Change(1)) - .Add(2, "B", v => v.Change(1).Change(2)) - .Add(3, "C", v => v.Change(2)) + .Add(0, "Before", v => v.Change().Change()) + .Add(1, "A", v => v.Change()) + .Add(2, "B", v => v.Change().Change()) + .Add(3, "C", v => v.Change()) .Versions; - ModelVersionService.ChangeVersionsToReplay(1, versions) - .Should().BeEquivalentTo([(nameof(SetTagChange), 1), (nameof(SetWordNoteChange), 2)]); - ModelVersionService.ChangeVersionsToReplay(0, versions) - .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 1)], + ModelVersionService.ChangeVersionsToInvalidate(1, versions) + .Should().BeEquivalentTo([(nameof(SetTagChange), 2), (nameof(SetWordNoteChange), 2)]); + ModelVersionService.ChangeVersionsToInvalidate(0, versions) + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 2)], "version 0 is the state before model versions, so its changes were applied as authored"); - ModelVersionService.ChangeVersionsToReplay(3, versions).Should().BeEmpty(); + ModelVersionService.ChangeVersionsToInvalidate(3, versions).Should().BeEmpty(); } [Fact] - public void ChangeVersionsToReplayStartAtTheInvalidatedVersion() + public void ChangeVersionsToInvalidateStartAtTheInvalidatedVersion() { var versions = new ModelVersionBuilder() - .Add(0, "Before", v => v.Change(0).Change(0)) - .Add(1, "A", v => v.Change(1)) + .Add(0, "Before", v => v.Change().Change()) + .Add(1, "A", v => v.Change()) //changed how tags apply, but forgot to list them - .Add(2, "B", v => v.Change(2)) - .Add(3, "C", v => v.Change(1, invalidateFrom: 0).Change(3)) - .Add(4, "D", v => v.Change(2)) + .Add(2, "B", v => v.Change()) + .Add(3, "C", v => v.Change(invalidateFrom: 0).Change()) + .Add(4, "D", v => v.Change()) .Versions; - ModelVersionService.ChangeVersionsToReplay(0, versions) + ModelVersionService.ChangeVersionsToInvalidate(0, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 0)], "a database at 0 applied version 0 notes as authored, but C replays all tags because B forgot them"); - ModelVersionService.ChangeVersionsToReplay(2, versions) + ModelVersionService.ChangeVersionsToInvalidate(2, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 0)], "the apps with versions 0 to 2 authored tags with version 0"); - ModelVersionService.ChangeVersionsToReplay(3, versions) - .Should().BeEquivalentTo([(nameof(SetTagChange), 2)], "the fix was already applied"); + ModelVersionService.ChangeVersionsToInvalidate(3, versions) + .Should().BeEquivalentTo([(nameof(SetTagChange), 4)], "the fix was already applied"); } [Fact] @@ -537,7 +539,7 @@ public void ModelVersionsMustBeNumberedInOrderFromZero() new ModelVersionBuilder().Invoking(b => b.Add(1, "A", _ => { })).Should().Throw(); new ModelVersionBuilder().Invoking(b => b.AddMajor(0, "Before")) .Should().Throw().WithMessage("Model version 0 (Before) can't be major*"); - var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change(0)); + var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change()); builder.Invoking(b => b.Add(2, "B", _ => { })).Should().Throw(); builder.Invoking(b => b.AddMajor(0, "Before again")).Should().Throw(); @@ -545,24 +547,20 @@ public void ModelVersionsMustBeNumberedInOrderFromZero() } [Fact] - public void ChangeVersionsMustBeValid() - { - new ModelVersionBuilder().Invoking(b => b.Add(0, "Before", v => v.Change(1))) - .Should().Throw().WithMessage("*in model version 0 (Before) must be 0*"); - var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change(0).Change(0)); - - builder.Invoking(b => b.Add(1, "A", v => v.Change(0))) - .Should().Throw().WithMessage("*must be 1, one more than its previous version 0"); - builder.Invoking(b => b.Add(1, "A", v => v.Change(2))) - .Should().Throw().WithMessage("*must be 1, one more than its previous version 0"); - builder.Invoking(b => b.Add(1, "A", v => v.Change(-1))).Should().Throw(); - builder.Invoking(b => b.Add(1, "A", v => v.Change(1, invalidateFrom: 2))).Should().Throw(); - builder.Invoking(b => b.Add(1, "A", v => v.Change(1, invalidateFrom: -1))).Should().Throw(); - builder.Invoking(b => b.Add(1, "A", v => v.Change(1).Change(1))) + public void InvalidateFromMustBeThisVersionOrAnEarlierOne() + { + var builder = new ModelVersionBuilder().Add(0, "Before", v => v.Change().Change()); + + builder.Invoking(b => b.Add(1, "A", v => v.Change(invalidateFrom: 2))).Should().Throw(); + builder.Invoking(b => b.Add(1, "A", v => v.Change(invalidateFrom: -1))).Should().Throw(); + builder.Invoking(b => b.Add(1, "A", v => v.Change().Change())) .Should().Throw().WithMessage("*more than once*"); builder.Versions.Should().ContainSingle("a version that failed validation isn't added"); - builder.Add(1, "A", v => v.Change(1).Change(1)) - .Versions.Should().HaveCount(2, "a type new in this version starts at 1"); + builder.Add(1, "A", v => v.Change(invalidateFrom: 0).Change(invalidateFrom: 1).Change()); + builder.Versions[1].Changes.Should().Equal( + new ModelVersionChange(nameof(EditExampleChange), InvalidateFrom: 1), + new ModelVersionChange(nameof(SetTagChange), InvalidateFrom: 0), + new ModelVersionChange(nameof(SetWordNoteChange), InvalidateFrom: 1)); } [Fact] @@ -575,7 +573,7 @@ public void RemoteResourceChangesMustBeListed() config.ModelVersionBuilder.Clear(); config.ModelVersionBuilder.Add(0, "Before model versions", v => { - foreach (var changeType in config.ChangeTypes) v.Change(changeType.Discriminator, 0); + foreach (var changeType in config.ChangeTypes) v.Change(changeType.Discriminator); }); }) .BuildServiceProvider(); diff --git a/src/SIL.Harmony.Tests/PersistExtraDataTests.cs b/src/SIL.Harmony.Tests/PersistExtraDataTests.cs index 597ab7a..34ee0b5 100644 --- a/src/SIL.Harmony.Tests/PersistExtraDataTests.cs +++ b/src/SIL.Harmony.Tests/PersistExtraDataTests.cs @@ -58,7 +58,7 @@ public PersistExtraDataTests() { config.ObjectTypeListBuilder.DefaultAdapter().Add(); config.ChangeTypeListBuilder.Add(); - config.ModelVersionBuilder.Add(1, "Extra data", v => v.Change(1)); + config.ModelVersionBuilder.Add(1, "Extra data", v => v.Change()); config.BeforeSaveObject = (obj, snapshot) => { if (obj is ExtraDataModel extraDataModel) diff --git a/src/SIL.Harmony/Config/HarmonyConfig.cs b/src/SIL.Harmony/Config/HarmonyConfig.cs index 9cd7535..0ce55db 100644 --- a/src/SIL.Harmony/Config/HarmonyConfig.cs +++ b/src/SIL.Harmony/Config/HarmonyConfig.cs @@ -135,7 +135,7 @@ private FrozenDictionary BuildChangeVersions() } /// - /// The version a change of this type is authored with: the version the latest model version that lists the type gave it, + /// The version a change of this type is authored with: the number of the latest model version that lists the type, /// 0 when none lists it. /// /// the change type's discriminator, its $type @@ -165,9 +165,9 @@ public string DescribeModelVersions() .Append(version.Major ? " major " : " ") .Append(version.Name) .Append(": ") - .AppendJoin(", ", version.Changes.Select(c => c.InvalidateFrom == c.Version - ? $"{c.ChangeType} {c.Version}" - : $"{c.ChangeType} {c.Version} invalidates from {c.InvalidateFrom}")) + .AppendJoin(", ", version.Changes.Select(c => c.InvalidateFrom == version.Number + ? c.ChangeType + : $"{c.ChangeType} invalidates from {c.InvalidateFrom}")) .Append('\n'); } return builder.ToString(); diff --git a/src/SIL.Harmony/Config/ModelVersion.cs b/src/SIL.Harmony/Config/ModelVersion.cs index 9ca41d0..15825fd 100644 --- a/src/SIL.Harmony/Config/ModelVersion.cs +++ b/src/SIL.Harmony/Config/ModelVersion.cs @@ -22,11 +22,14 @@ public override int GetHashCode() } } -/// a change type that a model version added or modified +/// +/// A change type that a model version added or modified. Changes of the type are authored with that version's number +/// from then on, see . +/// /// the discriminator of the change type, its $type -/// the version changes of this type are authored with, from this model version on /// -/// on upgrade, changes of this type with this version or higher are replayed. -/// Usually , lower when an earlier model version forgot to list the type. +/// on upgrade, changes of this type authored with this model version or later are invalidated, and the snapshots +/// are rebuilt from the checkpoint before the oldest of them. Usually the version's own number, lower when changes +/// authored by earlier versions must be invalidated too. /// -public sealed record ModelVersionChange(string ChangeType, int Version, int InvalidateFrom); +public sealed record ModelVersionChange(string ChangeType, int InvalidateFrom); diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs index 9e28416..ecd74c5 100644 --- a/src/SIL.Harmony/Config/ModelVersionBuilder.cs +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -6,9 +6,10 @@ namespace SIL.Harmony.Config; /// /// Declares the ordered list of model versions, which is required. Version 0 lists every change type that existed -/// before model versions, with change version 0. Add a version when a release adds a change type, or changes how -/// an existing change type applies. Every registered change type must be listed by a version. -/// Changes authored by a newer app are then replayed when an older app upgrades, see . +/// before model versions. Add a version when a release adds a change type, or changes how an existing change type +/// applies. Every registered change type must be listed by a version. A change is authored with the number of the +/// latest version that lists its type, and changes authored by a newer app are replayed when an older app upgrades, +/// see . /// Released versions must never be modified, use in a snapshot test to make sure. /// public class ModelVersionBuilder @@ -24,7 +25,7 @@ public class ModelVersionBuilder /// change types added with , which must be registered internal IReadOnlyCollection ChangeTypes => _changeTypes; - /// the latest version of each change type listed by a model version + /// for each change type, the number of the latest model version that lists it internal IReadOnlyDictionary ChangeVersions => _changeVersions; /// whether a version called @@ -42,7 +43,7 @@ public ModelVersionBuilder Add(int number, string name, Action /// Adds a version that regenerates all snapshots when upgrading to it. Version 0 can't be major. - /// The listed change types are still authored with their new version. + /// The listed change types are still authored with the new number. /// /// the next number: versions are numbered 0, 1, 2 and so on public ModelVersionBuilder AddMajor(int number, string name, Action? changes = null) @@ -60,30 +61,14 @@ private ModelVersionBuilder Add(int number, string name, bool major, Action c.ChangeType, StringComparer.Ordinal).ToArray())); return this; @@ -112,57 +97,63 @@ private void CheckFrozen() public class ModelVersionChanges { + private readonly int _number; + private readonly ModelVersionBuilder _builder; + + internal ModelVersionChanges(int number, ModelVersionBuilder builder) + { + _number = number; + _builder = builder; + } + internal Dictionary Changes { get; } = new(StringComparer.Ordinal); internal List Types { get; } = []; internal bool RemoteResourcesListed { get; private set; } /// /// The change type was added or modified in this version, or existed before model versions when this is version 0. + /// Changes of the type are authored with this version's number from now on. /// - /// - /// the version changes of this type are authored with from now on. 0 in model version 0, - /// otherwise one more than its previous version, so 1 the first time a new type is listed. - /// /// - /// on upgrade, changes of this type with this version or higher are replayed. Defaults to , - /// so only changes authored by this app or a newer one are replayed. When an earlier, released version modified - /// how the type applies but forgot to list it, pass the version its apps authored the type with, to replay those too. + /// on upgrade, changes of this type authored with this model version or later are invalidated, so only changes + /// authored by this app or a newer one. Pass an earlier version to invalidate the changes its apps authored too: + /// when the new behavior must be applied to older changes, or when that version changed how the type applies + /// but forgot to list it. /// - public ModelVersionChanges Change(int version, int? invalidateFrom = null) where T : IChange, IPolyType + public ModelVersionChanges Change(int? invalidateFrom = null) where T : IChange, IPolyType { - Change(T.TypeName, version, invalidateFrom); + Change(T.TypeName, invalidateFrom); Types.Add(typeof(T)); return this; } /// - /// The change types registered by , all with the same version. + /// The change types registered by . /// Options validation requires it when remote resources are enabled. Harmony doesn't version these types yet, /// see https://github.com/sillsdev/harmony/issues/141. /// - public ModelVersionChanges RemoteResourceChanges(int version, int? invalidateFrom = null) + public ModelVersionChanges RemoteResourceChanges() where TMetadata : class { RemoteResourcesListed = true; - return Change>(version, invalidateFrom) - .Change>(version, invalidateFrom) - .Change>(version, invalidateFrom) - .Change>(version, invalidateFrom) - .Change>(version, invalidateFrom); + return Change>() + .Change>() + .Change>() + .Change>() + .Change>(); } /// /// by discriminator. /// Only for change types that no longer exist in the code, use otherwise. /// - public ModelVersionChanges Change(string discriminator, int version, int? invalidateFrom = null) + public ModelVersionChanges Change(string discriminator, int? invalidateFrom = null) { ArgumentException.ThrowIfNullOrWhiteSpace(discriminator); - ArgumentOutOfRangeException.ThrowIfNegative(version); - var from = invalidateFrom ?? version; + var from = invalidateFrom ?? _number; ArgumentOutOfRangeException.ThrowIfNegative(from, nameof(invalidateFrom)); - ArgumentOutOfRangeException.ThrowIfGreaterThan(from, version, nameof(invalidateFrom)); - if (!Changes.TryAdd(discriminator, new ModelVersionChange(discriminator, version, from))) + ArgumentOutOfRangeException.ThrowIfGreaterThan(from, _number, nameof(invalidateFrom)); + if (!Changes.TryAdd(discriminator, new ModelVersionChange(discriminator, from))) throw new ArgumentException($"{discriminator} is listed more than once in the same model version"); return this; } diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index 08e5b79..b3739f0 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -52,7 +52,7 @@ public async Task ReconcileModelVersions() } else { - oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToReplay(storedVersion, current)); + oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToInvalidate(storedVersion, current)); if (oldestInvalidated is not null) { await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(oldestInvalidated), Config); @@ -90,10 +90,10 @@ private static IEnumerable NewVersions(int storedVersion, IReadOnl } /// - /// For each change type in the versions after , the lowest change version to replay: + /// For each change type in the versions after , the lowest change version to invalidate: /// the lowest those versions give the type. /// - internal static (string ChangeType, int FromVersion)[] ChangeVersionsToReplay(int storedVersion, + internal static (string ChangeType, int FromVersion)[] ChangeVersionsToInvalidate(int storedVersion, IReadOnlyList versions) { return NewVersions(storedVersion, versions) From b42426fddf14f136d3e677971d219fa8dc49fa6d Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 16 Oct 2026 16:36:44 +0200 Subject: [PATCH 20/25] Say in the README that invalidateFrom is a model version and RemoteResourceChanges is required Co-Authored-By: Claude Fable 5.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33a4b95..4d398e9 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ It's that easy. All the heavy lifting is done by the interface which is fairly s ### Model versions Clients on different app versions sync with each other, so an older app receives changes a newer app authored. It skips the ones whose type it doesn't know (stored as `OpaqueChange` with `UnknownChangeHandling.Fallback`) and applies the rest with its old code. When the older app upgrades, those changes must be replayed. -Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. A change is stored with the number of the latest version that lists its type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared or a registered change type isn't listed. Only add versions at the end, never modify a released one: +Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. A change is stored with the number of the latest version that lists its type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared, a registered change type isn't listed, or remote resources are enabled without `RemoteResourceChanges`. Only add versions at the end, never modify a released one: ```C# services.AddCrdtData(config => { @@ -200,7 +200,7 @@ Call `ReconcileModelVersions` when you open a database, before you use it. A dat ```C# await dataModel.ReconcileModelVersions(); ``` -Listing a type invalidates only the changes authored by apps on the new version or later. When the new behavior must also apply to older changes, pass `invalidateFrom`. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the earlier number: +Listing a type invalidates only the changes authored by apps on the new version or later. When the new behavior must also apply to older changes, pass `invalidateFrom` with the model version to invalidate from. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the earlier number: ```C# config.ModelVersionBuilder .Add(0, "Before model versions", v => v.Change().Change()) From be0a839965e3eee6a0b9da057b62fbf80b7e0d48 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Fri, 16 Oct 2026 16:39:53 +0200 Subject: [PATCH 21/25] Take the repository lock for public local-state writes SetLocalState reads the key and then adds it, so two concurrent first writes to one key both added it and the second failed on the primary key. Reconcile already writes under the lock. Co-Authored-By: Claude Fable 5.1 --- src/SIL.Harmony/DataModel.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 655afa8..b09af5c 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -243,7 +243,10 @@ public Task ReconcileModelVersions() /// public async Task SetLocalState(string key, T value) { - await _crdtRepositoryFactory.Execute(repo => repo.SetLocalState(key, value)); + await using var repo = await _crdtRepositoryFactory.CreateRepository(); + //the write is read-then-add, so two first writes to a key would both add it + using var locked = await repo.Lock(); + await repo.SetLocalState(key, value); } /// @@ -252,7 +255,9 @@ public async Task SetLocalState(string key, T value) /// public async Task RemoveLocalState(string key) { - await _crdtRepositoryFactory.Execute(repo => repo.RemoveLocalState(key)); + await using var repo = await _crdtRepositoryFactory.CreateRepository(); + using var locked = await repo.Lock(); + await repo.RemoveLocalState(key); } public async Task GetLatestSnapshotByObjectId(Guid entityId) From a2c7e3e69d5f1198fc9ba8a08e62bc6bb6dadecd Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 19 Oct 2026 10:36:37 +0700 Subject: [PATCH 22/25] Author every change with the app's latest model version A change was stored with the number of the latest model version that listed its type. When a release changed how a type applies but forgot to list it, its apps kept authoring that type with an earlier number, so the version that fixed it couldn't tell their changes from older ones and had to invalidate from that earlier number, replaying changes the old apps had applied correctly. Every change is now stored with the authoring app's latest model version, listed or not. A fix passes invalidateFrom: and replays only the changes its apps and later ones authored. Default replay decisions don't change: a listed type still invalidates changes authored at its version or later, and those carry the same numbers as before. HarmonyConfig.ChangeVersion(discriminator) is removed, ToChangeEntity uses CurrentModelVersion, and the builder keeps only the set of listed types for options validation. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 8 ++--- src/SIL.Harmony.Core/ChangeEntity.cs | 4 +-- src/SIL.Harmony.Tests/ModelVersionTests.cs | 31 ++++++++++--------- src/SIL.Harmony/Config/HarmonyConfig.cs | 29 +---------------- src/SIL.Harmony/Config/ModelVersion.cs | 5 +-- src/SIL.Harmony/Config/ModelVersionBuilder.cs | 28 +++++++---------- .../Config/ModelVersionsValidator.cs | 4 +-- src/SIL.Harmony/DataModel.cs | 2 +- 9 files changed, 40 insertions(+), 73 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 034437c..2a9e396 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,7 +66,7 @@ Harmony is the substrate other apps sync through, so a bug here corrupts every c - **Hash chain.** `Commit.Hash` chains over `ParentHash` in commit order; `AlwaysValidateCommits` (on in tests) checks it after every write. - **Serialized changes are forever.** Commits written by old clients must still deserialize. Keep every change type's `$type` discriminator and JSON property names stable; a rename or removal needs a migration path. `PeekThenConcreteChangeConverter` owns `IChange` discrimination and needs `$type` first; unknown types become `OpaqueChange` when configured. - **JSON options come from `HarmonyConfig.JsonSerializerOptions`**, never a fresh `JsonSerializerOptions`, so registered change and object types resolve. -- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type; a change is stored with the number of the latest version listing its type, and options validation fails while a registered type is unlisted. Listing a type invalidates changes authored at the new version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Harmony's own change types aren't versioned yet: changing how one applies needs sillsdev/harmony#141 first. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. +- **Model versions are append-only.** Adding a change type, or changing how one applies, means appending a model version to `ModelVersionBuilder` that lists the type; a change is stored with the number of the authoring app's latest version, listed or not, and options validation fails while a registered type is unlisted. Listing a type invalidates changes authored at the new version or later, and the snapshots are rebuilt from the checkpoint before the oldest of them; pass `invalidateFrom` when older changes must be invalidated too, including after a release that changed a type without listing it. A released version, the sample's version 0 included, stays untouched: reconcile throws on an edited one. Harmony's own change types aren't versioned yet: changing how one applies needs sillsdev/harmony#141 first. A test that needs a different history calls the internal `ModelVersionBuilder.Clear()`, as `ModelVersionTests` does. ## Review diff --git a/README.md b/README.md index 4d398e9..8972c24 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ It's that easy. All the heavy lifting is done by the interface which is fairly s ### Model versions Clients on different app versions sync with each other, so an older app receives changes a newer app authored. It skips the ones whose type it doesn't know (stored as `OpaqueChange` with `UnknownChangeHandling.Fallback`) and applies the rest with its old code. When the older app upgrades, those changes must be replayed. -Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. A change is stored with the number of the latest version that lists its type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared, a registered change type isn't listed, or remote resources are enabled without `RemoteResourceChanges`. Only add versions at the end, never modify a released one: +Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. Every change is stored with the number of the authoring app's latest version, whether or not that version lists the change's type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared, a registered change type isn't listed, or remote resources are enabled without `RemoteResourceChanges`. Only add versions at the end, never modify a released one: ```C# services.AddCrdtData(config => { @@ -200,14 +200,14 @@ Call `ReconcileModelVersions` when you open a database, before you use it. A dat ```C# await dataModel.ReconcileModelVersions(); ``` -Listing a type invalidates only the changes authored by apps on the new version or later. When the new behavior must also apply to older changes, pass `invalidateFrom` with the model version to invalidate from. That covers a released version that changed how a type applies but forgot to list it, since its apps kept authoring the type with the earlier number: +Listing a type invalidates only the changes authored by apps on the new version or later. When the new behavior must also apply to older changes, pass `invalidateFrom` with the model version to invalidate from. That covers a released version that changed how a type applies but forgot to list it: its apps still stamped their changes with its own number, so invalidating from that version replays only the changes those apps and later ones authored: ```C# config.ModelVersionBuilder .Add(0, "Before model versions", v => v.Change().Change()) .Add(1, "Word notes", v => v.Change()) - //changed how SetTagChange applies but forgot to list it, so tags are still authored as version 0 + //changed how SetTagChange applies but forgot to list it, its apps authored tags as version 2 .Add(2, "Notes again", v => v.Change()) - .Add(3, "Fix tags", v => v.Change(invalidateFrom: 0)); + .Add(3, "Fix tags", v => v.Change(invalidateFrom: 2)); ``` To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index ded7fb5..1dcbe88 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -14,8 +14,8 @@ public ChangeEntity() public required Guid EntityId { get; set; } public required TChange Change { get; set; } /// - /// The model version the change was authored under: the number of the latest model version that listed its type - /// in the authoring app. Changes from before model versions existed are version 0. + /// The model version the change was authored under: the number of the authoring app's latest model version, + /// whether or not that version lists the change's type. Changes from before model versions existed are version 0. /// public int ChangeVersion { get; set; } } diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index f77b280..e1e794a 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -148,7 +148,7 @@ private static async Task ChangeVersions(DataModelTestBase app, Commit co } [Fact] - public async Task AuthoredChangesGetTheVersionOfTheirType() + public async Task AuthoredChangesGetTheLatestModelVersion() { await using var app = ForkDatabase(AppWith(versions => versions .Add(1, "Notes", v => v.Change()) @@ -161,11 +161,12 @@ public async Task AuthoredChangesGetTheVersionOfTheirType() new SetWordNoteChange(wordId, "note") ]); - commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal([0, 2, 2], "the latest version that lists a type"); - (await ChangeVersions(app, commit)).Should().Equal(0, 2, 2); + commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal([2, 2, 2], + "every change gets the latest version, even NewWordChange, which no later version lists"); + (await ChangeVersions(app, commit)).Should().Equal(2, 2, 2); var json = JsonSerializer.Serialize(commit, app.CrdtConfig.JsonSerializerOptions); JsonSerializer.Deserialize(json, app.CrdtConfig.JsonSerializerOptions)! - .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(0, 2, 2); + .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(2, 2, 2); } [Fact] @@ -219,7 +220,7 @@ private static void AppThatFixedForgottenNotes(IServiceCollection services) OldApp(services); services.Configure(config => config.ModelVersionBuilder .Add(1, "Tags", v => v.Change()) - .Add(2, "Forgotten notes", v => v.Change(invalidateFrom: 0))); + .Add(2, "Forgotten notes", v => v.Change(invalidateFrom: 1))); } [Theory] @@ -230,8 +231,10 @@ public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedTo await DataModel.ReconcileModelVersions(); var wordId = Guid.NewGuid(); await WriteNextChange(new NewWordChange(wordId, "hello")); - //the app that forgot notes authors them with version 0, the same as the old app - var forgottenNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "a note"), version: 0); + var oldNote = await WriteNextChange(new SetWordNoteChange(wordId, "old note")); + (await ChangeVersions(this, oldNote)).Should().Equal(0); + //the app that forgot notes authors them with its own version, 1, though it doesn't list them + var forgottenNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "a note"), version: 1); var beforeFix = (DataModelTestBase)this; await using var upgraded = upgradedToTheAppThatForgot ? ForkDatabase(AppThatForgotNotes) : null; @@ -245,10 +248,10 @@ public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedTo await using var fixedApp = beforeFix.ForkDatabase(AppThatFixedForgottenNotes); var result = await fixedApp.ModelVersionService.ReconcileModelVersions(); - (result.OldestInvalidatedCommit?.Id).Should().Be(forgottenNote.Id); + (result.OldestInvalidatedCommit?.Id).Should().Be(forgottenNote.Id, + "the old app authored the old note and applied it the old way, so only notes from the app that forgot are replayed"); result.FullRegeneration.Should().BeFalse(); (await NoteOf(fixedApp, wordId)).Should().Be("a note"); - fixedApp.CrdtConfig.ChangeVersion(NoteChangeType).Should().Be(2, "notes authored after the fix can be told apart"); } [Fact] @@ -519,16 +522,16 @@ public void ChangeVersionsToInvalidateStartAtTheInvalidatedVersion() .Add(1, "A", v => v.Change()) //changed how tags apply, but forgot to list them .Add(2, "B", v => v.Change()) - .Add(3, "C", v => v.Change(invalidateFrom: 0).Change()) + .Add(3, "C", v => v.Change(invalidateFrom: 2).Change()) .Add(4, "D", v => v.Change()) .Versions; ModelVersionService.ChangeVersionsToInvalidate(0, versions) - .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 0)], - "a database at 0 applied version 0 notes as authored, but C replays all tags because B forgot them"); + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 2)], + "a database at 0 applied version 0 notes as authored, and C replays the tags B's apps authored because B forgot them"); ModelVersionService.ChangeVersionsToInvalidate(2, versions) - .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 0)], - "the apps with versions 0 to 2 authored tags with version 0"); + .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 2)], + "B's apps authored tags with version 2, tags from earlier apps still apply the way they did"); ModelVersionService.ChangeVersionsToInvalidate(3, versions) .Should().BeEquivalentTo([(nameof(SetTagChange), 4)], "the fix was already applied"); } diff --git a/src/SIL.Harmony/Config/HarmonyConfig.cs b/src/SIL.Harmony/Config/HarmonyConfig.cs index 0ce55db..38c8e40 100644 --- a/src/SIL.Harmony/Config/HarmonyConfig.cs +++ b/src/SIL.Harmony/Config/HarmonyConfig.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using System.Collections.Frozen; using System.Text; using System.Text.Json; using System.Text.Json.Serialization.Metadata; @@ -53,11 +52,10 @@ public class HarmonyConfig /// public ModelVersionBuilder ModelVersionBuilder { get; } = new(); public IReadOnlyList ModelVersions => ModelVersionBuilder.Versions; - /// the number of the last model version + /// the number of the last model version, which every change this app authors is stored with public int CurrentModelVersion => ModelVersions.Count == 0 ? throw new InvalidOperationException("No model versions are declared, see " + nameof(ModelVersionBuilder)) : ModelVersions[^1].Number; - private readonly Lazy> _lazyChangeVersions; public ObjectTypeListBuilder ObjectTypeListBuilder { get; } = new(); public IEnumerable ObjectTypes => ObjectTypeListBuilder.AdapterProviders.SelectMany(p => p.GetRegistrations().Select(r => r.ObjectDbType)); public JsonSerializerOptions JsonSerializerOptions => _lazyJsonSerializerOptions.Value; @@ -78,7 +76,6 @@ public class HarmonyConfig public HarmonyConfig() { _lazyChangeDiscriminatorMaps = new Lazy(BuildChangeDiscriminatorMaps); - _lazyChangeVersions = new Lazy>(BuildChangeVersions); _lazyJsonSerializerOptions = new Lazy(CreateJsonSerializerOptions); } @@ -128,30 +125,6 @@ private ChangeDiscriminatorMaps BuildChangeDiscriminatorMaps() return new ChangeDiscriminatorMaps(knownChanges, discriminators); } - private FrozenDictionary BuildChangeVersions() - { - ModelVersionBuilder.Freeze(); - return ModelVersionBuilder.ChangeVersions.ToFrozenDictionary(StringComparer.Ordinal); - } - - /// - /// The version a change of this type is authored with: the number of the latest model version that lists the type, - /// 0 when none lists it. - /// - /// the change type's discriminator, its $type - public int ChangeVersion(string discriminator) - { - return _lazyChangeVersions.Value.GetValueOrDefault(discriminator); - } - - internal int ChangeVersion(IChange change) - { - var discriminator = change is OpaqueChange opaque - ? opaque.TypeName - : _lazyChangeDiscriminatorMaps.Value.ByType.GetValueOrDefault(change.GetType()); - return discriminator is null ? 0 : ChangeVersion(discriminator); - } - /// /// A stable text form of , one line per version. /// Use it in a snapshot test so a released version can't be modified by accident: the snapshot may only grow at the end. diff --git a/src/SIL.Harmony/Config/ModelVersion.cs b/src/SIL.Harmony/Config/ModelVersion.cs index 15825fd..c580d76 100644 --- a/src/SIL.Harmony/Config/ModelVersion.cs +++ b/src/SIL.Harmony/Config/ModelVersion.cs @@ -22,10 +22,7 @@ public override int GetHashCode() } } -/// -/// A change type that a model version added or modified. Changes of the type are authored with that version's number -/// from then on, see . -/// +/// a change type that a model version added or modified /// the discriminator of the change type, its $type /// /// on upgrade, changes of this type authored with this model version or later are invalidated, and the snapshots diff --git a/src/SIL.Harmony/Config/ModelVersionBuilder.cs b/src/SIL.Harmony/Config/ModelVersionBuilder.cs index ecd74c5..a8a0973 100644 --- a/src/SIL.Harmony/Config/ModelVersionBuilder.cs +++ b/src/SIL.Harmony/Config/ModelVersionBuilder.cs @@ -8,8 +8,8 @@ namespace SIL.Harmony.Config; /// Declares the ordered list of model versions, which is required. Version 0 lists every change type that existed /// before model versions. Add a version when a release adds a change type, or changes how an existing change type /// applies. Every registered change type must be listed by a version. A change is authored with the number of the -/// latest version that lists its type, and changes authored by a newer app are replayed when an older app upgrades, -/// see . +/// latest version, whether or not it lists the change's type, and changes authored by a newer app are replayed when +/// an older app upgrades, see . /// Released versions must never be modified, use in a snapshot test to make sure. /// public class ModelVersionBuilder @@ -17,7 +17,7 @@ public class ModelVersionBuilder private bool _frozen; private readonly List _versions = []; private readonly HashSet _changeTypes = []; - private readonly Dictionary _changeVersions = new(StringComparer.Ordinal); + private readonly HashSet _listedChangeTypes = new(StringComparer.Ordinal); private bool _remoteResourcesListed; public IReadOnlyList Versions => _versions.AsReadOnly(); @@ -25,8 +25,8 @@ public class ModelVersionBuilder /// change types added with , which must be registered internal IReadOnlyCollection ChangeTypes => _changeTypes; - /// for each change type, the number of the latest model version that lists it - internal IReadOnlyDictionary ChangeVersions => _changeVersions; + /// the discriminators of the change types listed by a model version + internal IReadOnlySet ListedChangeTypes => _listedChangeTypes; /// whether a version called internal bool RemoteResourcesListed => _remoteResourcesListed; @@ -43,7 +43,6 @@ public ModelVersionBuilder Add(int number, string name, Action /// Adds a version that regenerates all snapshots when upgrading to it. Version 0 can't be major. - /// The listed change types are still authored with the new number. /// /// the next number: versions are numbered 0, 1, 2 and so on public ModelVersionBuilder AddMajor(int number, string name, Action? changes = null) @@ -61,12 +60,9 @@ private ModelVersionBuilder Add(int number, string name, bool major, Action Changes { get; } = new(StringComparer.Ordinal); @@ -112,13 +106,13 @@ internal ModelVersionChanges(int number, ModelVersionBuilder builder) /// /// The change type was added or modified in this version, or existed before model versions when this is version 0. - /// Changes of the type are authored with this version's number from now on. /// /// /// on upgrade, changes of this type authored with this model version or later are invalidated, so only changes /// authored by this app or a newer one. Pass an earlier version to invalidate the changes its apps authored too: /// when the new behavior must be applied to older changes, or when that version changed how the type applies - /// but forgot to list it. + /// but forgot to list it. Every change is authored with the app's latest version, so passing the version that + /// forgot invalidates only the changes its apps and later ones authored. /// public ModelVersionChanges Change(int? invalidateFrom = null) where T : IChange, IPolyType { diff --git a/src/SIL.Harmony/Config/ModelVersionsValidator.cs b/src/SIL.Harmony/Config/ModelVersionsValidator.cs index 3424312..c4106a2 100644 --- a/src/SIL.Harmony/Config/ModelVersionsValidator.cs +++ b/src/SIL.Harmony/Config/ModelVersionsValidator.cs @@ -20,10 +20,10 @@ public ValidateOptionsResult Validate(string? name, HarmonyConfig options) return ValidateOptionsResult.Fail( $"Call {nameof(ModelVersionChanges.RemoteResourceChanges)}<{options.RemoteResourceMetadataType!.Name}> in a model version, " + "it lists the change types AddCrdtRemoteResources registers"); - var listed = options.ModelVersionBuilder.ChangeVersions; + var listed = options.ModelVersionBuilder.ListedChangeTypes; var unlisted = options.ChangeTypes .Select(ct => ct.Discriminator) - .Where(discriminator => !listed.ContainsKey(discriminator)) + .Where(discriminator => !listed.Contains(discriminator)) .Order(StringComparer.Ordinal) .ToArray(); if (unlisted.Length > 0) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index b09af5c..a25fe81 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -130,7 +130,7 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, CommitId = commitId, EntityId = change.EntityId, Index = index, - ChangeVersion = config.ChangeVersion(change) + ChangeVersion = config.CurrentModelVersion }; } From 93db9feceafda81ded187918b12fad6a0ed0d9dd Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 19 Oct 2026 15:18:08 +0700 Subject: [PATCH 23/25] Rename ChangeEntity.ChangeVersion to AuthoredModelVersion The value is the authoring app's latest model version, not a version of the change type, and ChangeVersion read like a property of the Change payload. The JSON property and the ChangeEntities column are renamed with it, which is free while nothing has stored the old name. FindOldestCommitWithChangeVersions and ChangeVersionsToInvalidate follow as FindOldestCommitWithModelVersions and ModelVersionsToInvalidate. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/SIL.Harmony.Core/ChangeEntity.cs | 2 +- .../DbContextTests.VerifyModel.verified.txt | 6 +-- src/SIL.Harmony.Tests/ModelVersionTests.cs | 52 +++++++++---------- src/SIL.Harmony/DataModel.cs | 2 +- src/SIL.Harmony/Db/CrdtRepository.cs | 16 +++--- .../Db/EntityConfig/ChangeEntityConfig.cs | 2 +- src/SIL.Harmony/ModelVersionService.cs | 6 +-- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 8972c24..5ddb26b 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ config.ModelVersionBuilder To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. > [!NOTE] -> Model versions add a `ChangeVersion` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. +> Model versions add a `AuthoredModelVersion` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. ## Development diff --git a/src/SIL.Harmony.Core/ChangeEntity.cs b/src/SIL.Harmony.Core/ChangeEntity.cs index 1dcbe88..54c7df4 100644 --- a/src/SIL.Harmony.Core/ChangeEntity.cs +++ b/src/SIL.Harmony.Core/ChangeEntity.cs @@ -17,5 +17,5 @@ public ChangeEntity() /// The model version the change was authored under: the number of the authoring app's latest model version, /// whether or not that version lists the change's type. Changes from before model versions existed are version 0. /// - public int ChangeVersion { get; set; } + public int AuthoredModelVersion { get; set; } } diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 2a16464..15699bd 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -36,12 +36,12 @@ Properties: CommitId (Guid) Required PK FK AfterSave:Throw Index (int) Required PK AfterSave:Throw + AuthoredModelVersion (int) Required ValueGenerated.OnAdd + Annotations: + Relational:DefaultValue: 0 Change (IChange) Annotations: Relational:ColumnType: jsonb - ChangeVersion (int) Required ValueGenerated.OnAdd - Annotations: - Relational:DefaultValue: 0 EntityId (Guid) Required Keys: CommitId, Index PK diff --git a/src/SIL.Harmony.Tests/ModelVersionTests.cs b/src/SIL.Harmony.Tests/ModelVersionTests.cs index e1e794a..9e27781 100644 --- a/src/SIL.Harmony.Tests/ModelVersionTests.cs +++ b/src/SIL.Harmony.Tests/ModelVersionTests.cs @@ -102,11 +102,11 @@ private static IChange OpaqueNote(DataModelTestBase oldApp, Guid wordId, string return change.Should().BeOfType().Subject; } - /// a commit authored by another client, with the change version that client's app gave it + /// a commit authored by another client, with the model version that client's app authored it with private static async Task SyncFromOtherClient(DataModelTestBase app, IChange change, int version) { var commit = BuildCommit(Guid.NewGuid(), app.NextDate(), [change]); - commit.ChangeEntities.Single().ChangeVersion = version; + commit.ChangeEntities.Single().AuthoredModelVersion = version; await app.AddCommitsViaSync([commit]); return commit; } @@ -138,12 +138,12 @@ private static async Task ShouldBeRegenerated(DataModelTestBase app, Guid[] snap snapshotIdsAfter.Should().NotIntersectWith(snapshotIdsBefore, "every snapshot should be recreated"); } - private static async Task ChangeVersions(DataModelTestBase app, Commit commit) + private static async Task AuthoredModelVersions(DataModelTestBase app, Commit commit) { return await app.DbContext.Set>().AsNoTracking() .Where(ce => ce.CommitId == commit.Id) .OrderBy(ce => ce.Index) - .Select(ce => ce.ChangeVersion) + .Select(ce => ce.AuthoredModelVersion) .ToArrayAsync(TestContext.Current.CancellationToken); } @@ -161,12 +161,12 @@ public async Task AuthoredChangesGetTheLatestModelVersion() new SetWordNoteChange(wordId, "note") ]); - commit.ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal([2, 2, 2], + commit.ChangeEntities.Select(ce => ce.AuthoredModelVersion).Should().Equal([2, 2, 2], "every change gets the latest version, even NewWordChange, which no later version lists"); - (await ChangeVersions(app, commit)).Should().Equal(2, 2, 2); + (await AuthoredModelVersions(app, commit)).Should().Equal(2, 2, 2); var json = JsonSerializer.Serialize(commit, app.CrdtConfig.JsonSerializerOptions); JsonSerializer.Deserialize(json, app.CrdtConfig.JsonSerializerOptions)! - .ChangeEntities.Select(ce => ce.ChangeVersion).Should().Equal(2, 2, 2); + .ChangeEntities.Select(ce => ce.AuthoredModelVersion).Should().Equal(2, 2, 2); } [Fact] @@ -195,7 +195,7 @@ public async Task ModifiedTypeReplaysFromTheFirstChangeAuthoredByANewerApp() await WriteNextChange(new NewWordChange(wordId, "hello")); var oldNote = await WriteNextChange(new SetWordNoteChange(wordId, "old note")); var newerNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "newer note"), version: 1); - (await ChangeVersions(this, oldNote)).Should().Equal(0); + (await AuthoredModelVersions(this, oldNote)).Should().Equal(0); (await NoteOf(this, wordId)).Should().Be("newer note", "the old app applies a newer app's change with its own code"); await using var newApp = ForkDatabase(NewApp); @@ -232,7 +232,7 @@ public async Task ForgottenTypeIsReplayedByTheVersionThatFixesIt(bool upgradedTo var wordId = Guid.NewGuid(); await WriteNextChange(new NewWordChange(wordId, "hello")); var oldNote = await WriteNextChange(new SetWordNoteChange(wordId, "old note")); - (await ChangeVersions(this, oldNote)).Should().Equal(0); + (await AuthoredModelVersions(this, oldNote)).Should().Equal(0); //the app that forgot notes authors them with its own version, 1, though it doesn't list them var forgottenNote = await SyncFromOtherClient(this, new SetWordNoteChange(wordId, "a note"), version: 1); @@ -497,7 +497,7 @@ public void ChangeTypeNotListedByAModelVersionFailsValidation() } [Fact] - public void ChangeVersionsToInvalidateStartAtTheFirstNewVersion() + public void ModelVersionsToInvalidateStartAtTheFirstNewVersion() { var versions = new ModelVersionBuilder() .Add(0, "Before", v => v.Change().Change()) @@ -506,16 +506,16 @@ public void ChangeVersionsToInvalidateStartAtTheFirstNewVersion() .Add(3, "C", v => v.Change()) .Versions; - ModelVersionService.ChangeVersionsToInvalidate(1, versions) + ModelVersionService.ModelVersionsToInvalidate(1, versions) .Should().BeEquivalentTo([(nameof(SetTagChange), 2), (nameof(SetWordNoteChange), 2)]); - ModelVersionService.ChangeVersionsToInvalidate(0, versions) + ModelVersionService.ModelVersionsToInvalidate(0, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 2)], "version 0 is the state before model versions, so its changes were applied as authored"); - ModelVersionService.ChangeVersionsToInvalidate(3, versions).Should().BeEmpty(); + ModelVersionService.ModelVersionsToInvalidate(3, versions).Should().BeEmpty(); } [Fact] - public void ChangeVersionsToInvalidateStartAtTheInvalidatedVersion() + public void ModelVersionsToInvalidateStartAtTheInvalidatedVersion() { var versions = new ModelVersionBuilder() .Add(0, "Before", v => v.Change().Change()) @@ -526,13 +526,13 @@ public void ChangeVersionsToInvalidateStartAtTheInvalidatedVersion() .Add(4, "D", v => v.Change()) .Versions; - ModelVersionService.ChangeVersionsToInvalidate(0, versions) + ModelVersionService.ModelVersionsToInvalidate(0, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 1), (nameof(SetTagChange), 2)], "a database at 0 applied version 0 notes as authored, and C replays the tags B's apps authored because B forgot them"); - ModelVersionService.ChangeVersionsToInvalidate(2, versions) + ModelVersionService.ModelVersionsToInvalidate(2, versions) .Should().BeEquivalentTo([(nameof(SetWordNoteChange), 3), (nameof(SetTagChange), 2)], "B's apps authored tags with version 2, tags from earlier apps still apply the way they did"); - ModelVersionService.ChangeVersionsToInvalidate(3, versions) + ModelVersionService.ModelVersionsToInvalidate(3, versions) .Should().BeEquivalentTo([(nameof(SetTagChange), 4)], "the fix was already applied"); } @@ -588,7 +588,7 @@ public void RemoteResourceChangesMustBeListed() } [Fact] - public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() + public async Task FindOldestCommitWithModelVersionsUsesCommitOrderAndVersion() { var wordId = Guid.NewGuid(); var tagId = Guid.NewGuid(); @@ -598,17 +598,17 @@ public async Task FindOldestCommitWithChangeVersionsUsesCommitOrderAndVersion() var tagCommit = await SyncFromOtherClient(this, new SetTagChange(tagId, "tag"), version: 2); //added last, but sorts before the other version 1 note var earlierNote = CommitBefore(laterNote, new SetWordNoteChange(wordId, "earlier")); - earlierNote.ChangeEntities.Single().ChangeVersion = 1; + earlierNote.ChangeEntities.Single().AuthoredModelVersion = 1; await AddCommitsViaSync([earlierNote]); await using var repo = CreateRepository(); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 0)]))!.Id.Should().Be(versionZeroNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(NoteChangeType, 2)])).Should().BeNull(); - (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 2)]))!.Id.Should().Be(tagCommit.Id); - (await repo.FindOldestCommitWithChangeVersions([(nameof(SetTagChange), 2), (NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); - (await repo.FindOldestCommitWithChangeVersions([(nameof(EditExampleChange), 1)])).Should().BeNull(); - (await repo.FindOldestCommitWithChangeVersions([])).Should().BeNull(); + (await repo.FindOldestCommitWithModelVersions([(NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithModelVersions([(NoteChangeType, 0)]))!.Id.Should().Be(versionZeroNote.Id); + (await repo.FindOldestCommitWithModelVersions([(NoteChangeType, 2)])).Should().BeNull(); + (await repo.FindOldestCommitWithModelVersions([(nameof(SetTagChange), 2)]))!.Id.Should().Be(tagCommit.Id); + (await repo.FindOldestCommitWithModelVersions([(nameof(SetTagChange), 2), (NoteChangeType, 1)]))!.Id.Should().Be(earlierNote.Id); + (await repo.FindOldestCommitWithModelVersions([(nameof(EditExampleChange), 1)])).Should().BeNull(); + (await repo.FindOldestCommitWithModelVersions([])).Should().BeNull(); } [Fact] diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index a25fe81..2684447 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -130,7 +130,7 @@ internal static ChangeEntity ToChangeEntity(IChange change, int index, CommitId = commitId, EntityId = change.EntityId, Index = index, - ChangeVersion = config.CurrentModelVersion + AuthoredModelVersion = config.CurrentModelVersion }; } diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 8a88d99..bf385a9 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -393,28 +393,28 @@ public async Task ReplayWindowFrom(Commit commit) } /// - /// The oldest commit with a change whose $type is one of - /// and whose is at least the version given for that type. + /// The oldest commit with a change whose $type is one of + /// and whose is at least the model version given for that type. /// Reads every change, so it's slow, only use it for rare operations. SQLite only. /// - public async Task FindOldestCommitWithChangeVersions( - IReadOnlyCollection<(string ChangeType, int FromVersion)> changeVersions) + public async Task FindOldestCommitWithModelVersions( + IReadOnlyCollection<(string ChangeType, int FromVersion)> fromVersions) { - if (changeVersions.Count == 0) return null; + if (fromVersions.Count == 0) return null; if (!_dbContext.Database.IsSqlite()) throw new NotSupportedException( $"Querying changes by type is only supported on SQLite, not {_dbContext.Database.ProviderName}"); //the key must be quoted in the path, because $ has a special meaning in json paths var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; //one json array parameter, so the parameter count doesn't depend on how many types there are - var changeVersionsJson = JsonSerializer.Serialize(changeVersions.Select(cv => new { t = cv.ChangeType, v = cv.FromVersion })); + var fromVersionsJson = JsonSerializer.Serialize(fromVersions.Select(cv => new { t = cv.ChangeType, v = cv.FromVersion })); var commitIds = await _dbContext.Database.SqlQuery($""" SELECT c.Id AS Value FROM Commits c WHERE EXISTS ( - SELECT 1 FROM ChangeEntities ce, json_each({changeVersionsJson}) cv + SELECT 1 FROM ChangeEntities ce, json_each({fromVersionsJson}) cv WHERE ce.CommitId = c.Id AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(cv.value, '$.t') - AND ce.ChangeVersion >= json_extract(cv.value, '$.v') + AND ce.AuthoredModelVersion >= json_extract(cv.value, '$.v') ) ORDER BY c.DateTime, c.Counter, c.Id LIMIT 1 diff --git a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs index 67e7df7..68871f7 100644 --- a/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs +++ b/src/SIL.Harmony/Db/EntityConfig/ChangeEntityConfig.cs @@ -12,7 +12,7 @@ public void Configure(EntityTypeBuilder> builder) { builder.ToTable("ChangeEntities"); builder.HasKey(c => new { c.CommitId, c.Index }); - builder.Property(c => c.ChangeVersion).HasDefaultValue(0); + builder.Property(c => c.AuthoredModelVersion).HasDefaultValue(0); builder.Property(c => c.Change) .HasColumnType("jsonb") .HasConversion( diff --git a/src/SIL.Harmony/ModelVersionService.cs b/src/SIL.Harmony/ModelVersionService.cs index b3739f0..a829a59 100644 --- a/src/SIL.Harmony/ModelVersionService.cs +++ b/src/SIL.Harmony/ModelVersionService.cs @@ -52,7 +52,7 @@ public async Task ReconcileModelVersions() } else { - oldestInvalidated = await repo.FindOldestCommitWithChangeVersions(ChangeVersionsToInvalidate(storedVersion, current)); + oldestInvalidated = await repo.FindOldestCommitWithModelVersions(ModelVersionsToInvalidate(storedVersion, current)); if (oldestInvalidated is not null) { await SnapshotReplay.UpdateSnapshots(repo, await repo.ReplayWindowFrom(oldestInvalidated), Config); @@ -90,10 +90,10 @@ private static IEnumerable NewVersions(int storedVersion, IReadOnl } /// - /// For each change type in the versions after , the lowest change version to invalidate: + /// For each change type in the versions after , the lowest authored model version to invalidate: /// the lowest those versions give the type. /// - internal static (string ChangeType, int FromVersion)[] ChangeVersionsToInvalidate(int storedVersion, + internal static (string ChangeType, int FromVersion)[] ModelVersionsToInvalidate(int storedVersion, IReadOnlyList versions) { return NewVersions(storedVersion, versions) From 1dfa54a45e633b1aec6b55ec2ac629f23e31c4b0 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Mon, 19 Oct 2026 12:43:11 +0200 Subject: [PATCH 24/25] Make the README config examples pass options validation The tutorial registered a change type without declaring model versions, and the model versions example listed types it never registered. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ddb26b..a100fbe 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,10 @@ Once you have created your change types, you need to tell Harmony about them. Ag ```C# services.AddCrdtData(config => { -// add the following line +// add the following lines config.ChangeTypeListBuilder.Add(); + //every change type must be listed by a model version, see Model versions below + config.ModelVersionBuilder.Add(0, "Before model versions", v => v.Change()); config.ObjectTypeListBuilder .Add() .Add() @@ -183,8 +185,13 @@ Clients on different app versions sync with each other, so an older app receives Model versions make that possible. Declare them in order, numbered from 0, each listing the change types it added or changed. Version 0 lists every change type that existed before model versions. Every change is stored with the number of the authoring app's latest version, whether or not that version lists the change's type, so a change authored by a newer app carries a number the older app doesn't have yet. Startup fails if no versions are declared, a registered change type isn't listed, or remote resources are enabled without `RemoteResourceChanges`. Only add versions at the end, never modify a released one: ```C# +services.AddCrdtRemoteResources(); services.AddCrdtData(config => { + config.ChangeTypeListBuilder + .Add() + .Add() + .Add(); config.ModelVersionBuilder .Add(0, "Before model versions", v => v .Change() From 87eddf2658d2d94bbe134d23a4afe81c53266f9d Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Mon, 19 Oct 2026 12:47:26 +0200 Subject: [PATCH 25/25] Fix two leftovers from the AuthoredModelVersion rename Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/SIL.Harmony/Db/CrdtRepository.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a100fbe..5cdaad9 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ config.ModelVersionBuilder To make sure a released version is not modified by accident, add a snapshot test of `HarmonyConfig.DescribeModelVersions()` to your app. A new version only adds a line at the end of the snapshot. > [!NOTE] -> Model versions add a `AuthoredModelVersion` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. +> Model versions add an `AuthoredModelVersion` column to `ChangeEntities` and a `LocalState` table, so an EF migration is required. ## Development diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index bf385a9..ab334c2 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -407,14 +407,14 @@ public async Task ReplayWindowFrom(Commit commit) //the key must be quoted in the path, because $ has a special meaning in json paths var typeDiscriminatorPath = $"$.\"{CrdtConstants.ChangeDiscriminatorProperty}\""; //one json array parameter, so the parameter count doesn't depend on how many types there are - var fromVersionsJson = JsonSerializer.Serialize(fromVersions.Select(cv => new { t = cv.ChangeType, v = cv.FromVersion })); + var fromVersionsJson = JsonSerializer.Serialize(fromVersions.Select(fv => new { t = fv.ChangeType, v = fv.FromVersion })); var commitIds = await _dbContext.Database.SqlQuery($""" SELECT c.Id AS Value FROM Commits c WHERE EXISTS ( - SELECT 1 FROM ChangeEntities ce, json_each({fromVersionsJson}) cv + SELECT 1 FROM ChangeEntities ce, json_each({fromVersionsJson}) fv WHERE ce.CommitId = c.Id - AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(cv.value, '$.t') - AND ce.AuthoredModelVersion >= json_extract(cv.value, '$.v') + AND json_extract(ce.Change, {typeDiscriminatorPath}) = json_extract(fv.value, '$.t') + AND ce.AuthoredModelVersion >= json_extract(fv.value, '$.v') ) ORDER BY c.DateTime, c.Counter, c.Id LIMIT 1