From ed439478717da5dec42b99e30768aa683862aea1 Mon Sep 17 00:00:00 2001 From: Adolfo Marinucci Date: Wed, 26 Aug 2026 11:35:01 +0200 Subject: [PATCH 1/4] Report an aged-out sync anchor as a distinguishable condition A client whose anchor has fallen outside the store's change retention window cannot be served incremental changes, and no amount of retrying will change that: the history it asks for has been cleaned up. SqlServerCT already raised the typed SyncAnchorTooOldException for this, but the condition was still indistinguishable everywhere else - the remaining providers threw a bare InvalidOperationException, and over HTTP it reached the client as an unhandled exception that an application could only report as "unable to synchronize". - Throw SyncAnchorTooOldException from the SqlServer, PostgreSQL, Sqlite and MySql providers instead of InvalidOperationException. These stores keep one shared change journal rather than per-table history, so they get a constructor that carries no table name and a message that does not invent one. MinValidVersion is reported as MIN(ID) - 1, the oldest anchor that can still be resolved, which keeps the requested < min-valid invariant true. - Surface it over HTTP as 410 Gone with an X-CoreSync-Error: anchor-too-old header and a SyncAnchorTooOldError body. 410 is the honest code: the history existed once and is permanently gone, so unlike a 5xx there is nothing to retry. An endpoint filter applies this to every sync endpoint, covering both the download path and the upload guard. - Rebuild the exception client-side from that signal, so callers see the same typed error they would from a local provider, with the table and both versions intact. The header alone identifies the condition, so a body that cannot be parsed does not downgrade it back into a transport failure. - Stop the binary download retry loop from retrying it: three round trips and fifteen seconds to reach an answer that cannot change. This does not attempt automatic recovery - an aged-out client still has to be reinitialized from a fresh snapshot by hand. What changes is that both the server logs and the client can now say so, instead of reporting an anonymous failure. --- .../Implementation/SyncProviderHttpClient.cs | 64 ++++- .../WebApplicationExtensions.cs | 42 +++ src/CoreSync.Http/SyncAnchorTooOldError.cs | 30 ++ src/CoreSync.Http/SyncHttpErrorCodes.cs | 23 ++ src/CoreSync.Http/SyncHttpHeaders.cs | 7 + src/CoreSync.MySql/MySqlSyncProvider.cs | 4 +- .../PostgreSQLSyncProvider.cs | 4 +- src/CoreSync.SqlServer/SqlSyncProvider.cs | 4 +- src/CoreSync.Sqlite/SqliteSyncProvider.cs | 4 +- src/CoreSync.Tests/AnchorTooOldTests.cs | 268 ++++++++++++++++++ .../SqlServerCTResilienceTests.cs | 4 +- src/CoreSync.Tests/SyncTestServer.cs | 6 + src/CoreSync/SyncAnchorTooOldException.cs | 30 +- 13 files changed, 471 insertions(+), 19 deletions(-) create mode 100644 src/CoreSync.Http/SyncAnchorTooOldError.cs create mode 100644 src/CoreSync.Http/SyncHttpErrorCodes.cs create mode 100644 src/CoreSync.Tests/AnchorTooOldTests.cs diff --git a/src/CoreSync.Http.Client/Implementation/SyncProviderHttpClient.cs b/src/CoreSync.Http.Client/Implementation/SyncProviderHttpClient.cs index 2f212fe..9dfe28f 100644 --- a/src/CoreSync.Http.Client/Implementation/SyncProviderHttpClient.cs +++ b/src/CoreSync.Http.Client/Implementation/SyncProviderHttpClient.cs @@ -32,6 +32,43 @@ public SyncProviderHttpClient(IHttpClientFactory httpClientProvider, SyncProvide _options = options; } + /// + /// Throws for an unsuccessful response, rebuilding from the + /// server's 410 Gone signal so that callers see the same typed error they would from a local provider. + /// + /// + /// Used in place of on every sync endpoint: + /// without it an aged-out anchor arrives as a bare and the only + /// thing an application can tell the user is that synchronization failed. + /// + private static async Task EnsureSyncSuccessAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + if (response.StatusCode == HttpStatusCode.Gone && + response.Headers.TryGetValues(SyncHttpHeaders.ErrorCode, out var errorCodes) && + errorCodes.Contains(SyncHttpErrorCodes.AnchorTooOld)) + { + SyncAnchorTooOldError? error = null; + try + { + error = await response.Content.ReadFromJsonAsync(cancellationToken); + } + catch (JsonException) + { + // The header alone identifies the condition; a body we cannot read must not downgrade + // this into an ordinary transport failure. + } + + var requestedVersion = error?.RequestedVersion ?? -1; + var minValidVersion = error?.MinValidVersion ?? -1; + + throw error?.TableName is { } tableName + ? new SyncAnchorTooOldException(tableName, requestedVersion, minValidVersion) + : new SyncAnchorTooOldException(requestedVersion, minValidVersion); + } + + response.EnsureSuccessStatusCode(); + } + public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func? onConflictFunc = null, CancellationToken cancellationToken = default) { var httpClient = _httpClientFactory.CreateClient(_options.HttpClientName ?? Options.DefaultName); @@ -48,8 +85,9 @@ public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSe SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ComputingLocalChanges)); - (await httpClient.PostAsJsonAsync($"/{_options.SyncControllerRoute}/changes-bulk-begin", bulkChangeSet, cancellationToken)) - .EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync( + await httpClient.PostAsJsonAsync($"/{_options.SyncControllerRoute}/changes-bulk-begin", bulkChangeSet, cancellationToken), + cancellationToken); var listOfItemsToUpload = new List(); @@ -74,8 +112,9 @@ public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSe beginUploadItemContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-msgpack"); - (await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary", beginUploadItemContent, cancellationToken)) - .EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync( + await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary", beginUploadItemContent, cancellationToken), + cancellationToken); } else { @@ -85,8 +124,9 @@ public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSe Items = listOfItemsToUpload }), Encoding.UTF8, "application/json"); - (await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item", beginUploadItemContent, cancellationToken)) - .EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync( + await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item", beginUploadItemContent, cancellationToken), + cancellationToken); } SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ApplyChanges, skip / (double)changeSet.Items.Count)); @@ -94,7 +134,7 @@ public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSe var remoteChangeSetResponse = await httpClient.PostAsync( $"{_options.SyncControllerRoute}/changes-bulk-complete{(_options.UseBinaryFormat ? "-binary" : string.Empty)}/{sessionId}", null, cancellationToken); - remoteChangeSetResponse.EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync(remoteChangeSetResponse, cancellationToken); SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ApplyChanges, 1.0)); @@ -115,7 +155,7 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa } var response = await httpClient.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync(response, cancellationToken); var bulkSyncChangeSet = await response.Content.ReadFromJsonAsync(cancellationToken) ?? throw new InvalidOperationException(); @@ -181,7 +221,7 @@ private async Task DownloadBulkChangeSetBinary(BulkSyncChangeSet { var response = await httpClient.GetAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary/{bulkSyncChangeSet.SessionId}/{skip}/{_options.BulkItemSize}", cancellationToken); - response.EnsureSuccessStatusCode(); + await EnsureSyncSuccessAsync(response, cancellationToken); using var s = await response.Content.ReadAsStreamAsync(cancellationToken); var downloadedItems = await CoreSyncMessagePackSerializer.DeserializeAsync(s, cancellationToken); @@ -189,6 +229,12 @@ private async Task DownloadBulkChangeSetBinary(BulkSyncChangeSet items.AddRange(bulkItems); break; } + catch (SyncAnchorTooOldException) + { + // Permanent for this client: the change history it asked for is gone. Retrying + // burns three round trips and fifteen seconds to reach the same answer. + throw; + } catch (Exception) { if (retry == 0) diff --git a/src/CoreSync.Http.Server/WebApplicationExtensions.cs b/src/CoreSync.Http.Server/WebApplicationExtensions.cs index 6b1826b..180c780 100644 --- a/src/CoreSync.Http.Server/WebApplicationExtensions.cs +++ b/src/CoreSync.Http.Server/WebApplicationExtensions.cs @@ -50,6 +50,9 @@ public static void MapCoreSyncHttpServerEndpoints(this IEndpointRouteBuilder web void ConfigureEndpoint(T endpoint, Action? specificAction = null) where T : IEndpointConventionBuilder { + // Added before the caller's own conventions so that an aged-out anchor reaches the client + // as a documented 410 rather than whatever the host makes of an unhandled exception. + endpoint.AddSyncErrorEndpointFilter(); options.AllEndpoints?.Invoke(endpoint); specificAction?.Invoke(endpoint); } @@ -131,6 +134,45 @@ void ConfigureEndpoint(T endpoint, Action? specificAction = null) where T } +internal static class SyncErrorFilterEndpointExtensions +{ + /// + /// Translates into an HTTP 410 Gone carrying the + /// header and a body. + /// + /// + /// 410 is the honest code here: the change history the client asked for existed once and is + /// permanently gone, so unlike a 5xx there is nothing to retry. Without this the condition + /// surfaced as an unhandled exception, which the client could only report as a generic + /// "unable to synchronize". + /// + public static TBuilder AddSyncErrorEndpointFilter(this TBuilder builder) where TBuilder : IEndpointConventionBuilder + { + builder.AddEndpointFilter(async (context, next) => + { + try + { + return await next(context); + } + catch (SyncAnchorTooOldException ex) + { + context.HttpContext.Response.Headers[SyncHttpHeaders.ErrorCode] = SyncHttpErrorCodes.AnchorTooOld; + + return Results.Json( + new SyncAnchorTooOldError + { + TableName = ex.TableName, + RequestedVersion = ex.RequestedVersion, + MinValidVersion = ex.MinValidVersion + }, + statusCode: StatusCodes.Status410Gone); + } + }); + + return builder; + } +} + internal static class EndpointMessagePackFilterEndpointExtensions { public static TBuilder AddMessagePackEndpointFilter(this TBuilder builder) where TBuilder : IEndpointConventionBuilder diff --git a/src/CoreSync.Http/SyncAnchorTooOldError.cs b/src/CoreSync.Http/SyncAnchorTooOldError.cs new file mode 100644 index 0000000..5616687 --- /dev/null +++ b/src/CoreSync.Http/SyncAnchorTooOldError.cs @@ -0,0 +1,30 @@ +namespace CoreSync.Http; + +/// +/// Body of the HTTP 410 Gone response returned when a client's sync anchor has fallen outside the +/// store's change retention window. +/// +/// +/// The wire form of CoreSync.SyncAnchorTooOldException: the client rebuilds that exception +/// from these values, so the condition reaches application code typed rather than as a generic +/// transport failure. +/// +public class SyncAnchorTooOldError +{ + /// + /// Gets or sets the table whose change history no longer covers , + /// or null when the store keeps a single shared change journal. + /// + public string? TableName { get; set; } + + /// + /// Gets or sets the version the client asked for. A negative value means the server holds no + /// anchor at all for the client. + /// + public long RequestedVersion { get; set; } + + /// + /// Gets or sets the oldest version the server can still resolve changes from. + /// + public long MinValidVersion { get; set; } +} diff --git a/src/CoreSync.Http/SyncHttpErrorCodes.cs b/src/CoreSync.Http/SyncHttpErrorCodes.cs new file mode 100644 index 0000000..ddac2d8 --- /dev/null +++ b/src/CoreSync.Http/SyncHttpErrorCodes.cs @@ -0,0 +1,23 @@ +namespace CoreSync.Http; + +/// +/// Machine-readable values carried by the response header. +/// +/// +/// The header lets a client branch on a failure without parsing the response body or matching on +/// message text, and stays readable even when the body is a message pack payload the client did not +/// expect. +/// +public static class SyncHttpErrorCodes +{ + /// + /// The client asked for changes relative to a version the server can no longer resolve, because + /// the change history covering it has been discarded by the store's retention policy. + /// Accompanied by HTTP 410 Gone and a body. + /// + /// + /// This is permanent for the client involved: retrying with the same anchor can never succeed. + /// The client has to be reinitialized from a fresh snapshot. + /// + public const string AnchorTooOld = "anchor-too-old"; +} diff --git a/src/CoreSync.Http/SyncHttpHeaders.cs b/src/CoreSync.Http/SyncHttpHeaders.cs index 81150bf..1f2f759 100644 --- a/src/CoreSync.Http/SyncHttpHeaders.cs +++ b/src/CoreSync.Http/SyncHttpHeaders.cs @@ -16,4 +16,11 @@ public static class SyncHttpHeaders /// The server uses this to detect truncated headers. /// public const string TablesCount = "X-CoreSync-Tables-Count"; + + /// + /// A machine-readable identifier for a failure the client is expected to handle specifically, + /// set on the response alongside the corresponding status code. + /// See for the defined values. + /// + public const string ErrorCode = "X-CoreSync-Error"; } diff --git a/src/CoreSync.MySql/MySqlSyncProvider.cs b/src/CoreSync.MySql/MySqlSyncProvider.cs index bf8cc24..443b349 100644 --- a/src/CoreSync.MySql/MySqlSyncProvider.cs +++ b/src/CoreSync.MySql/MySqlSyncProvider.cs @@ -299,9 +299,11 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa cmd.Parameters.Clear(); var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + // MIN(`id`) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) { - throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})"); + throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); } foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) diff --git a/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs b/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs index 47c7d52..ce05b66 100644 --- a/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs +++ b/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs @@ -315,8 +315,10 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa cmd.Parameters.Clear(); var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + // MIN(id) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})"); + throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) { diff --git a/src/CoreSync.SqlServer/SqlSyncProvider.cs b/src/CoreSync.SqlServer/SqlSyncProvider.cs index 5f24f56..3fe5c0a 100644 --- a/src/CoreSync.SqlServer/SqlSyncProvider.cs +++ b/src/CoreSync.SqlServer/SqlSyncProvider.cs @@ -730,8 +730,10 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa cmd.Parameters.Clear(); var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})"); + throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); foreach (SqlSyncTable table in tablesToSync.Cast()) { diff --git a/src/CoreSync.Sqlite/SqliteSyncProvider.cs b/src/CoreSync.Sqlite/SqliteSyncProvider.cs index fcb608c..f51cdb7 100644 --- a/src/CoreSync.Sqlite/SqliteSyncProvider.cs +++ b/src/CoreSync.Sqlite/SqliteSyncProvider.cs @@ -460,8 +460,10 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa cmd.Parameters.Clear(); var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})"); + throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) { diff --git a/src/CoreSync.Tests/AnchorTooOldTests.cs b/src/CoreSync.Tests/AnchorTooOldTests.cs new file mode 100644 index 0000000..34dd170 --- /dev/null +++ b/src/CoreSync.Tests/AnchorTooOldTests.cs @@ -0,0 +1,268 @@ +using CoreSync.Http; +using CoreSync.Sqlite; +using CoreSync.Tests.Data; +using Microsoft.Data.Sqlite; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Shouldly; +using SQLite; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace CoreSync.Tests; + +/// +/// Covers the aged-out anchor path: a peer asking for changes relative to a version the store no +/// longer retains must get a typed , and that condition must +/// survive the trip over HTTP instead of collapsing into a generic transport failure. +/// +/// +/// This is the failure behind "synchronization rarely works": any client idle longer than the change +/// retention window asks for a version that has been cleaned up. It is permanent for that client - +/// the only cure is a fresh snapshot - so what matters is that callers can *recognise* it rather than +/// see an anonymous error they are tempted to retry forever. +/// +[TestClass] +public class AnchorTooOldTests +{ + private static string CreateDatabase(string testName) + { + var dbFile = Path.Combine(Path.GetTempPath(), $"{testName}_{Guid.NewGuid():N}.sqlite"); + + using (var db = new SQLiteConnection(dbFile)) + { + db.CreateTable(); + } + + SqliteConnection.ClearAllPools(); + + return dbFile; + } + + private static SqliteSyncProvider CreateProvider(string dbFile) + => new(new SqliteSyncConfigurationBuilder($"Data Source={dbFile}") + .Table() + .Build(), logger: new ConsoleLogger("LOC")); + + [TestMethod] + public async Task GetChangesAsync_ThrowsTypedExceptionWhenAnchorFellOutOfRetention() + { + var dbFile = CreateDatabase(nameof(GetChangesAsync_ThrowsTypedExceptionWhenAnchorFellOutOfRetention)); + var syncProvider = CreateProvider(dbFile); + + await syncProvider.ApplyProvisionAsync(); + + using (var db = new SQLiteConnection(dbFile)) + { + for (var i = 0; i < 10; i++) + { + db.Insert(new Stock { Id = Guid.NewGuid(), Symbol = $"SYM{i}" }); + } + } + + SqliteConnection.ClearAllPools(); + + var otherStoreId = Guid.NewGuid(); + + //the peer last synchronized at version 1 and then went quiet + await syncProvider.SaveVersionForStoreAsync(otherStoreId, 1); + + //retention cleanup discards everything below version 5, exactly like SQL Server change + //tracking auto-cleanup does once CHANGE_RETENTION elapses + await syncProvider.ApplyRetentionPolicyAsync(5); + + var ex = await Should.ThrowAsync( + () => syncProvider.GetChangesAsync(otherStoreId, syncDirection: SyncDirection.DownloadOnly)); + + ex.RequestedVersion.ShouldBe(1); + + //the oldest retained change is version 5, so the oldest anchor still resolvable is 4 + ex.MinValidVersion.ShouldBe(4); + + //this store keeps one shared journal, so no individual table is implicated + ex.TableName.ShouldBeNull(); + + //the message has to name both versions: it is the only diagnostic that reaches the operator + ex.Message.ShouldContain("1"); + ex.Message.ShouldContain("4"); + } + + [TestMethod] + public async Task GetChangesAsync_SucceedsWhenAnchorIsStillWithinRetention() + { + var dbFile = CreateDatabase(nameof(GetChangesAsync_SucceedsWhenAnchorIsStillWithinRetention)); + var syncProvider = CreateProvider(dbFile); + + await syncProvider.ApplyProvisionAsync(); + + using (var db = new SQLiteConnection(dbFile)) + { + for (var i = 0; i < 10; i++) + { + db.Insert(new Stock { Id = Guid.NewGuid(), Symbol = $"SYM{i}" }); + } + } + + SqliteConnection.ClearAllPools(); + + var otherStoreId = Guid.NewGuid(); + + //anchor sits exactly on the retention boundary: the oldest retained change is 5, so an + //anchor of 4 is still resolvable and must not be rejected + await syncProvider.SaveVersionForStoreAsync(otherStoreId, 4); + await syncProvider.ApplyRetentionPolicyAsync(5); + + var changeSet = await syncProvider.GetChangesAsync(otherStoreId, syncDirection: SyncDirection.DownloadOnly); + + changeSet.ShouldNotBeNull(); + } + + [TestMethod] + public async Task HttpClient_RebuildsTypedExceptionFromServerSignal() + { + var thrown = new SyncAnchorTooOldException("[admin].[UserRole]", 1378, 1925); + using var server = SyncTestServer.Create(new ThrowingSyncProvider(thrown)); + + var ex = await Should.ThrowAsync( + () => server.HttpSyncProvider.GetChangesAsync(Guid.NewGuid(), syncDirection: SyncDirection.DownloadOnly)); + + //every field the caller needs to diagnose the failure has to survive the wire + ex.TableName.ShouldBe("[admin].[UserRole]"); + ex.RequestedVersion.ShouldBe(1378); + ex.MinValidVersion.ShouldBe(1925); + } + + [TestMethod] + public async Task HttpClient_RebuildsTypedExceptionForAStoreWideFailure() + { + var thrown = new SyncAnchorTooOldException(1378, 1925); + using var server = SyncTestServer.Create(new ThrowingSyncProvider(thrown)); + + var ex = await Should.ThrowAsync( + () => server.HttpSyncProvider.GetChangesAsync(Guid.NewGuid(), syncDirection: SyncDirection.DownloadOnly)); + + ex.TableName.ShouldBeNull(); + ex.RequestedVersion.ShouldBe(1378); + ex.MinValidVersion.ShouldBe(1925); + } + + [TestMethod] + public async Task HttpClient_RebuildsTypedExceptionRaisedWhileApplyingChanges() + { + var thrown = new SyncAnchorTooOldException("[core].[Survey]", 1378, 1925); + using var server = SyncTestServer.Create(new ThrowingSyncProvider(thrown)); + + //the upload path guards the anchor too, and has to report it just as distinguishably + var changeSet = new SyncChangeSet( + new SyncAnchor(Guid.NewGuid(), 1378), + SyncAnchor.Null, + new List()); + + var ex = await Should.ThrowAsync( + () => server.HttpSyncProvider.ApplyChangesAsync(changeSet)); + + ex.TableName.ShouldBe("[core].[Survey]"); + ex.RequestedVersion.ShouldBe(1378); + ex.MinValidVersion.ShouldBe(1925); + } + + [TestMethod] + public async Task HttpServer_RespondsWithGoneAndAMachineReadableCode() + { + var thrown = new SyncAnchorTooOldException("[ref].[AgeClass]", 2107, 2120); + using var server = SyncTestServer.Create(new ThrowingSyncProvider(thrown)); + + using var response = await server.RawHttpClient.GetAsync( + $"/api/sync-agent/changes-bulk/{Guid.NewGuid()}"); + + //410 rather than 500: the requested history existed once and is permanently gone, so a + //client that retries it can never succeed + response.StatusCode.ShouldBe(HttpStatusCode.Gone); + + response.Headers.GetValues(SyncHttpHeaders.ErrorCode) + .ShouldContain(SyncHttpErrorCodes.AnchorTooOld); + + var error = await response.Content.ReadFromJsonAsync(); + + error.ShouldNotBeNull(); + error!.TableName.ShouldBe("[ref].[AgeClass]"); + error.RequestedVersion.ShouldBe(2107); + error.MinValidVersion.ShouldBe(2120); + } + + [TestMethod] + public async Task HttpClient_LeavesUnrelatedFailuresAlone() + { + using var server = SyncTestServer.Create( + new ThrowingSyncProvider(new InvalidOperationException("something else went wrong"))); + + //an ordinary server-side error must not be mistaken for an aged-out anchor, which would tell + //the operator to reinitialize a client that has nothing wrong with it + var ex = await Should.ThrowAsync( + () => server.HttpSyncProvider.GetChangesAsync(Guid.NewGuid(), syncDirection: SyncDirection.DownloadOnly)); + + ex.ShouldNotBeAssignableTo(); + } + + [TestMethod] + public async Task HttpServer_LeavesUnrelatedFailuresWithoutTheErrorHeader() + { + using var server = SyncTestServer.Create( + new ThrowingSyncProvider(new InvalidOperationException("something else went wrong"))); + + //the filter must only claim the failures it actually recognises + var ex = await Should.ThrowAsync( + () => server.RawHttpClient.GetAsync($"/api/sync-agent/changes-bulk/{Guid.NewGuid()}")); + + ex.ShouldNotBeAssignableTo(); + } + + /// + /// A provider whose only job is to fail in a specific way, so the HTTP layer can be tested + /// independently of any particular database. + /// + private sealed class ThrowingSyncProvider : ISyncProvider + { + private readonly Exception _exception; + + public ThrowingSyncProvider(Exception exception) => _exception = exception; + + public string[]? SyncTableNames => null; + + public Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters = null, SyncDirection syncDirection = SyncDirection.UploadAndDownload, string[]? tables = null, CancellationToken cancellationToken = default) + => throw _exception; + + public Task ApplyChangesAsync(SyncChangeSet changeSet, Func? onConflictFunc = null, CancellationToken cancellationToken = default) + => throw _exception; + + public Task GetStoreIdAsync(CancellationToken cancellationToken = default) + => Task.FromResult(Guid.NewGuid()); + + public Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task ApplyProvisionAsync(CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task RemoveProvisionAsync(CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task GetSyncVersionAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task EnableChangeTrackingForTable(string tableName, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + public Task DisableChangeTrackingForTable(string tableName, CancellationToken cancellationToken = default) + => Task.CompletedTask; + } +} diff --git a/src/CoreSync.Tests/SqlServerCTResilienceTests.cs b/src/CoreSync.Tests/SqlServerCTResilienceTests.cs index ce48da8..a0067f6 100644 --- a/src/CoreSync.Tests/SqlServerCTResilienceTests.cs +++ b/src/CoreSync.Tests/SqlServerCTResilienceTests.cs @@ -211,7 +211,7 @@ await RunInDatabase("CoreSyncCT_UpdateNullAnchor", async connStr => var ex = await Should.ThrowAsync(() => provider.ApplyChangesAsync(changeSet)); - ex.TableName.ShouldContain("Authors"); + ex.TableName!.ShouldContain("Authors"); ex.RequestedVersion.ShouldBe(-1); ex.MinValidVersion.ShouldBeGreaterThanOrEqualTo(0); @@ -253,7 +253,7 @@ await RunInDatabase("CoreSyncCT_StaleAnchor", async connStr => var ex = await Should.ThrowAsync(() => provider.ApplyChangesAsync(changeSet)); - ex.TableName.ShouldContain("Authors"); + ex.TableName!.ShouldContain("Authors"); ex.RequestedVersion.ShouldBe(0); ex.MinValidVersion.ShouldBeGreaterThan(0); diff --git a/src/CoreSync.Tests/SyncTestServer.cs b/src/CoreSync.Tests/SyncTestServer.cs index 7bb31c1..03eeee1 100644 --- a/src/CoreSync.Tests/SyncTestServer.cs +++ b/src/CoreSync.Tests/SyncTestServer.cs @@ -22,6 +22,12 @@ internal sealed class SyncTestServer : IDisposable public ISyncProviderHttpClient HttpSyncProvider { get; } + /// + /// The underlying , for tests that need to assert on the raw response + /// (status code, headers, body) rather than on what the sync provider makes of it. + /// + public HttpClient RawHttpClient => _httpClient; + private SyncTestServer(WebApplication app, HttpClient httpClient, ServiceProvider clientServiceProvider, ISyncProviderHttpClient httpSyncProvider) { _app = app; diff --git a/src/CoreSync/SyncAnchorTooOldException.cs b/src/CoreSync/SyncAnchorTooOldException.cs index 2fe64f2..8b714e4 100644 --- a/src/CoreSync/SyncAnchorTooOldException.cs +++ b/src/CoreSync/SyncAnchorTooOldException.cs @@ -56,9 +56,27 @@ public SyncAnchorTooOldException(string tableName, long requestedVersion, long m } /// - /// Gets the name of the table whose change history no longer covers . + /// Initializes a new instance of the class for a store + /// that tracks changes in a single shared journal, where no individual table is implicated. /// - public string TableName { get; } + /// + /// The version the operation asked for. A negative value means no anchor was recorded at all + /// (see ), which is equally unrecoverable. + /// + /// The oldest version the store can still resolve changes from. + public SyncAnchorTooOldException(long requestedVersion, long minValidVersion) + : base(BuildMessage(null, requestedVersion, minValidVersion)) + { + RequestedVersion = requestedVersion; + MinValidVersion = minValidVersion; + } + + /// + /// Gets the name of the table whose change history no longer covers , + /// or null when the store keeps a single shared change journal and no individual table is + /// implicated. + /// + public string? TableName { get; } /// /// Gets the version the operation asked for. A negative value means the store holds no anchor @@ -71,13 +89,17 @@ public SyncAnchorTooOldException(string tableName, long requestedVersion, long m /// public long MinValidVersion { get; } - private static string BuildMessage(string tableName, long requestedVersion, long minValidVersion) + private static string BuildMessage(string? tableName, long requestedVersion, long minValidVersion) { var requested = requestedVersion < 0 ? "no anchor is recorded for the peer store" : $"version {requestedVersion} is older than the retained history"; - return $"Unable to resolve changes for table '{tableName}': {requested} " + + var target = tableName == null + ? "Unable to resolve changes" + : $"Unable to resolve changes for table '{tableName}'"; + + return $"{target}: {requested} " + $"(requested version {requestedVersion}, minimum valid version {minValidVersion}). " + "The peer store must be reinitialized from a fresh snapshot; retrying will not help."; } From 776f757e231498e12117c9211d665d1f3eeda26c Mon Sep 17 00:00:00 2001 From: Adolfo Marinucci Date: Wed, 26 Aug 2026 11:38:15 +0200 Subject: [PATCH 2/4] Trace an aged-out anchor before the failure escapes the provider The anchor validation runs ahead of the per-table loop that emits the ordinary progress traces, so a session that failed here recorded its opening "Begin GetChanges" line and nothing else. The reason was reachable, but only outside the session's own trace list. Log the condition at error level, with the table and both versions, at each point it is raised. A failed session now carries its own explanation. --- src/CoreSync.MySql/MySqlSyncProvider.cs | 8 +- .../PostgreSQLSyncProvider.cs | 10 +- src/CoreSync.SqlServer/SqlSyncProvider.cs | 2102 +++++++++-------- .../SqlServerCTProvider.cs | 12 +- src/CoreSync.Sqlite/SqliteSyncProvider.cs | 2014 ++++++++-------- 5 files changed, 2088 insertions(+), 2058 deletions(-) diff --git a/src/CoreSync.MySql/MySqlSyncProvider.cs b/src/CoreSync.MySql/MySqlSyncProvider.cs index 443b349..c967a36 100644 --- a/src/CoreSync.MySql/MySqlSyncProvider.cs +++ b/src/CoreSync.MySql/MySqlSyncProvider.cs @@ -1,4 +1,4 @@ -using JetBrains.Annotations; +using JetBrains.Annotations; using MySqlConnector; using System; using System.Collections.Generic; @@ -303,7 +303,11 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) { - throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; } foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) diff --git a/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs b/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs index ce05b66..6d3e04e 100644 --- a/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs +++ b/src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs @@ -1,4 +1,4 @@ -using JetBrains.Annotations; +using JetBrains.Annotations; using Npgsql; using System; using System.Collections.Generic; @@ -318,7 +318,13 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa // MIN(id) is the oldest change still journalled, so the oldest anchor that can still be // served is the one immediately before it. if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + { + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) { diff --git a/src/CoreSync.SqlServer/SqlSyncProvider.cs b/src/CoreSync.SqlServer/SqlSyncProvider.cs index 3fe5c0a..1cdacb6 100644 --- a/src/CoreSync.SqlServer/SqlSyncProvider.cs +++ b/src/CoreSync.SqlServer/SqlSyncProvider.cs @@ -1,1049 +1,1055 @@ -using JetBrains.Annotations; -using Microsoft.Data.SqlClient; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; -using System.Data; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace CoreSync.SqlServer -{ - public class SqlSyncProvider : ISyncProvider - { - private bool _initialized = false; - private Guid _storeId; - private readonly ISyncLogger? _logger; - - public SqlSyncProvider(SqlSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) - { - Configuration = configuration; - ProviderMode = providerMode; - _logger = logger; - - if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && - providerMode == ProviderMode.Bidirectional) - { - throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); - } - } - - public SqlSyncConfiguration Configuration { get; } - public ProviderMode ProviderMode { get; } - public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); - - public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, Func? onConflictFunc = null, CancellationToken cancellationToken = default) - { - Validate.NotNull(changeSet, nameof(changeSet)); - Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); - Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); - - await InitializeStoreAsync(cancellationToken); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); - - using var c = new SqlConnection(Configuration.ConnectionString); - var messageLog = new List(); - - try - { - c.InfoMessage += (s, e) => messageLog.Add(e); - await c.OpenAsync(cancellationToken); - - //await DisableConstraintsForChangeSetTables(c, changeSet); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = $"DECLARE @session uniqueidentifier; SET @session = @sync_client_id_binary; SET CONTEXT_INFO @session"; - cmd.Parameters.Add(new SqlParameter("@sync_client_id_binary", changeSet.SourceAnchor.StoreId)); - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.Parameters.Clear(); - - cmd.CommandText = $"SELECT CONTEXT_INFO()"; - var contextInfo = await cmd.ExecuteScalarAsync(cancellationToken); - cmd.Parameters.Clear(); - - var remainingItems = changeSet.Items.ToList(); - int pass = 0; - while (remainingItems.Count > 0) - { - pass++; - var failedItems = new List(); - int appliedInPass = 0; - - foreach (var item in remainingItems) - { - var table = (SqlSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); - - if (table == null) - { - continue; - } - - bool syncForceWrite = false; - var itemChangeType = item.ChangeType; - bool itemHandled = false; - - retryWrite: - cmd.Parameters.Clear(); - - table.SetupCommand(cmd, itemChangeType, item.Values); - - cmd.Parameters.Add(new SqlParameter("@last_sync_version", changeSet.TargetAnchor.Version)); - cmd.Parameters.Add(new SqlParameter("@sync_force_write", syncForceWrite)); - - int affectedRows; - - try - { - affectedRows = cmd.ExecuteNonQuery(); - - if (affectedRows > 0) - { - _logger?.Trace($"[{_storeId}] Successfully applied {item}"); - } - } - catch (Exception ex) - { - _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}Generated SQL:{Environment.NewLine}{cmd.CommandText}"); - throw new SynchronizationException($"Unable to {itemChangeType} item {item} to store for table {table}", ex); - } - - if (affectedRows == 0) - { - if (itemChangeType == ChangeType.Insert) - { - //If we can't apply an insert means that we already - //applied the insert or another record with same values (see primary key) - //is already present in table. - var executedCommand = cmd.CommandText; - - cmd.CommandText = table.SelectExistingQuery; - cmd.Parameters.Clear(); - var valueItem = item.Values[table.PrimaryColumnName]; - - cmd.Parameters.Add(table.Columns[table.PrimaryColumnName].CreateParameter("@PrimaryColumnParameter", valueItem)); - - if (1 == (int)await cmd.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) - { - itemChangeType = ChangeType.Update; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Unable to {itemChangeType} item {item} on table {table} (pass {pass}). Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}{Environment.NewLine}Generated SQL:{Environment.NewLine}{executedCommand}"); - } - } - else if (itemChangeType == ChangeType.Update || - itemChangeType == ChangeType.Delete) - { - if (syncForceWrite) - { - if (itemChangeType == ChangeType.Delete) - { - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemHandled = true; - } - else - { - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemChangeType = ChangeType.Insert; - goto retryWrite; - } - } - else - { - //conflict detected - var res = onConflictFunc?.Invoke(item); - if (res.HasValue && res.Value == ConflictResolution.ForceWrite) - { - _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); - - syncForceWrite = true; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); - itemHandled = true; - } - } - } - } - - if (affectedRows > 0) - { - itemHandled = true; - appliedInPass++; - } - - if (!itemHandled) - { - failedItems.Add(item); - } - } - - if (failedItems.Count == 0 || appliedInPass == 0) - { - if (failedItems.Count > 0) - { - _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); - } - break; - } - - _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); - remainingItems = failedItems; - } - - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - var resAnchor = new SyncAnchor(_storeId, version); - - _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resAnchor; - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - var exceptionMessage = $"An exception occurred during synchronization:{Environment.NewLine}Errors:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.SelectMany(_ => _.Errors.Cast()))}{Environment.NewLine}Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}"; - throw new SyncErrorException(exceptionMessage, ex); - } - finally - { - //await RestoreConstraintsForChangeSetTables(c, changeSet); - } - } - - private async Task DisableConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) - { - using var cmd = new SqlCommand(); - cmd.Connection = connection; - - foreach (SqlSyncTable table in changeSet.Items - .Select(_ => _.TableName) - .Distinct() - .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) - .Cast()) - { - cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} NOCHECK CONSTRAINT ALL"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - private async Task RestoreConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) - { - using var cmd = new SqlCommand(); - cmd.Connection = connection; - - foreach (SqlSyncTable table in changeSet.Items - .Select(_ => _.TableName) - .Distinct() - .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) - .Cast()) - { - cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} WITH CHECK CHECK CONSTRAINT ALL"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) - { - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - try - { - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [dbo].[__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(_storeId, (long)version); - } - - private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - // Report a missing anchor row as the null sentinel, matching - // GetLastLocalAnchorForStoreAsync and the other providers. Version 0 is not a sentinel: it - // is a real-looking version that callers cannot tell apart from "synchronized up to 0". - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(otherStoreId, (long)version); - } - - public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - return _storeId; - } - - private ChangeType DetectChangeType(Dictionary values) - { - if (values.TryGetValue("__OP", out var syncChangeOperation)) - { - return (syncChangeOperation?.ToString()) switch - { - "I" => ChangeType.Insert, - "U" => ChangeType.Update, - "D" => ChangeType.Delete, - _ => throw new NotSupportedException(), - }; - } - - return ChangeType.Insert; - } - - private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) - { - if (_initialized) - return; - - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using (var connection = new SqlConnection(Configuration.ConnectionString)) - { - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - if (!tableNames.Contains("__CORE_SYNC_CT")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_CT]( - [ID] [bigint] IDENTITY(1,1) NOT NULL, - [TBL] [nvarchar](1024) NOT NULL, - [OP] [char](1) NOT NULL, - [PK_Int] [int] SPARSE NULL, - [PK_String] [nvarchar](1024) SPARSE NULL, - [PK_Guid] [uniqueidentifier] SPARSE NULL, - [SRC] [varbinary](128) NULL, - CONSTRAINT [PK___CORE_SYNC_CT] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Int-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_Int] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_String-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_String] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Guid-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_Guid] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (!tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]( - [ID] [uniqueidentifier] NOT NULL, - [LOCAL_VERSION] [BIGINT] NULL, - [REMOTE_VERSION] [BIGINT] NULL - CONSTRAINT [PK___CORE_SYNC_REMOTE_ANCHOR] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (!tableNames.Contains("__CORE_SYNC_LOCAL_ID")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_LOCAL_ID]( - [ID] [uniqueidentifier] NOT NULL - CONSTRAINT [PK___CORE_SYNC_LOCAL_ID] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - using (var cmd = connection.CreateCommand()) - { - cmd.CommandText = "SELECT TOP 1 [ID] FROM [__CORE_SYNC_LOCAL_ID]"; - var localId = await cmd.ExecuteScalarAsync(cancellationToken); - if (localId == null) - { - localId = Guid.NewGuid(); - cmd.CommandText = $"INSERT INTO [__CORE_SYNC_LOCAL_ID] ([ID]) VALUES (@id)"; - cmd.Parameters.Add(new SqlParameter("@id", localId)); - if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - throw new InvalidOperationException(); - } - cmd.Parameters.Clear(); - } - - _storeId = (Guid)localId; - } - - using (var cmd = connection.CreateCommand()) - { - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); - if (primaryKeyIndexName == null) - { - throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't exist or it doesn't have a primary key"); - } - - var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); - - if (primaryKeyColumns.Length != 1) - { - throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); - } - - table.Columns = (await connection.GetTableColumnsAsync(table, cancellationToken)) - .ToDictionary(_ => _.Name, _ => _, StringComparer.OrdinalIgnoreCase); - table.PrimaryColumnName = primaryKeyColumns[0]; - table.PrimaryKeyColumns = primaryKeyColumns; - - if (table.SkipColumns.Any(skipColumn => string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0)) - { - throw new InvalidOperationException($"Column to skip in synchronization ('{table.PrimaryColumnName}') can't be the primary column"); - } - - var allColumns = table.Columns.Keys.Except(table.SkipColumns); - - var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); - - if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) - { - throw new NotSupportedException($"Table '{table.Name}' has one column with reserved name '__OP'"); - } - - cmd.CommandText = $@"SELECT COUNT(*) FROM SYS.IDENTITY_COLUMNS WHERE OBJECT_NAME(OBJECT_ID) = @tablename AND OBJECT_SCHEMA_NAME(object_id) = @schemaname"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@tablename", table.Name); - cmd.Parameters.AddWithValue("@schemaname", table.Schema); - - table.HasTableIdentityColumn = ((int)await cmd.ExecuteScalarAsync(cancellationToken) == 1); - } - } - } - - _initialized = true; - } - - public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - using var cmd = connection.CreateCommand(); - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); //dbTable.Indexes.Cast().FirstOrDefault(_ => _.IsClustered && _.IndexKeyType == IndexKeyType.DriPrimaryKey); - if (primaryKeyIndexName == null) - { - throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't have a primary key"); - } - - var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); //primaryKeyIndexName.IndexedColumns.Cast().ToList(); - var allColumns = (await connection.GetTableColumnsAsync(table, cancellationToken)).Select(_ => _.Name).ToArray(); - //var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); - - //if (primaryKeyColumns.Length != 1) - //{ - // throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); - //} - - if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) - { - throw new NotSupportedException($"Table '{table.NameWithSchema}' has one column with reserved name '__OP'"); - } - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - } - - private async Task SetupTableForFullChangeDetection(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) - { - var forceReloadOnInsert = table.ForceReloadInsertedRecords || table.HasTableIdentityColumn; - var forceReloadCode = "-- Table is set to NOT force reload the record after inserting"; - if (forceReloadOnInsert) - { - forceReloadCode = $@" - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}) - SELECT '{table.NameWithSchema}', 'U' , INSERTED{$".[{table.PrimaryColumnName}]"} - FROM INSERTED"; - } - - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] -ON {table.NameWithSchema} -AFTER {op} -AS -BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; - - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) - SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() - FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} - - {forceReloadCode} - -END"); - - cmd.CommandText = existsTriggerCommand("INSERT"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - private async Task SetupTableForUpdatesOrDeletesOnly(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) - { - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] -ON {table.NameWithSchema} -AFTER {op} -AS -BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; - - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) - SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() - FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} -END"); - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) - { - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - if (tableNames.Contains("__CORE_SYNC_CT")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_CT]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (tableNames.Contains("__CORE_SYNC_LOCAL_ID")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_LOCAL_ID]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - using (var cmd = connection.CreateCommand()) - { - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - await DisableChangeTrackingForTable(cmd, table, cancellationToken); - } - } - } - - public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters, SyncDirection syncDirection, string[]? tables = null, CancellationToken cancellationToken = default) - { - syncFilterParameters ??= []; - - var tablesToSync = Configuration.ResolveTableFilter(tables); - - var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); - - - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqlCommand(); - var items = new List(); - - using var tr = c.BeginTransaction();// IsolationLevel.Snapshot)) - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be - // served is the one immediately before it. - if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); - - foreach (SqlSyncTable table in tablesToSync.Cast()) - { - if (table.SyncDirection != SyncDirection.UploadAndDownload && - table.SyncDirection != syncDirection) - continue; - - _logger?.Info($"Getting changes for table {table.NameWithSchema}"); - //var snapshotItems = new HashSet(); - - if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) - { - if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) - { - throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); - } - - cmd.CommandText = table.InitialSnapshotQuery; - cmd.Parameters.Clear(); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - try - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(index => r.GetName(index), index => - { - var dbValue = r.GetValue(index); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }, StringComparer.OrdinalIgnoreCase); - - foreach (var skipColumn in table.SkipColumns) - { - if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) - throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); - } - - items.Add(new SqlSyncItem(table, ChangeType.Insert, values)); - //snapshotItems.Add(values[table.PrimaryColumnName]); - _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); - } - catch (ArgumentException) - { - var duplicateColumns = Enumerable.Range(0, r.FieldCount).Select(index => r.GetName(index)).GroupBy(_ => _).Where(_ => _.Count() > 1).Select(_ => _.Key).ToArray(); - _logger?.Error($"Duplicate columns found in table {table.NameWithSchema}: {string.Join(",", duplicateColumns)}"); - throw; - } - } - } - catch (Exception) - { - _logger?.Error($"InitialSnapshotQuery: {cmd.CommandText}"); - throw; - } - } - - if (!fromAnchor.IsNull()) - { - cmd.CommandText = table.IncrementalAddOrUpdatesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(index => r.GetName(index), index => - { - var dbValue = r.GetValue(index); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }, StringComparer.OrdinalIgnoreCase); - - foreach (var skipColumn in table.SkipColumns) - { - if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) - throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); - } - - items.Add(new SqlSyncItem(table, DetectChangeType(values), - values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); - - _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); - } - } - catch (Exception) - { - _logger?.Error($"IncrementalAddOrUpdatesQuery: {cmd.CommandText}"); - throw; - } - - cmd.CommandText = table.IncrementalDeletesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId); - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(_ => r.GetName(_), _ => - { - var dbValue = r.GetValue(_); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }); - items.Add(new SqlSyncItem(table, ChangeType.Delete, values)); - _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); - } - } - catch (Exception) - { - _logger?.Error($"IncrementalDeletesQuery: {cmd.CommandText}"); - throw; - } - } - } - - tr.Commit(); - - var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); - - _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resChangeSet; - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - return new SyncVersion(version, minVersion); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); - } - } - - public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(cancellationToken); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - tr.Commit(); - - return new SyncVersion(version, newMinVersion); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); - } - } - - public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - await InitializeStoreAsync(cancellationToken); - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); - - table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table '{name}'"); - } - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); - - table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table '{name}'"); - } - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - await DisableChangeTrackingForTable(cmd, table, cancellationToken); - } - - private async Task DisableChangeTrackingForTable(SqlCommand cmd, SqlSyncTable table, CancellationToken cancellationToken) - { - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var dropTriggerCommand = new Func((op) => $@"DROP TRIGGER [{table.Schema}].[__{table.NameWithSchemaRaw}_ct-{op}__]"); - - cmd.CommandText = existsTriggerCommand("INSERT"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - } +using JetBrains.Annotations; +using Microsoft.Data.SqlClient; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace CoreSync.SqlServer +{ + public class SqlSyncProvider : ISyncProvider + { + private bool _initialized = false; + private Guid _storeId; + private readonly ISyncLogger? _logger; + + public SqlSyncProvider(SqlSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) + { + Configuration = configuration; + ProviderMode = providerMode; + _logger = logger; + + if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && + providerMode == ProviderMode.Bidirectional) + { + throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); + } + } + + public SqlSyncConfiguration Configuration { get; } + public ProviderMode ProviderMode { get; } + public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); + + public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, Func? onConflictFunc = null, CancellationToken cancellationToken = default) + { + Validate.NotNull(changeSet, nameof(changeSet)); + Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); + Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); + + await InitializeStoreAsync(cancellationToken); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); + + using var c = new SqlConnection(Configuration.ConnectionString); + var messageLog = new List(); + + try + { + c.InfoMessage += (s, e) => messageLog.Add(e); + await c.OpenAsync(cancellationToken); + + //await DisableConstraintsForChangeSetTables(c, changeSet); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = $"DECLARE @session uniqueidentifier; SET @session = @sync_client_id_binary; SET CONTEXT_INFO @session"; + cmd.Parameters.Add(new SqlParameter("@sync_client_id_binary", changeSet.SourceAnchor.StoreId)); + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.Parameters.Clear(); + + cmd.CommandText = $"SELECT CONTEXT_INFO()"; + var contextInfo = await cmd.ExecuteScalarAsync(cancellationToken); + cmd.Parameters.Clear(); + + var remainingItems = changeSet.Items.ToList(); + int pass = 0; + while (remainingItems.Count > 0) + { + pass++; + var failedItems = new List(); + int appliedInPass = 0; + + foreach (var item in remainingItems) + { + var table = (SqlSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); + + if (table == null) + { + continue; + } + + bool syncForceWrite = false; + var itemChangeType = item.ChangeType; + bool itemHandled = false; + + retryWrite: + cmd.Parameters.Clear(); + + table.SetupCommand(cmd, itemChangeType, item.Values); + + cmd.Parameters.Add(new SqlParameter("@last_sync_version", changeSet.TargetAnchor.Version)); + cmd.Parameters.Add(new SqlParameter("@sync_force_write", syncForceWrite)); + + int affectedRows; + + try + { + affectedRows = cmd.ExecuteNonQuery(); + + if (affectedRows > 0) + { + _logger?.Trace($"[{_storeId}] Successfully applied {item}"); + } + } + catch (Exception ex) + { + _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}Generated SQL:{Environment.NewLine}{cmd.CommandText}"); + throw new SynchronizationException($"Unable to {itemChangeType} item {item} to store for table {table}", ex); + } + + if (affectedRows == 0) + { + if (itemChangeType == ChangeType.Insert) + { + //If we can't apply an insert means that we already + //applied the insert or another record with same values (see primary key) + //is already present in table. + var executedCommand = cmd.CommandText; + + cmd.CommandText = table.SelectExistingQuery; + cmd.Parameters.Clear(); + var valueItem = item.Values[table.PrimaryColumnName]; + + cmd.Parameters.Add(table.Columns[table.PrimaryColumnName].CreateParameter("@PrimaryColumnParameter", valueItem)); + + if (1 == (int)await cmd.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) + { + itemChangeType = ChangeType.Update; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Unable to {itemChangeType} item {item} on table {table} (pass {pass}). Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}{Environment.NewLine}Generated SQL:{Environment.NewLine}{executedCommand}"); + } + } + else if (itemChangeType == ChangeType.Update || + itemChangeType == ChangeType.Delete) + { + if (syncForceWrite) + { + if (itemChangeType == ChangeType.Delete) + { + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemHandled = true; + } + else + { + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemChangeType = ChangeType.Insert; + goto retryWrite; + } + } + else + { + //conflict detected + var res = onConflictFunc?.Invoke(item); + if (res.HasValue && res.Value == ConflictResolution.ForceWrite) + { + _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); + + syncForceWrite = true; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); + itemHandled = true; + } + } + } + } + + if (affectedRows > 0) + { + itemHandled = true; + appliedInPass++; + } + + if (!itemHandled) + { + failedItems.Add(item); + } + } + + if (failedItems.Count == 0 || appliedInPass == 0) + { + if (failedItems.Count > 0) + { + _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); + } + break; + } + + _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); + remainingItems = failedItems; + } + + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + var resAnchor = new SyncAnchor(_storeId, version); + + _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resAnchor; + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + var exceptionMessage = $"An exception occurred during synchronization:{Environment.NewLine}Errors:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.SelectMany(_ => _.Errors.Cast()))}{Environment.NewLine}Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}"; + throw new SyncErrorException(exceptionMessage, ex); + } + finally + { + //await RestoreConstraintsForChangeSetTables(c, changeSet); + } + } + + private async Task DisableConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) + { + using var cmd = new SqlCommand(); + cmd.Connection = connection; + + foreach (SqlSyncTable table in changeSet.Items + .Select(_ => _.TableName) + .Distinct() + .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) + .Cast()) + { + cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} NOCHECK CONSTRAINT ALL"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + private async Task RestoreConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) + { + using var cmd = new SqlCommand(); + cmd.Connection = connection; + + foreach (SqlSyncTable table in changeSet.Items + .Select(_ => _.TableName) + .Distinct() + .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) + .Cast()) + { + cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} WITH CHECK CHECK CONSTRAINT ALL"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) + { + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + try + { + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [dbo].[__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(_storeId, (long)version); + } + + private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + // Report a missing anchor row as the null sentinel, matching + // GetLastLocalAnchorForStoreAsync and the other providers. Version 0 is not a sentinel: it + // is a real-looking version that callers cannot tell apart from "synchronized up to 0". + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(otherStoreId, (long)version); + } + + public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + return _storeId; + } + + private ChangeType DetectChangeType(Dictionary values) + { + if (values.TryGetValue("__OP", out var syncChangeOperation)) + { + return (syncChangeOperation?.ToString()) switch + { + "I" => ChangeType.Insert, + "U" => ChangeType.Update, + "D" => ChangeType.Delete, + _ => throw new NotSupportedException(), + }; + } + + return ChangeType.Insert; + } + + private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) + { + if (_initialized) + return; + + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using (var connection = new SqlConnection(Configuration.ConnectionString)) + { + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + if (!tableNames.Contains("__CORE_SYNC_CT")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_CT]( + [ID] [bigint] IDENTITY(1,1) NOT NULL, + [TBL] [nvarchar](1024) NOT NULL, + [OP] [char](1) NOT NULL, + [PK_Int] [int] SPARSE NULL, + [PK_String] [nvarchar](1024) SPARSE NULL, + [PK_Guid] [uniqueidentifier] SPARSE NULL, + [SRC] [varbinary](128) NULL, + CONSTRAINT [PK___CORE_SYNC_CT] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Int-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_Int] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_String-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_String] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Guid-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_Guid] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (!tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]( + [ID] [uniqueidentifier] NOT NULL, + [LOCAL_VERSION] [BIGINT] NULL, + [REMOTE_VERSION] [BIGINT] NULL + CONSTRAINT [PK___CORE_SYNC_REMOTE_ANCHOR] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (!tableNames.Contains("__CORE_SYNC_LOCAL_ID")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_LOCAL_ID]( + [ID] [uniqueidentifier] NOT NULL + CONSTRAINT [PK___CORE_SYNC_LOCAL_ID] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "SELECT TOP 1 [ID] FROM [__CORE_SYNC_LOCAL_ID]"; + var localId = await cmd.ExecuteScalarAsync(cancellationToken); + if (localId == null) + { + localId = Guid.NewGuid(); + cmd.CommandText = $"INSERT INTO [__CORE_SYNC_LOCAL_ID] ([ID]) VALUES (@id)"; + cmd.Parameters.Add(new SqlParameter("@id", localId)); + if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + throw new InvalidOperationException(); + } + cmd.Parameters.Clear(); + } + + _storeId = (Guid)localId; + } + + using (var cmd = connection.CreateCommand()) + { + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); + if (primaryKeyIndexName == null) + { + throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't exist or it doesn't have a primary key"); + } + + var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); + + if (primaryKeyColumns.Length != 1) + { + throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); + } + + table.Columns = (await connection.GetTableColumnsAsync(table, cancellationToken)) + .ToDictionary(_ => _.Name, _ => _, StringComparer.OrdinalIgnoreCase); + table.PrimaryColumnName = primaryKeyColumns[0]; + table.PrimaryKeyColumns = primaryKeyColumns; + + if (table.SkipColumns.Any(skipColumn => string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0)) + { + throw new InvalidOperationException($"Column to skip in synchronization ('{table.PrimaryColumnName}') can't be the primary column"); + } + + var allColumns = table.Columns.Keys.Except(table.SkipColumns); + + var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); + + if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) + { + throw new NotSupportedException($"Table '{table.Name}' has one column with reserved name '__OP'"); + } + + cmd.CommandText = $@"SELECT COUNT(*) FROM SYS.IDENTITY_COLUMNS WHERE OBJECT_NAME(OBJECT_ID) = @tablename AND OBJECT_SCHEMA_NAME(object_id) = @schemaname"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@tablename", table.Name); + cmd.Parameters.AddWithValue("@schemaname", table.Schema); + + table.HasTableIdentityColumn = ((int)await cmd.ExecuteScalarAsync(cancellationToken) == 1); + } + } + } + + _initialized = true; + } + + public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + using var cmd = connection.CreateCommand(); + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); //dbTable.Indexes.Cast().FirstOrDefault(_ => _.IsClustered && _.IndexKeyType == IndexKeyType.DriPrimaryKey); + if (primaryKeyIndexName == null) + { + throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't have a primary key"); + } + + var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); //primaryKeyIndexName.IndexedColumns.Cast().ToList(); + var allColumns = (await connection.GetTableColumnsAsync(table, cancellationToken)).Select(_ => _.Name).ToArray(); + //var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); + + //if (primaryKeyColumns.Length != 1) + //{ + // throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); + //} + + if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) + { + throw new NotSupportedException($"Table '{table.NameWithSchema}' has one column with reserved name '__OP'"); + } + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + } + + private async Task SetupTableForFullChangeDetection(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) + { + var forceReloadOnInsert = table.ForceReloadInsertedRecords || table.HasTableIdentityColumn; + var forceReloadCode = "-- Table is set to NOT force reload the record after inserting"; + if (forceReloadOnInsert) + { + forceReloadCode = $@" + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}) + SELECT '{table.NameWithSchema}', 'U' , INSERTED{$".[{table.PrimaryColumnName}]"} + FROM INSERTED"; + } + + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] +ON {table.NameWithSchema} +AFTER {op} +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) + SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() + FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} + + {forceReloadCode} + +END"); + + cmd.CommandText = existsTriggerCommand("INSERT"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + private async Task SetupTableForUpdatesOrDeletesOnly(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) + { + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] +ON {table.NameWithSchema} +AFTER {op} +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) + SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() + FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} +END"); + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) + { + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + if (tableNames.Contains("__CORE_SYNC_CT")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_CT]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (tableNames.Contains("__CORE_SYNC_LOCAL_ID")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_LOCAL_ID]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + using (var cmd = connection.CreateCommand()) + { + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + await DisableChangeTrackingForTable(cmd, table, cancellationToken); + } + } + } + + public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters, SyncDirection syncDirection, string[]? tables = null, CancellationToken cancellationToken = default) + { + syncFilterParameters ??= []; + + var tablesToSync = Configuration.ResolveTableFilter(tables); + + var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); + + + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqlCommand(); + var items = new List(); + + using var tr = c.BeginTransaction();// IsolationLevel.Snapshot)) + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. + if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) + { + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } + + foreach (SqlSyncTable table in tablesToSync.Cast()) + { + if (table.SyncDirection != SyncDirection.UploadAndDownload && + table.SyncDirection != syncDirection) + continue; + + _logger?.Info($"Getting changes for table {table.NameWithSchema}"); + //var snapshotItems = new HashSet(); + + if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) + { + if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) + { + throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); + } + + cmd.CommandText = table.InitialSnapshotQuery; + cmd.Parameters.Clear(); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + try + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(index => r.GetName(index), index => + { + var dbValue = r.GetValue(index); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }, StringComparer.OrdinalIgnoreCase); + + foreach (var skipColumn in table.SkipColumns) + { + if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) + throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); + } + + items.Add(new SqlSyncItem(table, ChangeType.Insert, values)); + //snapshotItems.Add(values[table.PrimaryColumnName]); + _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); + } + catch (ArgumentException) + { + var duplicateColumns = Enumerable.Range(0, r.FieldCount).Select(index => r.GetName(index)).GroupBy(_ => _).Where(_ => _.Count() > 1).Select(_ => _.Key).ToArray(); + _logger?.Error($"Duplicate columns found in table {table.NameWithSchema}: {string.Join(",", duplicateColumns)}"); + throw; + } + } + } + catch (Exception) + { + _logger?.Error($"InitialSnapshotQuery: {cmd.CommandText}"); + throw; + } + } + + if (!fromAnchor.IsNull()) + { + cmd.CommandText = table.IncrementalAddOrUpdatesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(index => r.GetName(index), index => + { + var dbValue = r.GetValue(index); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }, StringComparer.OrdinalIgnoreCase); + + foreach (var skipColumn in table.SkipColumns) + { + if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) + throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); + } + + items.Add(new SqlSyncItem(table, DetectChangeType(values), + values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); + + _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); + } + } + catch (Exception) + { + _logger?.Error($"IncrementalAddOrUpdatesQuery: {cmd.CommandText}"); + throw; + } + + cmd.CommandText = table.IncrementalDeletesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId); + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(_ => r.GetName(_), _ => + { + var dbValue = r.GetValue(_); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }); + items.Add(new SqlSyncItem(table, ChangeType.Delete, values)); + _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); + } + } + catch (Exception) + { + _logger?.Error($"IncrementalDeletesQuery: {cmd.CommandText}"); + throw; + } + } + } + + tr.Commit(); + + var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); + + _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resChangeSet; + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + return new SyncVersion(version, minVersion); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); + } + } + + public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(cancellationToken); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + tr.Commit(); + + return new SyncVersion(version, newMinVersion); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); + } + } + + public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + await InitializeStoreAsync(cancellationToken); + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); + + table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table '{name}'"); + } + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); + + table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table '{name}'"); + } + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + await DisableChangeTrackingForTable(cmd, table, cancellationToken); + } + + private async Task DisableChangeTrackingForTable(SqlCommand cmd, SqlSyncTable table, CancellationToken cancellationToken) + { + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var dropTriggerCommand = new Func((op) => $@"DROP TRIGGER [{table.Schema}].[__{table.NameWithSchemaRaw}_ct-{op}__]"); + + cmd.CommandText = existsTriggerCommand("INSERT"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + } } \ No newline at end of file diff --git a/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs b/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs index feffa28..2c7b1ba 100644 --- a/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs +++ b/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs @@ -283,10 +283,11 @@ public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSe throw; } } - catch (SyncAnchorTooOldException) + catch (SyncAnchorTooOldException ex) { // Deliberately not wrapped: callers have to be able to tell "reinitialize this client" // apart from a transient error worth retrying. + _logger?.Error($"[{_storeId}] {ex.Message}"); throw; } catch (Exception ex) @@ -758,7 +759,14 @@ public async Task GetChangesAsync(Guid otherStoreId, SyncFilterPa var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); if (fromAnchor.Version < minVersion) - throw new SyncAnchorTooOldException(table.NameWithSchema, fromAnchor.Version, minVersion); + { + // Traced before throwing: this runs ahead of the per-table loop below, so + // without it a session that fails here records nothing but its opening + // "Begin GetChanges" line and leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(table.NameWithSchema, fromAnchor.Version, minVersion); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } } } diff --git a/src/CoreSync.Sqlite/SqliteSyncProvider.cs b/src/CoreSync.Sqlite/SqliteSyncProvider.cs index f51cdb7..4777b88 100644 --- a/src/CoreSync.Sqlite/SqliteSyncProvider.cs +++ b/src/CoreSync.Sqlite/SqliteSyncProvider.cs @@ -1,1005 +1,1011 @@ -using JetBrains.Annotations; -using Microsoft.Data.Sqlite; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; - -namespace CoreSync.Sqlite -{ - public class SqliteSyncProvider : ISyncProvider - { - private bool _initialized = false; - private Guid _storeId; - private readonly ISyncLogger? _logger; - - public SqliteSyncProvider(SqliteSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) - { - Configuration = configuration; - ProviderMode = providerMode; - _logger = logger; - - if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && - providerMode == ProviderMode.Bidirectional) - { - throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); - } - } - - public SqliteSyncConfiguration Configuration { get; } - public ProviderMode ProviderMode { get; } - public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); - - public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func? onConflictFunc = null, CancellationToken cancellationToken = default) - { - Validate.NotNull(changeSet, nameof(changeSet)); - Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); - Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); - - await InitializeStoreAsync(cancellationToken); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); - - using var c = new SqliteConnection(Configuration.ConnectionString); // +";Foreign Keys=False")) - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - // Reusable prepared commands, cached by (change type + table + column shape). The - // per-item write loop then only binds parameter values instead of rebuilding the SQL - // text and re-allocating parameters for each of (potentially) 100K+ items; SQLite keeps - // the compiled statement as long as the command text is unchanged. - var preparedCommands = new Dictionary(); - var existsCommands = new Dictionary(); - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - var remainingItems = changeSet.Items.ToList(); - - PreparedItemCommand GetItemCommand(SqliteSyncTable itemTable, ChangeType changeType, List> validColumns) - { - var columns = new string[validColumns.Count]; - for (int i = 0; i < validColumns.Count; i++) - { - columns[i] = validColumns[i].Key; - } - - var cacheKey = $"{(int)changeType}|{itemTable.Name}|{string.Join(",", columns)}"; - if (preparedCommands.TryGetValue(cacheKey, out var cached)) - { - return cached; - } - - var command = new SqliteCommand { Connection = c, Transaction = tr }; - var valueParameters = new SqliteParameter[columns.Length]; - - void AddValueParameters() - { - for (int i = 0; i < columns.Length; i++) - { - var valueParameter = new SqliteParameter($"@p{i}", DBNull.Value); - command.Parameters.Add(valueParameter); - valueParameters[i] = valueParameter; - } - } - - SqliteParameter? primaryKeyParameter = null; - SqliteParameter? lastSyncVersionParameter = null; - SqliteParameter? forceWriteParameter = null; - - switch (changeType) - { - case ChangeType.Insert: - command.CommandText = itemTable.BuildInsertCommandText(columns); - AddValueParameters(); - break; - - case ChangeType.Update: - command.CommandText = itemTable.BuildUpdateCommandText(columns); - primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); - command.Parameters.Add(primaryKeyParameter); - AddValueParameters(); - lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); - command.Parameters.Add(lastSyncVersionParameter); - forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); - command.Parameters.Add(forceWriteParameter); - break; - - case ChangeType.Delete: - command.CommandText = itemTable.BuildDeleteCommandText(); - valueParameters = Array.Empty(); - primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); - command.Parameters.Add(primaryKeyParameter); - lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); - command.Parameters.Add(lastSyncVersionParameter); - forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); - command.Parameters.Add(forceWriteParameter); - break; - } - - var preparedCommand = new PreparedItemCommand(command, valueParameters, primaryKeyParameter, lastSyncVersionParameter, forceWriteParameter); - preparedCommands[cacheKey] = preparedCommand; - return preparedCommand; - } - - SqliteCommand GetExistsCommand(SqliteSyncTable itemTable) - { - if (!existsCommands.TryGetValue(itemTable.Name, out var existsCommand)) - { - existsCommand = new SqliteCommand { Connection = c, Transaction = tr }; - existsCommand.CommandText = itemTable.SelectExistingQuery; - existsCommand.Parameters.Add(new SqliteParameter("@PrimaryColumnParameter", DBNull.Value)); - existsCommands[itemTable.Name] = existsCommand; - } - - return existsCommand; - } - int pass = 0; - while (remainingItems.Count > 0) - { - pass++; - var failedItems = new List(); - int appliedInPass = 0; - - foreach (var item in remainingItems) - { - var table = (SqliteSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); - if (table == null) - { - continue; - } - - bool syncForceWrite = false; - var itemChangeType = item.ChangeType; - bool itemHandled = false; - - retryWrite: - var validColumns = table.GetValuesForValidColumns(item.Values); - var prepared = GetItemCommand(table, itemChangeType, validColumns); - var itemCommand = prepared.Command; - - // ValueParameters is empty for deletes (which bind only the primary key); - // for inserts/updates it is parallel to validColumns. - for (int i = 0; i < prepared.ValueParameters.Length; i++) - { - prepared.ValueParameters[i].Value = validColumns[i].Value.Value ?? DBNull.Value; - } - - if (prepared.PrimaryKeyParameter != null) - { - prepared.PrimaryKeyParameter.Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; - } - - if (prepared.LastSyncVersionParameter != null) - { - prepared.LastSyncVersionParameter.Value = changeSet.TargetAnchor.Version; - } - - if (prepared.ForceWriteParameter != null) - { - prepared.ForceWriteParameter.Value = syncForceWrite; - } - - int affectedRows = 0; - - try - { - affectedRows = await itemCommand.ExecuteNonQueryAsync(cancellationToken); - - if (affectedRows > 0) - { - _logger?.Trace($"[{_storeId}] Successfully applied {item}"); - } - else - { - _logger?.Trace($"[{_storeId}] No row affected on insert for {item} (maybe it's a conflict? or a different table schema between client and server):{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); - } - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - //throw new SynchronizationException($"Unable to {item} item to store for table {table}", ex); - _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}{ex}{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); - } - - if (affectedRows == 0) - { - if (itemChangeType == ChangeType.Insert) - { - //If we can't apply an insert means that we already - //applied the insert or another record with same values (see primary key) - //is already present in table. - //In any case we can't proceed - var existsCommand = GetExistsCommand(table); - existsCommand.Parameters[0].Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; - if (1 == (long)await existsCommand.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) - { - itemChangeType = ChangeType.Update; - goto retryWrite; - } - else - { - _logger?.Warning($"Unable to {item}: much probably there is a foreign key constraint issue logged before (pass {pass})"); - } - } - else if (itemChangeType == ChangeType.Update || - itemChangeType == ChangeType.Delete) - { - if (syncForceWrite) - { - if (itemChangeType == ChangeType.Delete) - { - //item is already deleted in data store - //so this means that we're going to delete a already deleted record - //i.e. nothing to do - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemHandled = true; - } - else - { - //if user wants to update forcely a deleted record means that - //he asctually wants to insert it again in store - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemChangeType = ChangeType.Insert; - goto retryWrite; - } - } - else - { - //conflict detected - var res = onConflictFunc?.Invoke(item); - if (res.HasValue && res.Value == ConflictResolution.ForceWrite) - { - _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); - - syncForceWrite = true; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); - itemHandled = true; - } - } - } - } - - if (affectedRows > 0) - { - itemHandled = true; - appliedInPass++; - } - - if (!itemHandled) - { - failedItems.Add(item); - } - } - - if (failedItems.Count == 0 || appliedInPass == 0) - { - if (failedItems.Count > 0) - { - _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); - } - break; - } - - _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); - remainingItems = failedItems; - } - - // Stamp every change-tracking row produced while applying this change set as - // originating from the source store, so those changes are not echoed back on the - // next GetChanges (IncrementalAddOrUpdatesQuery filters on SRC != @sourceId). - // The AFTER INSERT/UPDATE/DELETE triggers create these rows with SRC = NULL and - // IDs strictly greater than the version captured before applying, so a single bulk - // UPDATE here replaces the former per-item SELECT MAX(ID) + UPDATE (which executed - // two extra statements for every applied item). - cmd.CommandText = "UPDATE [__CORE_SYNC_CT] SET [SRC] = @sourceId WHERE [ID] > @version AND [SRC] IS NULL"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@sourceId", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - var resAnchor = new SyncAnchor(_storeId, version); - - _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resAnchor; - } - catch (Exception ex) - { - _logger?.Error($"[{_storeId}] Unable to complete the apply changes:{Environment.NewLine}{ex}"); - tr.Rollback(); - throw; - } - finally - { - foreach (var preparedCommand in preparedCommands.Values) - { - preparedCommand.Command.Dispose(); - } - - foreach (var existsCommand in existsCommands.Values) - { - existsCommand.Dispose(); - } - } - } - - /// - /// A reusable command for applying a single change type against a table with a fixed column - /// shape. The command text is built once; each item only rebinds the parameter values. - /// - private sealed class PreparedItemCommand - { - public PreparedItemCommand( - SqliteCommand command, - SqliteParameter[] valueParameters, - SqliteParameter? primaryKeyParameter, - SqliteParameter? lastSyncVersionParameter, - SqliteParameter? forceWriteParameter) - { - Command = command; - ValueParameters = valueParameters; - PrimaryKeyParameter = primaryKeyParameter; - LastSyncVersionParameter = lastSyncVersionParameter; - ForceWriteParameter = forceWriteParameter; - } - - public SqliteCommand Command { get; } - - public SqliteParameter[] ValueParameters { get; } - - public SqliteParameter? PrimaryKeyParameter { get; } - - public SqliteParameter? LastSyncVersionParameter { get; } - - public SqliteParameter? ForceWriteParameter { get; } - } - - public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) - { - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters = null, SyncDirection syncDirection = SyncDirection.UploadAndDownload, string[]? tables = null, CancellationToken cancellationToken = default) - { - syncFilterParameters ??= []; - - var tablesToSync = Configuration.ResolveTableFilter(tables); - - var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); - - //if fromVersion is 0 means that client needs actually to initialize its local datastore - - //await InitializeStoreAsync(); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - var items = new List(); - - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be - // served is the one immediately before it. - if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - throw new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); - - foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) - { - if (table.SyncDirection != SyncDirection.UploadAndDownload && - table.SyncDirection != syncDirection) - continue; - - //var snapshotItems = new HashSet(); - - if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) - { - if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) - { - throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); - } - - cmd.CommandText = table.InitialSnapshotQuery; - cmd.Parameters.Clear(); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - items.Add(new SqliteSyncItem(table, ChangeType.Insert, values)); - //snapshotItems.Add(values[table.PrimaryColumnName]); - _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); - } - } - - if (!fromAnchor.IsNull()) - { - cmd.CommandText = table.IncrementalAddOrUpdatesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) - { - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - //if (snapshotItems.Contains(values[table.PrimaryColumnName])) - // continue; - - items.Add(new SqliteSyncItem(table, DetectChangeType(values), - values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); - _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); - } - } - - cmd.CommandText = table.IncrementalDeletesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); - using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) - { - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - items.Add(new SqliteSyncItem(table, ChangeType.Delete, values)); - _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); - } - } - } - } - - tr.Commit(); - - var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); - - _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resChangeSet; - - } - catch (Exception) - { - tr.Rollback(); - throw; - } - - } - - private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(_storeId, (long)version); - } - - private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - //await InitializeAsync(); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(otherStoreId, (long)version); - } - - public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - return _storeId; - } - - private static ChangeType DetectChangeType(Dictionary values) - { - if (values.TryGetValue("__OP", out var syncChangeOperation)) - { - switch (syncChangeOperation?.ToString()) - { - case "I": - return ChangeType.Insert; - - case "U": - return ChangeType.Update; - - case "D": - return ChangeType.Delete; - - default: - throw new NotSupportedException(); - } - } - - return ChangeType.Insert; - } - - private static object? GetValueFromRecord(SqliteSyncTable table, string columnName, int columnOrdinal, SqliteDataReader r) - { - if (r.IsDBNull(columnOrdinal)) - return null; - - if (table.RecordType == null) - return r.GetValue(columnOrdinal); - - var property = table.RecordType.GetProperty(columnName); - if (property != null) - return GetValueFromRecord(r, columnOrdinal, property.PropertyType); - - property = table.RecordType.GetProperties().FirstOrDefault(_ => - { - var columnAttribute = _.GetCustomAttribute(false); - if (columnAttribute != null) - { - return columnAttribute.Name == columnName; - } - - return false; - }); - - if (property != null) - return GetValueFromRecord(r, columnOrdinal, property.PropertyType); - - //fallback to getvalue - return r.GetValue(columnOrdinal); - } - - private static object GetValueFromRecord(SqliteDataReader r, int columnOrdinal, Type propertyType) - { - if (propertyType == typeof(string)) - return r.GetString(columnOrdinal); - if (propertyType == typeof(DateTime)) - return r.GetDateTime(columnOrdinal); - if (propertyType == typeof(int)) - return r.GetInt32(columnOrdinal); - if (propertyType == typeof(bool)) - return r.GetBoolean(columnOrdinal); - if (propertyType == typeof(byte)) - return r.GetByte(columnOrdinal); - if (propertyType == typeof(char)) - return r.GetChar(columnOrdinal); - if (propertyType == typeof(short)) - return r.GetInt16(columnOrdinal); - if (propertyType == typeof(long)) - return r.GetInt64(columnOrdinal); - if (propertyType == typeof(decimal)) - return r.GetDecimal(columnOrdinal); - if (propertyType == typeof(double)) - return r.GetDouble(columnOrdinal); - if (propertyType == typeof(float)) - return r.GetFloat(columnOrdinal); - - //fallback to getvalue - return r.GetValue(columnOrdinal); - } - - private static string TypeToPrintFormat(string type) - { - if (type == "INTEGER") - return "%d"; - if (type == "TEXT") - return "%s"; - - return "%s"; - } - - private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) - { - if (_initialized) - return; - - using (var connection = new SqliteConnection(Configuration.ConnectionString)) - { - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE IF NOT EXISTS __CORE_SYNC_CT -(ID INTEGER PRIMARY KEY, TBL TEXT NOT NULL COLLATE NOCASE, OP CHAR NOT NULL, PK_{SqlitePrimaryColumnType.Int} INTEGER NULL, PK_{SqlitePrimaryColumnType.Text} TEXT NULL COLLATE NOCASE, PK_{SqlitePrimaryColumnType.Blob} BLOB NULL, SRC TEXT NULL COLLATE NOCASE)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Int}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Int})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Text}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Text})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Blob}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Blob})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_REMOTE_ANCHOR (ID TEXT NOT NULL PRIMARY KEY, LOCAL_VERSION LONG NULL, REMOTE_VERSION LONG NULL)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_LOCAL_ID (ID TEXT NOT NULL PRIMARY KEY)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"SELECT ID FROM __CORE_SYNC_LOCAL_ID LIMIT 1"; - var localId = await cmd.ExecuteScalarAsync(cancellationToken); - if (localId == null) - { - localId = Guid.NewGuid().ToString(); - cmd.CommandText = $"INSERT INTO __CORE_SYNC_LOCAL_ID (ID) VALUES (@localId)"; - cmd.Parameters.Add(new SqliteParameter("@localId", localId)); - if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - throw new InvalidOperationException(); - } - cmd.Parameters.Clear(); - } - - _storeId = Guid.Parse((string)localId); - - foreach (SqliteSyncTable table in Configuration.Tables) - { - cmd.CommandText = $"PRAGMA table_info('{table.Name}')"; - using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) - { - /* - cid name type notnull dflt_value pk - ---------- ---------- ---------- ---------- ---------- ---------- - */ - while (await reader.ReadAsync(cancellationToken)) - { - var colName = reader.GetString(1); - var colType = reader.GetString(2); - var pk = reader.GetBoolean(5); - - if (string.CompareOrdinal(colName, "__OP") == 0) - { - throw new NotSupportedException($"Unable to synchronize table '{table.Name}': one column has a reserved name '__OP'"); - } - - table.Columns.Add(colName, new SqliteColumn(colName, colType, pk)); - } - } - - if (table.Columns.Count == 0) - { - throw new InvalidOperationException($"Unable to configure table '{table}': does it exist with at least one column?"); - } - - if (table.Columns.Count(_ => _.Value.IsPrimaryKey) == 0) - { - throw new NotSupportedException($"Unable to configure table '{table}': no primary key defined"); - } - - if (table.Columns.Count(_ => _.Value.IsPrimaryKey) > 1) - { - throw new NotSupportedException($"Unable to configure table '{table}': it has more than one column as primary key"); - } - } - } - - _initialized = true; - } - - public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) - { - //if (_initialized) - // return; - - await InitializeStoreAsync(cancellationToken); - - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - foreach (var table in Configuration.Tables.Cast().Where(_ => _.Columns.Any())) - { - //var primaryKeyColumns = table.Columns.Where(_ => _.IsPrimaryKey).ToArray(); - //var tableColumns = table.Columns.Where(_ => !_.IsPrimaryKey).ToArray(); - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - //_initialized = true; - } - - private async Task SetupTableForFullChangeDetection(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) - { - var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] - AFTER {op} ON [{table.Name}] - FOR EACH ROW - BEGIN - INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); - END"); - cmd.CommandText = commandTextBase("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - private async Task SetupTableForUpdatesOrDeletesOnly(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) - { - var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] - AFTER {op} ON [{table.Name}] - FOR EACH ROW - BEGIN - INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); - END"); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) - { - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - - //1. drop ct table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_CT"; - await cmd.ExecuteNonQueryAsync(); - - //2. drop remote anchor table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_REMOTE_ANCHOR"; - await cmd.ExecuteNonQueryAsync(); - - //3. drop local anchor table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_LOCAL_ID"; - await cmd.ExecuteNonQueryAsync(); - - //4. drop triggers — iterate table names directly; DROP TRIGGER IF EXISTS is safe even if never provisioned - foreach (SqliteSyncTable table in Configuration.Tables) - { - await DisableChangeTrackingForTable(cmd, table.Name, cancellationToken); - } - } - - public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - return new SyncVersion(version, minVersion); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); - } - } - - public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - tr.Commit(); - - return new SyncVersion(version, newMinVersion); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); - } - } - - public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - await InitializeStoreAsync(cancellationToken); - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table with name '{name}'"); - } - - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name) ?? throw new InvalidOperationException($"Unable to find table with name '{name}'"); - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - await DisableChangeTrackingForTable(cmd, name, cancellationToken); - } - - private async Task DisableChangeTrackingForTable(SqliteCommand cmd, string tableName, CancellationToken cancellationToken) - { - var commandTextBase = new Func((op) => $@"DROP TRIGGER IF EXISTS [__{tableName}_ct-{op}__]"); - cmd.CommandText = commandTextBase("INSERT"); - await cmd.ExecuteNonQueryAsync(); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(); - } - - } +using JetBrains.Annotations; +using Microsoft.Data.Sqlite; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +namespace CoreSync.Sqlite +{ + public class SqliteSyncProvider : ISyncProvider + { + private bool _initialized = false; + private Guid _storeId; + private readonly ISyncLogger? _logger; + + public SqliteSyncProvider(SqliteSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) + { + Configuration = configuration; + ProviderMode = providerMode; + _logger = logger; + + if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && + providerMode == ProviderMode.Bidirectional) + { + throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); + } + } + + public SqliteSyncConfiguration Configuration { get; } + public ProviderMode ProviderMode { get; } + public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); + + public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func? onConflictFunc = null, CancellationToken cancellationToken = default) + { + Validate.NotNull(changeSet, nameof(changeSet)); + Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); + Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); + + await InitializeStoreAsync(cancellationToken); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); + + using var c = new SqliteConnection(Configuration.ConnectionString); // +";Foreign Keys=False")) + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + // Reusable prepared commands, cached by (change type + table + column shape). The + // per-item write loop then only binds parameter values instead of rebuilding the SQL + // text and re-allocating parameters for each of (potentially) 100K+ items; SQLite keeps + // the compiled statement as long as the command text is unchanged. + var preparedCommands = new Dictionary(); + var existsCommands = new Dictionary(); + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + var remainingItems = changeSet.Items.ToList(); + + PreparedItemCommand GetItemCommand(SqliteSyncTable itemTable, ChangeType changeType, List> validColumns) + { + var columns = new string[validColumns.Count]; + for (int i = 0; i < validColumns.Count; i++) + { + columns[i] = validColumns[i].Key; + } + + var cacheKey = $"{(int)changeType}|{itemTable.Name}|{string.Join(",", columns)}"; + if (preparedCommands.TryGetValue(cacheKey, out var cached)) + { + return cached; + } + + var command = new SqliteCommand { Connection = c, Transaction = tr }; + var valueParameters = new SqliteParameter[columns.Length]; + + void AddValueParameters() + { + for (int i = 0; i < columns.Length; i++) + { + var valueParameter = new SqliteParameter($"@p{i}", DBNull.Value); + command.Parameters.Add(valueParameter); + valueParameters[i] = valueParameter; + } + } + + SqliteParameter? primaryKeyParameter = null; + SqliteParameter? lastSyncVersionParameter = null; + SqliteParameter? forceWriteParameter = null; + + switch (changeType) + { + case ChangeType.Insert: + command.CommandText = itemTable.BuildInsertCommandText(columns); + AddValueParameters(); + break; + + case ChangeType.Update: + command.CommandText = itemTable.BuildUpdateCommandText(columns); + primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); + command.Parameters.Add(primaryKeyParameter); + AddValueParameters(); + lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); + command.Parameters.Add(lastSyncVersionParameter); + forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); + command.Parameters.Add(forceWriteParameter); + break; + + case ChangeType.Delete: + command.CommandText = itemTable.BuildDeleteCommandText(); + valueParameters = Array.Empty(); + primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); + command.Parameters.Add(primaryKeyParameter); + lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); + command.Parameters.Add(lastSyncVersionParameter); + forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); + command.Parameters.Add(forceWriteParameter); + break; + } + + var preparedCommand = new PreparedItemCommand(command, valueParameters, primaryKeyParameter, lastSyncVersionParameter, forceWriteParameter); + preparedCommands[cacheKey] = preparedCommand; + return preparedCommand; + } + + SqliteCommand GetExistsCommand(SqliteSyncTable itemTable) + { + if (!existsCommands.TryGetValue(itemTable.Name, out var existsCommand)) + { + existsCommand = new SqliteCommand { Connection = c, Transaction = tr }; + existsCommand.CommandText = itemTable.SelectExistingQuery; + existsCommand.Parameters.Add(new SqliteParameter("@PrimaryColumnParameter", DBNull.Value)); + existsCommands[itemTable.Name] = existsCommand; + } + + return existsCommand; + } + int pass = 0; + while (remainingItems.Count > 0) + { + pass++; + var failedItems = new List(); + int appliedInPass = 0; + + foreach (var item in remainingItems) + { + var table = (SqliteSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); + if (table == null) + { + continue; + } + + bool syncForceWrite = false; + var itemChangeType = item.ChangeType; + bool itemHandled = false; + + retryWrite: + var validColumns = table.GetValuesForValidColumns(item.Values); + var prepared = GetItemCommand(table, itemChangeType, validColumns); + var itemCommand = prepared.Command; + + // ValueParameters is empty for deletes (which bind only the primary key); + // for inserts/updates it is parallel to validColumns. + for (int i = 0; i < prepared.ValueParameters.Length; i++) + { + prepared.ValueParameters[i].Value = validColumns[i].Value.Value ?? DBNull.Value; + } + + if (prepared.PrimaryKeyParameter != null) + { + prepared.PrimaryKeyParameter.Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; + } + + if (prepared.LastSyncVersionParameter != null) + { + prepared.LastSyncVersionParameter.Value = changeSet.TargetAnchor.Version; + } + + if (prepared.ForceWriteParameter != null) + { + prepared.ForceWriteParameter.Value = syncForceWrite; + } + + int affectedRows = 0; + + try + { + affectedRows = await itemCommand.ExecuteNonQueryAsync(cancellationToken); + + if (affectedRows > 0) + { + _logger?.Trace($"[{_storeId}] Successfully applied {item}"); + } + else + { + _logger?.Trace($"[{_storeId}] No row affected on insert for {item} (maybe it's a conflict? or a different table schema between client and server):{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + //throw new SynchronizationException($"Unable to {item} item to store for table {table}", ex); + _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}{ex}{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); + } + + if (affectedRows == 0) + { + if (itemChangeType == ChangeType.Insert) + { + //If we can't apply an insert means that we already + //applied the insert or another record with same values (see primary key) + //is already present in table. + //In any case we can't proceed + var existsCommand = GetExistsCommand(table); + existsCommand.Parameters[0].Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; + if (1 == (long)await existsCommand.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) + { + itemChangeType = ChangeType.Update; + goto retryWrite; + } + else + { + _logger?.Warning($"Unable to {item}: much probably there is a foreign key constraint issue logged before (pass {pass})"); + } + } + else if (itemChangeType == ChangeType.Update || + itemChangeType == ChangeType.Delete) + { + if (syncForceWrite) + { + if (itemChangeType == ChangeType.Delete) + { + //item is already deleted in data store + //so this means that we're going to delete a already deleted record + //i.e. nothing to do + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemHandled = true; + } + else + { + //if user wants to update forcely a deleted record means that + //he asctually wants to insert it again in store + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemChangeType = ChangeType.Insert; + goto retryWrite; + } + } + else + { + //conflict detected + var res = onConflictFunc?.Invoke(item); + if (res.HasValue && res.Value == ConflictResolution.ForceWrite) + { + _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); + + syncForceWrite = true; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); + itemHandled = true; + } + } + } + } + + if (affectedRows > 0) + { + itemHandled = true; + appliedInPass++; + } + + if (!itemHandled) + { + failedItems.Add(item); + } + } + + if (failedItems.Count == 0 || appliedInPass == 0) + { + if (failedItems.Count > 0) + { + _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); + } + break; + } + + _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); + remainingItems = failedItems; + } + + // Stamp every change-tracking row produced while applying this change set as + // originating from the source store, so those changes are not echoed back on the + // next GetChanges (IncrementalAddOrUpdatesQuery filters on SRC != @sourceId). + // The AFTER INSERT/UPDATE/DELETE triggers create these rows with SRC = NULL and + // IDs strictly greater than the version captured before applying, so a single bulk + // UPDATE here replaces the former per-item SELECT MAX(ID) + UPDATE (which executed + // two extra statements for every applied item). + cmd.CommandText = "UPDATE [__CORE_SYNC_CT] SET [SRC] = @sourceId WHERE [ID] > @version AND [SRC] IS NULL"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@sourceId", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + var resAnchor = new SyncAnchor(_storeId, version); + + _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resAnchor; + } + catch (Exception ex) + { + _logger?.Error($"[{_storeId}] Unable to complete the apply changes:{Environment.NewLine}{ex}"); + tr.Rollback(); + throw; + } + finally + { + foreach (var preparedCommand in preparedCommands.Values) + { + preparedCommand.Command.Dispose(); + } + + foreach (var existsCommand in existsCommands.Values) + { + existsCommand.Dispose(); + } + } + } + + /// + /// A reusable command for applying a single change type against a table with a fixed column + /// shape. The command text is built once; each item only rebinds the parameter values. + /// + private sealed class PreparedItemCommand + { + public PreparedItemCommand( + SqliteCommand command, + SqliteParameter[] valueParameters, + SqliteParameter? primaryKeyParameter, + SqliteParameter? lastSyncVersionParameter, + SqliteParameter? forceWriteParameter) + { + Command = command; + ValueParameters = valueParameters; + PrimaryKeyParameter = primaryKeyParameter; + LastSyncVersionParameter = lastSyncVersionParameter; + ForceWriteParameter = forceWriteParameter; + } + + public SqliteCommand Command { get; } + + public SqliteParameter[] ValueParameters { get; } + + public SqliteParameter? PrimaryKeyParameter { get; } + + public SqliteParameter? LastSyncVersionParameter { get; } + + public SqliteParameter? ForceWriteParameter { get; } + } + + public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) + { + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters = null, SyncDirection syncDirection = SyncDirection.UploadAndDownload, string[]? tables = null, CancellationToken cancellationToken = default) + { + syncFilterParameters ??= []; + + var tablesToSync = Configuration.ResolveTableFilter(tables); + + var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); + + //if fromVersion is 0 means that client needs actually to initialize its local datastore + + //await InitializeStoreAsync(); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + var items = new List(); + + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. + if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) + { + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } + + foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) + { + if (table.SyncDirection != SyncDirection.UploadAndDownload && + table.SyncDirection != syncDirection) + continue; + + //var snapshotItems = new HashSet(); + + if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) + { + if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) + { + throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); + } + + cmd.CommandText = table.InitialSnapshotQuery; + cmd.Parameters.Clear(); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + items.Add(new SqliteSyncItem(table, ChangeType.Insert, values)); + //snapshotItems.Add(values[table.PrimaryColumnName]); + _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); + } + } + + if (!fromAnchor.IsNull()) + { + cmd.CommandText = table.IncrementalAddOrUpdatesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) + { + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + //if (snapshotItems.Contains(values[table.PrimaryColumnName])) + // continue; + + items.Add(new SqliteSyncItem(table, DetectChangeType(values), + values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); + _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); + } + } + + cmd.CommandText = table.IncrementalDeletesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); + using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) + { + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + items.Add(new SqliteSyncItem(table, ChangeType.Delete, values)); + _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); + } + } + } + } + + tr.Commit(); + + var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); + + _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resChangeSet; + + } + catch (Exception) + { + tr.Rollback(); + throw; + } + + } + + private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(_storeId, (long)version); + } + + private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + //await InitializeAsync(); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(otherStoreId, (long)version); + } + + public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + return _storeId; + } + + private static ChangeType DetectChangeType(Dictionary values) + { + if (values.TryGetValue("__OP", out var syncChangeOperation)) + { + switch (syncChangeOperation?.ToString()) + { + case "I": + return ChangeType.Insert; + + case "U": + return ChangeType.Update; + + case "D": + return ChangeType.Delete; + + default: + throw new NotSupportedException(); + } + } + + return ChangeType.Insert; + } + + private static object? GetValueFromRecord(SqliteSyncTable table, string columnName, int columnOrdinal, SqliteDataReader r) + { + if (r.IsDBNull(columnOrdinal)) + return null; + + if (table.RecordType == null) + return r.GetValue(columnOrdinal); + + var property = table.RecordType.GetProperty(columnName); + if (property != null) + return GetValueFromRecord(r, columnOrdinal, property.PropertyType); + + property = table.RecordType.GetProperties().FirstOrDefault(_ => + { + var columnAttribute = _.GetCustomAttribute(false); + if (columnAttribute != null) + { + return columnAttribute.Name == columnName; + } + + return false; + }); + + if (property != null) + return GetValueFromRecord(r, columnOrdinal, property.PropertyType); + + //fallback to getvalue + return r.GetValue(columnOrdinal); + } + + private static object GetValueFromRecord(SqliteDataReader r, int columnOrdinal, Type propertyType) + { + if (propertyType == typeof(string)) + return r.GetString(columnOrdinal); + if (propertyType == typeof(DateTime)) + return r.GetDateTime(columnOrdinal); + if (propertyType == typeof(int)) + return r.GetInt32(columnOrdinal); + if (propertyType == typeof(bool)) + return r.GetBoolean(columnOrdinal); + if (propertyType == typeof(byte)) + return r.GetByte(columnOrdinal); + if (propertyType == typeof(char)) + return r.GetChar(columnOrdinal); + if (propertyType == typeof(short)) + return r.GetInt16(columnOrdinal); + if (propertyType == typeof(long)) + return r.GetInt64(columnOrdinal); + if (propertyType == typeof(decimal)) + return r.GetDecimal(columnOrdinal); + if (propertyType == typeof(double)) + return r.GetDouble(columnOrdinal); + if (propertyType == typeof(float)) + return r.GetFloat(columnOrdinal); + + //fallback to getvalue + return r.GetValue(columnOrdinal); + } + + private static string TypeToPrintFormat(string type) + { + if (type == "INTEGER") + return "%d"; + if (type == "TEXT") + return "%s"; + + return "%s"; + } + + private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) + { + if (_initialized) + return; + + using (var connection = new SqliteConnection(Configuration.ConnectionString)) + { + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE IF NOT EXISTS __CORE_SYNC_CT +(ID INTEGER PRIMARY KEY, TBL TEXT NOT NULL COLLATE NOCASE, OP CHAR NOT NULL, PK_{SqlitePrimaryColumnType.Int} INTEGER NULL, PK_{SqlitePrimaryColumnType.Text} TEXT NULL COLLATE NOCASE, PK_{SqlitePrimaryColumnType.Blob} BLOB NULL, SRC TEXT NULL COLLATE NOCASE)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Int}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Int})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Text}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Text})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Blob}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Blob})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_REMOTE_ANCHOR (ID TEXT NOT NULL PRIMARY KEY, LOCAL_VERSION LONG NULL, REMOTE_VERSION LONG NULL)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_LOCAL_ID (ID TEXT NOT NULL PRIMARY KEY)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"SELECT ID FROM __CORE_SYNC_LOCAL_ID LIMIT 1"; + var localId = await cmd.ExecuteScalarAsync(cancellationToken); + if (localId == null) + { + localId = Guid.NewGuid().ToString(); + cmd.CommandText = $"INSERT INTO __CORE_SYNC_LOCAL_ID (ID) VALUES (@localId)"; + cmd.Parameters.Add(new SqliteParameter("@localId", localId)); + if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + throw new InvalidOperationException(); + } + cmd.Parameters.Clear(); + } + + _storeId = Guid.Parse((string)localId); + + foreach (SqliteSyncTable table in Configuration.Tables) + { + cmd.CommandText = $"PRAGMA table_info('{table.Name}')"; + using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) + { + /* + cid name type notnull dflt_value pk + ---------- ---------- ---------- ---------- ---------- ---------- + */ + while (await reader.ReadAsync(cancellationToken)) + { + var colName = reader.GetString(1); + var colType = reader.GetString(2); + var pk = reader.GetBoolean(5); + + if (string.CompareOrdinal(colName, "__OP") == 0) + { + throw new NotSupportedException($"Unable to synchronize table '{table.Name}': one column has a reserved name '__OP'"); + } + + table.Columns.Add(colName, new SqliteColumn(colName, colType, pk)); + } + } + + if (table.Columns.Count == 0) + { + throw new InvalidOperationException($"Unable to configure table '{table}': does it exist with at least one column?"); + } + + if (table.Columns.Count(_ => _.Value.IsPrimaryKey) == 0) + { + throw new NotSupportedException($"Unable to configure table '{table}': no primary key defined"); + } + + if (table.Columns.Count(_ => _.Value.IsPrimaryKey) > 1) + { + throw new NotSupportedException($"Unable to configure table '{table}': it has more than one column as primary key"); + } + } + } + + _initialized = true; + } + + public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) + { + //if (_initialized) + // return; + + await InitializeStoreAsync(cancellationToken); + + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + foreach (var table in Configuration.Tables.Cast().Where(_ => _.Columns.Any())) + { + //var primaryKeyColumns = table.Columns.Where(_ => _.IsPrimaryKey).ToArray(); + //var tableColumns = table.Columns.Where(_ => !_.IsPrimaryKey).ToArray(); + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + //_initialized = true; + } + + private async Task SetupTableForFullChangeDetection(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) + { + var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] + AFTER {op} ON [{table.Name}] + FOR EACH ROW + BEGIN + INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); + END"); + cmd.CommandText = commandTextBase("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + private async Task SetupTableForUpdatesOrDeletesOnly(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) + { + var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] + AFTER {op} ON [{table.Name}] + FOR EACH ROW + BEGIN + INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); + END"); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) + { + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + + //1. drop ct table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_CT"; + await cmd.ExecuteNonQueryAsync(); + + //2. drop remote anchor table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_REMOTE_ANCHOR"; + await cmd.ExecuteNonQueryAsync(); + + //3. drop local anchor table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_LOCAL_ID"; + await cmd.ExecuteNonQueryAsync(); + + //4. drop triggers — iterate table names directly; DROP TRIGGER IF EXISTS is safe even if never provisioned + foreach (SqliteSyncTable table in Configuration.Tables) + { + await DisableChangeTrackingForTable(cmd, table.Name, cancellationToken); + } + } + + public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + return new SyncVersion(version, minVersion); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); + } + } + + public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + tr.Commit(); + + return new SyncVersion(version, newMinVersion); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); + } + } + + public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + await InitializeStoreAsync(cancellationToken); + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table with name '{name}'"); + } + + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name) ?? throw new InvalidOperationException($"Unable to find table with name '{name}'"); + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + await DisableChangeTrackingForTable(cmd, name, cancellationToken); + } + + private async Task DisableChangeTrackingForTable(SqliteCommand cmd, string tableName, CancellationToken cancellationToken) + { + var commandTextBase = new Func((op) => $@"DROP TRIGGER IF EXISTS [__{tableName}_ct-{op}__]"); + cmd.CommandText = commandTextBase("INSERT"); + await cmd.ExecuteNonQueryAsync(); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(); + } + + } } \ No newline at end of file From bab4cf91f0e9f4df8c5a8e8f4cb4e55352d2c493 Mon Sep 17 00:00:00 2001 From: Adolfo Marinucci Date: Wed, 26 Aug 2026 11:45:10 +0200 Subject: [PATCH 3/4] Allow change retention to be applied without a full provision Retention is reconciled as part of ApplyProvisionAsync, which is the right place for it during provisioning but leaves no way to act on the setting on its own. A host that lets an operator edit the retention needs one: storing the value changes nothing on a database that already has change tracking enabled, and a setting that appears saved but was never applied is how a database quietly keeps a week of history when it was configured for a month. Add SqlServerCTProvider.ApplyChangeRetentionAsync, which issues the same reconciliation and then reads the settings back from sys.change_tracking_databases so the caller can report what the database actually holds rather than what was asked for. Making that return value useful means ChangeTrackingDatabaseOptions becomes public. --- src/CoreSync.SqlServer/SqlSyncProvider.cs | 2108 ++++++++--------- .../ChangeTrackingDatabaseOptions.cs | 20 +- .../SqlServerCTProvider.cs | 30 + src/CoreSync.Sqlite/SqliteSyncProvider.cs | 2020 ++++++++-------- 4 files changed, 2112 insertions(+), 2066 deletions(-) diff --git a/src/CoreSync.SqlServer/SqlSyncProvider.cs b/src/CoreSync.SqlServer/SqlSyncProvider.cs index 1cdacb6..cb4b1a9 100644 --- a/src/CoreSync.SqlServer/SqlSyncProvider.cs +++ b/src/CoreSync.SqlServer/SqlSyncProvider.cs @@ -1,1055 +1,1055 @@ -using JetBrains.Annotations; -using Microsoft.Data.SqlClient; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; -using System.Data; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace CoreSync.SqlServer -{ - public class SqlSyncProvider : ISyncProvider - { - private bool _initialized = false; - private Guid _storeId; - private readonly ISyncLogger? _logger; - - public SqlSyncProvider(SqlSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) - { - Configuration = configuration; - ProviderMode = providerMode; - _logger = logger; - - if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && - providerMode == ProviderMode.Bidirectional) - { - throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); - } - } - - public SqlSyncConfiguration Configuration { get; } - public ProviderMode ProviderMode { get; } - public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); - - public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, Func? onConflictFunc = null, CancellationToken cancellationToken = default) - { - Validate.NotNull(changeSet, nameof(changeSet)); - Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); - Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); - - await InitializeStoreAsync(cancellationToken); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); - - using var c = new SqlConnection(Configuration.ConnectionString); - var messageLog = new List(); - - try - { - c.InfoMessage += (s, e) => messageLog.Add(e); - await c.OpenAsync(cancellationToken); - - //await DisableConstraintsForChangeSetTables(c, changeSet); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = $"DECLARE @session uniqueidentifier; SET @session = @sync_client_id_binary; SET CONTEXT_INFO @session"; - cmd.Parameters.Add(new SqlParameter("@sync_client_id_binary", changeSet.SourceAnchor.StoreId)); - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.Parameters.Clear(); - - cmd.CommandText = $"SELECT CONTEXT_INFO()"; - var contextInfo = await cmd.ExecuteScalarAsync(cancellationToken); - cmd.Parameters.Clear(); - - var remainingItems = changeSet.Items.ToList(); - int pass = 0; - while (remainingItems.Count > 0) - { - pass++; - var failedItems = new List(); - int appliedInPass = 0; - - foreach (var item in remainingItems) - { - var table = (SqlSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); - - if (table == null) - { - continue; - } - - bool syncForceWrite = false; - var itemChangeType = item.ChangeType; - bool itemHandled = false; - - retryWrite: - cmd.Parameters.Clear(); - - table.SetupCommand(cmd, itemChangeType, item.Values); - - cmd.Parameters.Add(new SqlParameter("@last_sync_version", changeSet.TargetAnchor.Version)); - cmd.Parameters.Add(new SqlParameter("@sync_force_write", syncForceWrite)); - - int affectedRows; - - try - { - affectedRows = cmd.ExecuteNonQuery(); - - if (affectedRows > 0) - { - _logger?.Trace($"[{_storeId}] Successfully applied {item}"); - } - } - catch (Exception ex) - { - _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}Generated SQL:{Environment.NewLine}{cmd.CommandText}"); - throw new SynchronizationException($"Unable to {itemChangeType} item {item} to store for table {table}", ex); - } - - if (affectedRows == 0) - { - if (itemChangeType == ChangeType.Insert) - { - //If we can't apply an insert means that we already - //applied the insert or another record with same values (see primary key) - //is already present in table. - var executedCommand = cmd.CommandText; - - cmd.CommandText = table.SelectExistingQuery; - cmd.Parameters.Clear(); - var valueItem = item.Values[table.PrimaryColumnName]; - - cmd.Parameters.Add(table.Columns[table.PrimaryColumnName].CreateParameter("@PrimaryColumnParameter", valueItem)); - - if (1 == (int)await cmd.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) - { - itemChangeType = ChangeType.Update; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Unable to {itemChangeType} item {item} on table {table} (pass {pass}). Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}{Environment.NewLine}Generated SQL:{Environment.NewLine}{executedCommand}"); - } - } - else if (itemChangeType == ChangeType.Update || - itemChangeType == ChangeType.Delete) - { - if (syncForceWrite) - { - if (itemChangeType == ChangeType.Delete) - { - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemHandled = true; - } - else - { - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemChangeType = ChangeType.Insert; - goto retryWrite; - } - } - else - { - //conflict detected - var res = onConflictFunc?.Invoke(item); - if (res.HasValue && res.Value == ConflictResolution.ForceWrite) - { - _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); - - syncForceWrite = true; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); - itemHandled = true; - } - } - } - } - - if (affectedRows > 0) - { - itemHandled = true; - appliedInPass++; - } - - if (!itemHandled) - { - failedItems.Add(item); - } - } - - if (failedItems.Count == 0 || appliedInPass == 0) - { - if (failedItems.Count > 0) - { - _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); - } - break; - } - - _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); - remainingItems = failedItems; - } - - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - var resAnchor = new SyncAnchor(_storeId, version); - - _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resAnchor; - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - var exceptionMessage = $"An exception occurred during synchronization:{Environment.NewLine}Errors:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.SelectMany(_ => _.Errors.Cast()))}{Environment.NewLine}Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}"; - throw new SyncErrorException(exceptionMessage, ex); - } - finally - { - //await RestoreConstraintsForChangeSetTables(c, changeSet); - } - } - - private async Task DisableConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) - { - using var cmd = new SqlCommand(); - cmd.Connection = connection; - - foreach (SqlSyncTable table in changeSet.Items - .Select(_ => _.TableName) - .Distinct() - .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) - .Cast()) - { - cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} NOCHECK CONSTRAINT ALL"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - private async Task RestoreConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) - { - using var cmd = new SqlCommand(); - cmd.Connection = connection; - - foreach (SqlSyncTable table in changeSet.Items - .Select(_ => _.TableName) - .Distinct() - .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) - .Cast()) - { - cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} WITH CHECK CHECK CONSTRAINT ALL"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) - { - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - try - { - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [dbo].[__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(_storeId, (long)version); - } - - private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - // Report a missing anchor row as the null sentinel, matching - // GetLastLocalAnchorForStoreAsync and the other providers. Version 0 is not a sentinel: it - // is a real-looking version that callers cannot tell apart from "synchronized up to 0". - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(otherStoreId, (long)version); - } - - public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - return _storeId; - } - - private ChangeType DetectChangeType(Dictionary values) - { - if (values.TryGetValue("__OP", out var syncChangeOperation)) - { - return (syncChangeOperation?.ToString()) switch - { - "I" => ChangeType.Insert, - "U" => ChangeType.Update, - "D" => ChangeType.Delete, - _ => throw new NotSupportedException(), - }; - } - - return ChangeType.Insert; - } - - private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) - { - if (_initialized) - return; - - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using (var connection = new SqlConnection(Configuration.ConnectionString)) - { - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - if (!tableNames.Contains("__CORE_SYNC_CT")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_CT]( - [ID] [bigint] IDENTITY(1,1) NOT NULL, - [TBL] [nvarchar](1024) NOT NULL, - [OP] [char](1) NOT NULL, - [PK_Int] [int] SPARSE NULL, - [PK_String] [nvarchar](1024) SPARSE NULL, - [PK_Guid] [uniqueidentifier] SPARSE NULL, - [SRC] [varbinary](128) NULL, - CONSTRAINT [PK___CORE_SYNC_CT] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Int-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_Int] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_String-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_String] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Guid-Index] ON [dbo].[__CORE_SYNC_CT] -( - [PK_Guid] ASC -) -INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (!tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]( - [ID] [uniqueidentifier] NOT NULL, - [LOCAL_VERSION] [BIGINT] NULL, - [REMOTE_VERSION] [BIGINT] NULL - CONSTRAINT [PK___CORE_SYNC_REMOTE_ANCHOR] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (!tableNames.Contains("__CORE_SYNC_LOCAL_ID")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_LOCAL_ID]( - [ID] [uniqueidentifier] NOT NULL - CONSTRAINT [PK___CORE_SYNC_LOCAL_ID] PRIMARY KEY CLUSTERED -( - [ID] ASC -)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] -) ON [PRIMARY]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - using (var cmd = connection.CreateCommand()) - { - cmd.CommandText = "SELECT TOP 1 [ID] FROM [__CORE_SYNC_LOCAL_ID]"; - var localId = await cmd.ExecuteScalarAsync(cancellationToken); - if (localId == null) - { - localId = Guid.NewGuid(); - cmd.CommandText = $"INSERT INTO [__CORE_SYNC_LOCAL_ID] ([ID]) VALUES (@id)"; - cmd.Parameters.Add(new SqlParameter("@id", localId)); - if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - throw new InvalidOperationException(); - } - cmd.Parameters.Clear(); - } - - _storeId = (Guid)localId; - } - - using (var cmd = connection.CreateCommand()) - { - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); - if (primaryKeyIndexName == null) - { - throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't exist or it doesn't have a primary key"); - } - - var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); - - if (primaryKeyColumns.Length != 1) - { - throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); - } - - table.Columns = (await connection.GetTableColumnsAsync(table, cancellationToken)) - .ToDictionary(_ => _.Name, _ => _, StringComparer.OrdinalIgnoreCase); - table.PrimaryColumnName = primaryKeyColumns[0]; - table.PrimaryKeyColumns = primaryKeyColumns; - - if (table.SkipColumns.Any(skipColumn => string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0)) - { - throw new InvalidOperationException($"Column to skip in synchronization ('{table.PrimaryColumnName}') can't be the primary column"); - } - - var allColumns = table.Columns.Keys.Except(table.SkipColumns); - - var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); - - if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) - { - throw new NotSupportedException($"Table '{table.Name}' has one column with reserved name '__OP'"); - } - - cmd.CommandText = $@"SELECT COUNT(*) FROM SYS.IDENTITY_COLUMNS WHERE OBJECT_NAME(OBJECT_ID) = @tablename AND OBJECT_SCHEMA_NAME(object_id) = @schemaname"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@tablename", table.Name); - cmd.Parameters.AddWithValue("@schemaname", table.Schema); - - table.HasTableIdentityColumn = ((int)await cmd.ExecuteScalarAsync(cancellationToken) == 1); - } - } - } - - _initialized = true; - } - - public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - using var cmd = connection.CreateCommand(); - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); //dbTable.Indexes.Cast().FirstOrDefault(_ => _.IsClustered && _.IndexKeyType == IndexKeyType.DriPrimaryKey); - if (primaryKeyIndexName == null) - { - throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't have a primary key"); - } - - var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); //primaryKeyIndexName.IndexedColumns.Cast().ToList(); - var allColumns = (await connection.GetTableColumnsAsync(table, cancellationToken)).Select(_ => _.Name).ToArray(); - //var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); - - //if (primaryKeyColumns.Length != 1) - //{ - // throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); - //} - - if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) - { - throw new NotSupportedException($"Table '{table.NameWithSchema}' has one column with reserved name '__OP'"); - } - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - } - - private async Task SetupTableForFullChangeDetection(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) - { - var forceReloadOnInsert = table.ForceReloadInsertedRecords || table.HasTableIdentityColumn; - var forceReloadCode = "-- Table is set to NOT force reload the record after inserting"; - if (forceReloadOnInsert) - { - forceReloadCode = $@" - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}) - SELECT '{table.NameWithSchema}', 'U' , INSERTED{$".[{table.PrimaryColumnName}]"} - FROM INSERTED"; - } - - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] -ON {table.NameWithSchema} -AFTER {op} -AS -BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; - - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) - SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() - FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} - - {forceReloadCode} - -END"); - - cmd.CommandText = existsTriggerCommand("INSERT"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - private async Task SetupTableForUpdatesOrDeletesOnly(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) - { - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] -ON {table.NameWithSchema} -AFTER {op} -AS -BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from - -- interfering with SELECT statements. - SET NOCOUNT ON; - - INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) - SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() - FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} -END"); - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) - { - cmd.CommandText = createTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - - public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) - { - var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); - if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) - throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - var tableNames = await connection.GetTableNamesAsync(cancellationToken); - if (tableNames.Contains("__CORE_SYNC_CT")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_CT]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - if (tableNames.Contains("__CORE_SYNC_LOCAL_ID")) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_LOCAL_ID]"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - using (var cmd = connection.CreateCommand()) - { - foreach (SqlSyncTable table in Configuration.Tables.Cast()) - { - await DisableChangeTrackingForTable(cmd, table, cancellationToken); - } - } - } - - public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters, SyncDirection syncDirection, string[]? tables = null, CancellationToken cancellationToken = default) - { - syncFilterParameters ??= []; - - var tablesToSync = Configuration.ResolveTableFilter(tables); - - var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); - - - using var c = new SqlConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqlCommand(); - var items = new List(); - - using var tr = c.BeginTransaction();// IsolationLevel.Snapshot)) - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be - // served is the one immediately before it. - if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - { - // Traced before throwing: this runs ahead of the per-table loop below, so without - // it a session that fails here leaves no account of why it stopped. - var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); - _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); - throw anchorTooOld; - } - - foreach (SqlSyncTable table in tablesToSync.Cast()) - { - if (table.SyncDirection != SyncDirection.UploadAndDownload && - table.SyncDirection != syncDirection) - continue; - - _logger?.Info($"Getting changes for table {table.NameWithSchema}"); - //var snapshotItems = new HashSet(); - - if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) - { - if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) - { - throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); - } - - cmd.CommandText = table.InitialSnapshotQuery; - cmd.Parameters.Clear(); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - try - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(index => r.GetName(index), index => - { - var dbValue = r.GetValue(index); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }, StringComparer.OrdinalIgnoreCase); - - foreach (var skipColumn in table.SkipColumns) - { - if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) - throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); - } - - items.Add(new SqlSyncItem(table, ChangeType.Insert, values)); - //snapshotItems.Add(values[table.PrimaryColumnName]); - _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); - } - catch (ArgumentException) - { - var duplicateColumns = Enumerable.Range(0, r.FieldCount).Select(index => r.GetName(index)).GroupBy(_ => _).Where(_ => _.Count() > 1).Select(_ => _.Key).ToArray(); - _logger?.Error($"Duplicate columns found in table {table.NameWithSchema}: {string.Join(",", duplicateColumns)}"); - throw; - } - } - } - catch (Exception) - { - _logger?.Error($"InitialSnapshotQuery: {cmd.CommandText}"); - throw; - } - } - - if (!fromAnchor.IsNull()) - { - cmd.CommandText = table.IncrementalAddOrUpdatesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(index => r.GetName(index), index => - { - var dbValue = r.GetValue(index); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }, StringComparer.OrdinalIgnoreCase); - - foreach (var skipColumn in table.SkipColumns) - { - if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) - throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); - } - - items.Add(new SqlSyncItem(table, DetectChangeType(values), - values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); - - _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); - } - } - catch (Exception) - { - _logger?.Error($"IncrementalAddOrUpdatesQuery: {cmd.CommandText}"); - throw; - } - - cmd.CommandText = table.IncrementalDeletesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId); - - try - { - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(_ => r.GetName(_), _ => - { - var dbValue = r.GetValue(_); - if (dbValue == DBNull.Value) - { - return null; - } - return dbValue; - }); - items.Add(new SqlSyncItem(table, ChangeType.Delete, values)); - _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); - } - } - catch (Exception) - { - _logger?.Error($"IncrementalDeletesQuery: {cmd.CommandText}"); - throw; - } - } - } - - tr.Commit(); - - var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); - - _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resChangeSet; - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - return new SyncVersion(version, minVersion); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); - } - } - - public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqlConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(cancellationToken); - - using var cmd = new SqlCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - tr.Commit(); - - return new SyncVersion(version, newMinVersion); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); - } - } - - public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - await InitializeStoreAsync(cancellationToken); - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); - - table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table '{name}'"); - } - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); - - table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table '{name}'"); - } - - using var connection = new SqlConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - await DisableChangeTrackingForTable(cmd, table, cancellationToken); - } - - private async Task DisableChangeTrackingForTable(SqlCommand cmd, SqlSyncTable table, CancellationToken cancellationToken) - { - var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); - var dropTriggerCommand = new Func((op) => $@"DROP TRIGGER [{table.Schema}].[__{table.NameWithSchemaRaw}_ct-{op}__]"); - - cmd.CommandText = existsTriggerCommand("INSERT"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("UPDATE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - cmd.CommandText = existsTriggerCommand("DELETE"); - if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) - { - cmd.CommandText = dropTriggerCommand("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - } - } +using JetBrains.Annotations; +using Microsoft.Data.SqlClient; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace CoreSync.SqlServer +{ + public class SqlSyncProvider : ISyncProvider + { + private bool _initialized = false; + private Guid _storeId; + private readonly ISyncLogger? _logger; + + public SqlSyncProvider(SqlSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) + { + Configuration = configuration; + ProviderMode = providerMode; + _logger = logger; + + if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && + providerMode == ProviderMode.Bidirectional) + { + throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); + } + } + + public SqlSyncConfiguration Configuration { get; } + public ProviderMode ProviderMode { get; } + public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); + + public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, Func? onConflictFunc = null, CancellationToken cancellationToken = default) + { + Validate.NotNull(changeSet, nameof(changeSet)); + Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); + Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); + + await InitializeStoreAsync(cancellationToken); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); + + using var c = new SqlConnection(Configuration.ConnectionString); + var messageLog = new List(); + + try + { + c.InfoMessage += (s, e) => messageLog.Add(e); + await c.OpenAsync(cancellationToken); + + //await DisableConstraintsForChangeSetTables(c, changeSet); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = $"DECLARE @session uniqueidentifier; SET @session = @sync_client_id_binary; SET CONTEXT_INFO @session"; + cmd.Parameters.Add(new SqlParameter("@sync_client_id_binary", changeSet.SourceAnchor.StoreId)); + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.Parameters.Clear(); + + cmd.CommandText = $"SELECT CONTEXT_INFO()"; + var contextInfo = await cmd.ExecuteScalarAsync(cancellationToken); + cmd.Parameters.Clear(); + + var remainingItems = changeSet.Items.ToList(); + int pass = 0; + while (remainingItems.Count > 0) + { + pass++; + var failedItems = new List(); + int appliedInPass = 0; + + foreach (var item in remainingItems) + { + var table = (SqlSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); + + if (table == null) + { + continue; + } + + bool syncForceWrite = false; + var itemChangeType = item.ChangeType; + bool itemHandled = false; + + retryWrite: + cmd.Parameters.Clear(); + + table.SetupCommand(cmd, itemChangeType, item.Values); + + cmd.Parameters.Add(new SqlParameter("@last_sync_version", changeSet.TargetAnchor.Version)); + cmd.Parameters.Add(new SqlParameter("@sync_force_write", syncForceWrite)); + + int affectedRows; + + try + { + affectedRows = cmd.ExecuteNonQuery(); + + if (affectedRows > 0) + { + _logger?.Trace($"[{_storeId}] Successfully applied {item}"); + } + } + catch (Exception ex) + { + _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}Generated SQL:{Environment.NewLine}{cmd.CommandText}"); + throw new SynchronizationException($"Unable to {itemChangeType} item {item} to store for table {table}", ex); + } + + if (affectedRows == 0) + { + if (itemChangeType == ChangeType.Insert) + { + //If we can't apply an insert means that we already + //applied the insert or another record with same values (see primary key) + //is already present in table. + var executedCommand = cmd.CommandText; + + cmd.CommandText = table.SelectExistingQuery; + cmd.Parameters.Clear(); + var valueItem = item.Values[table.PrimaryColumnName]; + + cmd.Parameters.Add(table.Columns[table.PrimaryColumnName].CreateParameter("@PrimaryColumnParameter", valueItem)); + + if (1 == (int)await cmd.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) + { + itemChangeType = ChangeType.Update; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Unable to {itemChangeType} item {item} on table {table} (pass {pass}). Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}{Environment.NewLine}Generated SQL:{Environment.NewLine}{executedCommand}"); + } + } + else if (itemChangeType == ChangeType.Update || + itemChangeType == ChangeType.Delete) + { + if (syncForceWrite) + { + if (itemChangeType == ChangeType.Delete) + { + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemHandled = true; + } + else + { + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemChangeType = ChangeType.Insert; + goto retryWrite; + } + } + else + { + //conflict detected + var res = onConflictFunc?.Invoke(item); + if (res.HasValue && res.Value == ConflictResolution.ForceWrite) + { + _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); + + syncForceWrite = true; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); + itemHandled = true; + } + } + } + } + + if (affectedRows > 0) + { + itemHandled = true; + appliedInPass++; + } + + if (!itemHandled) + { + failedItems.Add(item); + } + } + + if (failedItems.Count == 0 || appliedInPass == 0) + { + if (failedItems.Count > 0) + { + _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); + } + break; + } + + _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); + remainingItems = failedItems; + } + + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + var resAnchor = new SyncAnchor(_storeId, version); + + _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resAnchor; + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + var exceptionMessage = $"An exception occurred during synchronization:{Environment.NewLine}Errors:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.SelectMany(_ => _.Errors.Cast()))}{Environment.NewLine}Messages:{Environment.NewLine}{string.Join(Environment.NewLine, messageLog.Select(_ => _.Message))}"; + throw new SyncErrorException(exceptionMessage, ex); + } + finally + { + //await RestoreConstraintsForChangeSetTables(c, changeSet); + } + } + + private async Task DisableConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) + { + using var cmd = new SqlCommand(); + cmd.Connection = connection; + + foreach (SqlSyncTable table in changeSet.Items + .Select(_ => _.TableName) + .Distinct() + .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) + .Cast()) + { + cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} NOCHECK CONSTRAINT ALL"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + private async Task RestoreConstraintsForChangeSetTables(SqlConnection connection, SyncChangeSet changeSet, CancellationToken cancellationToken = default) + { + using var cmd = new SqlCommand(); + cmd.Connection = connection; + + foreach (SqlSyncTable table in changeSet.Items + .Select(_ => _.TableName) + .Distinct() + .Select(tableName => Configuration.Tables.First(_ => _.Name == tableName)) + .Cast()) + { + cmd.CommandText = $"ALTER TABLE {table.NameWithSchema} WITH CHECK CHECK CONSTRAINT ALL"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) + { + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + try + { + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [dbo].[__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(_storeId, (long)version); + } + + private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + // Report a missing anchor row as the null sentinel, matching + // GetLastLocalAnchorForStoreAsync and the other providers. Version 0 is not a sentinel: it + // is a real-looking version that callers cannot tell apart from "synchronized up to 0". + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(otherStoreId, (long)version); + } + + public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + return _storeId; + } + + private ChangeType DetectChangeType(Dictionary values) + { + if (values.TryGetValue("__OP", out var syncChangeOperation)) + { + return (syncChangeOperation?.ToString()) switch + { + "I" => ChangeType.Insert, + "U" => ChangeType.Update, + "D" => ChangeType.Delete, + _ => throw new NotSupportedException(), + }; + } + + return ChangeType.Insert; + } + + private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) + { + if (_initialized) + return; + + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using (var connection = new SqlConnection(Configuration.ConnectionString)) + { + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + if (!tableNames.Contains("__CORE_SYNC_CT")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_CT]( + [ID] [bigint] IDENTITY(1,1) NOT NULL, + [TBL] [nvarchar](1024) NOT NULL, + [OP] [char](1) NOT NULL, + [PK_Int] [int] SPARSE NULL, + [PK_String] [nvarchar](1024) SPARSE NULL, + [PK_Guid] [uniqueidentifier] SPARSE NULL, + [SRC] [varbinary](128) NULL, + CONSTRAINT [PK___CORE_SYNC_CT] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Int-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_Int] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_String-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_String] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $@"CREATE NONCLUSTERED INDEX [PK_Guid-Index] ON [dbo].[__CORE_SYNC_CT] +( + [PK_Guid] ASC +) +INCLUDE([TBL]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (!tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]( + [ID] [uniqueidentifier] NOT NULL, + [LOCAL_VERSION] [BIGINT] NULL, + [REMOTE_VERSION] [BIGINT] NULL + CONSTRAINT [PK___CORE_SYNC_REMOTE_ANCHOR] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (!tableNames.Contains("__CORE_SYNC_LOCAL_ID")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE [dbo].[__CORE_SYNC_LOCAL_ID]( + [ID] [uniqueidentifier] NOT NULL + CONSTRAINT [PK___CORE_SYNC_LOCAL_ID] PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "SELECT TOP 1 [ID] FROM [__CORE_SYNC_LOCAL_ID]"; + var localId = await cmd.ExecuteScalarAsync(cancellationToken); + if (localId == null) + { + localId = Guid.NewGuid(); + cmd.CommandText = $"INSERT INTO [__CORE_SYNC_LOCAL_ID] ([ID]) VALUES (@id)"; + cmd.Parameters.Add(new SqlParameter("@id", localId)); + if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + throw new InvalidOperationException(); + } + cmd.Parameters.Clear(); + } + + _storeId = (Guid)localId; + } + + using (var cmd = connection.CreateCommand()) + { + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); + if (primaryKeyIndexName == null) + { + throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't exist or it doesn't have a primary key"); + } + + var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); + + if (primaryKeyColumns.Length != 1) + { + throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); + } + + table.Columns = (await connection.GetTableColumnsAsync(table, cancellationToken)) + .ToDictionary(_ => _.Name, _ => _, StringComparer.OrdinalIgnoreCase); + table.PrimaryColumnName = primaryKeyColumns[0]; + table.PrimaryKeyColumns = primaryKeyColumns; + + if (table.SkipColumns.Any(skipColumn => string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0)) + { + throw new InvalidOperationException($"Column to skip in synchronization ('{table.PrimaryColumnName}') can't be the primary column"); + } + + var allColumns = table.Columns.Keys.Except(table.SkipColumns); + + var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); + + if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) + { + throw new NotSupportedException($"Table '{table.Name}' has one column with reserved name '__OP'"); + } + + cmd.CommandText = $@"SELECT COUNT(*) FROM SYS.IDENTITY_COLUMNS WHERE OBJECT_NAME(OBJECT_ID) = @tablename AND OBJECT_SCHEMA_NAME(object_id) = @schemaname"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@tablename", table.Name); + cmd.Parameters.AddWithValue("@schemaname", table.Schema); + + table.HasTableIdentityColumn = ((int)await cmd.ExecuteScalarAsync(cancellationToken) == 1); + } + } + } + + _initialized = true; + } + + public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + using var cmd = connection.CreateCommand(); + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + var primaryKeyIndexName = (await connection.GetPrimaryKeyIndexesAsync(table, cancellationToken)).FirstOrDefault(); //dbTable.Indexes.Cast().FirstOrDefault(_ => _.IsClustered && _.IndexKeyType == IndexKeyType.DriPrimaryKey); + if (primaryKeyIndexName == null) + { + throw new InvalidOperationException($"Table '{table.NameWithSchema}' doesn't have a primary key"); + } + + var primaryKeyColumns = await connection.GetIndexColumnNamesAsync(table, primaryKeyIndexName, cancellationToken); //primaryKeyIndexName.IndexedColumns.Cast().ToList(); + var allColumns = (await connection.GetTableColumnsAsync(table, cancellationToken)).Select(_ => _.Name).ToArray(); + //var tableColumns = allColumns.Where(_ => !primaryKeyColumns.Any(kc => kc == _)).ToArray(); + + //if (primaryKeyColumns.Length != 1) + //{ + // throw new NotSupportedException($"Table '{table.Name}' has more than one column as primary key"); + //} + + if (allColumns.Any(_ => string.CompareOrdinal(_, "__OP") == 0)) + { + throw new NotSupportedException($"Table '{table.NameWithSchema}' has one column with reserved name '__OP'"); + } + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + } + + private async Task SetupTableForFullChangeDetection(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) + { + var forceReloadOnInsert = table.ForceReloadInsertedRecords || table.HasTableIdentityColumn; + var forceReloadCode = "-- Table is set to NOT force reload the record after inserting"; + if (forceReloadOnInsert) + { + forceReloadCode = $@" + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}) + SELECT '{table.NameWithSchema}', 'U' , INSERTED{$".[{table.PrimaryColumnName}]"} + FROM INSERTED"; + } + + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] +ON {table.NameWithSchema} +AFTER {op} +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) + SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() + FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} + + {forceReloadCode} + +END"); + + cmd.CommandText = existsTriggerCommand("INSERT"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + private async Task SetupTableForUpdatesOrDeletesOnly(SqlSyncTable table, SqlCommand cmd, CancellationToken cancellationToken = default) + { + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var createTriggerCommand = new Func((op) => $@"CREATE TRIGGER [__{table.NameWithSchemaRaw}_ct-{op}__] +ON {table.NameWithSchema} +AFTER {op} +AS +BEGIN + -- SET NOCOUNT ON added to prevent extra result sets from + -- interfering with SELECT statements. + SET NOCOUNT ON; + + INSERT INTO [__CORE_SYNC_CT](TBL, OP, PK_{table.PrimaryColumnType}, SRC) + SELECT '{table.NameWithSchema}', '{op[0]}' , {(op == "DELETE" ? "DELETED" : "INSERTED")}{$".[{table.PrimaryColumnName}]"}, CONTEXT_INFO() + FROM {(op == "DELETE" ? "DELETED" : "INSERTED")} +END"); + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 0) + { + cmd.CommandText = createTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + + public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) + { + var connStringBuilder = new SqlConnectionStringBuilder(Configuration.ConnectionString); + if (string.IsNullOrWhiteSpace(connStringBuilder.InitialCatalog)) + throw new InvalidOperationException("Invalid connection string: InitialCatalog property is missing"); + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + var tableNames = await connection.GetTableNamesAsync(cancellationToken); + if (tableNames.Contains("__CORE_SYNC_CT")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_CT]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (tableNames.Contains("__CORE_SYNC_REMOTE_ANCHOR")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_REMOTE_ANCHOR]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + if (tableNames.Contains("__CORE_SYNC_LOCAL_ID")) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE [dbo].[__CORE_SYNC_LOCAL_ID]"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + using (var cmd = connection.CreateCommand()) + { + foreach (SqlSyncTable table in Configuration.Tables.Cast()) + { + await DisableChangeTrackingForTable(cmd, table, cancellationToken); + } + } + } + + public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters, SyncDirection syncDirection, string[]? tables = null, CancellationToken cancellationToken = default) + { + syncFilterParameters ??= []; + + var tablesToSync = Configuration.ResolveTableFilter(tables); + + var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); + + + using var c = new SqlConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqlCommand(); + var items = new List(); + + using var tr = c.BeginTransaction();// IsolationLevel.Snapshot)) + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. + if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) + { + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } + + foreach (SqlSyncTable table in tablesToSync.Cast()) + { + if (table.SyncDirection != SyncDirection.UploadAndDownload && + table.SyncDirection != syncDirection) + continue; + + _logger?.Info($"Getting changes for table {table.NameWithSchema}"); + //var snapshotItems = new HashSet(); + + if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) + { + if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) + { + throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); + } + + cmd.CommandText = table.InitialSnapshotQuery; + cmd.Parameters.Clear(); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + try + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(index => r.GetName(index), index => + { + var dbValue = r.GetValue(index); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }, StringComparer.OrdinalIgnoreCase); + + foreach (var skipColumn in table.SkipColumns) + { + if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) + throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); + } + + items.Add(new SqlSyncItem(table, ChangeType.Insert, values)); + //snapshotItems.Add(values[table.PrimaryColumnName]); + _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); + } + catch (ArgumentException) + { + var duplicateColumns = Enumerable.Range(0, r.FieldCount).Select(index => r.GetName(index)).GroupBy(_ => _).Where(_ => _.Count() > 1).Select(_ => _.Key).ToArray(); + _logger?.Error($"Duplicate columns found in table {table.NameWithSchema}: {string.Join(",", duplicateColumns)}"); + throw; + } + } + } + catch (Exception) + { + _logger?.Error($"InitialSnapshotQuery: {cmd.CommandText}"); + throw; + } + } + + if (!fromAnchor.IsNull()) + { + cmd.CommandText = table.IncrementalAddOrUpdatesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(index => r.GetName(index), index => + { + var dbValue = r.GetValue(index); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }, StringComparer.OrdinalIgnoreCase); + + foreach (var skipColumn in table.SkipColumns) + { + if (string.CompareOrdinal(skipColumn, table.PrimaryColumnName) == 0) + throw new InvalidOperationException($"Column to skip in synchronization ('{skipColumn}') can't be the primary column"); + } + + items.Add(new SqlSyncItem(table, DetectChangeType(values), + values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); + + _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); + } + } + catch (Exception) + { + _logger?.Error($"IncrementalAddOrUpdatesQuery: {cmd.CommandText}"); + throw; + } + + cmd.CommandText = table.IncrementalDeletesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId); + + try + { + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(_ => r.GetName(_), _ => + { + var dbValue = r.GetValue(_); + if (dbValue == DBNull.Value) + { + return null; + } + return dbValue; + }); + items.Add(new SqlSyncItem(table, ChangeType.Delete, values)); + _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); + } + } + catch (Exception) + { + _logger?.Error($"IncrementalDeletesQuery: {cmd.CommandText}"); + throw; + } + } + } + + tr.Commit(); + + var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); + + _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resChangeSet; + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + return new SyncVersion(version, minVersion); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); + } + } + + public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqlConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(cancellationToken); + + using var cmd = new SqlCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + tr.Commit(); + + return new SyncVersion(version, newMinVersion); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); + } + } + + public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + await InitializeStoreAsync(cancellationToken); + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); + + table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table '{name}'"); + } + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.NameWithSchema == name); + + table ??= Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table '{name}'"); + } + + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + await DisableChangeTrackingForTable(cmd, table, cancellationToken); + } + + private async Task DisableChangeTrackingForTable(SqlCommand cmd, SqlSyncTable table, CancellationToken cancellationToken) + { + var existsTriggerCommand = new Func((op) => $@"select COUNT(*) from sys.objects where schema_id=SCHEMA_ID('{table.Schema}') AND type='TR' and name='__{table.NameWithSchemaRaw}_ct-{op}__'"); + var dropTriggerCommand = new Func((op) => $@"DROP TRIGGER [{table.Schema}].[__{table.NameWithSchemaRaw}_ct-{op}__]"); + + cmd.CommandText = existsTriggerCommand("INSERT"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("UPDATE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + cmd.CommandText = existsTriggerCommand("DELETE"); + if (((int)await cmd.ExecuteScalarAsync(cancellationToken)) == 1) + { + cmd.CommandText = dropTriggerCommand("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + } + } } \ No newline at end of file diff --git a/src/CoreSync.SqlServerCT/ChangeTrackingDatabaseOptions.cs b/src/CoreSync.SqlServerCT/ChangeTrackingDatabaseOptions.cs index d2b464e..1ad9e99 100644 --- a/src/CoreSync.SqlServerCT/ChangeTrackingDatabaseOptions.cs +++ b/src/CoreSync.SqlServerCT/ChangeTrackingDatabaseOptions.cs @@ -1,10 +1,16 @@ -namespace CoreSync.SqlServerCT +namespace CoreSync.SqlServerCT { /// /// The database-level change tracking settings as reported by sys.change_tracking_databases. /// - internal sealed class ChangeTrackingDatabaseOptions + public sealed class ChangeTrackingDatabaseOptions { + /// + /// Initializes a new instance of the class. + /// + /// The retention period, expressed in . + /// The unit is expressed in. + /// Whether SQL Server automatically removes expired change tracking data. public ChangeTrackingDatabaseOptions(int retentionPeriod, ChangeRetentionUnit retentionPeriodUnit, bool autoCleanup) { RetentionPeriod = retentionPeriod; @@ -12,12 +18,22 @@ public ChangeTrackingDatabaseOptions(int retentionPeriod, ChangeRetentionUnit re AutoCleanup = autoCleanup; } + /// + /// Gets how long change history is kept, expressed in . + /// public int RetentionPeriod { get; } + /// + /// Gets the unit is expressed in. + /// public ChangeRetentionUnit RetentionPeriodUnit { get; } + /// + /// Gets a value indicating whether SQL Server automatically removes expired change tracking data. + /// public bool AutoCleanup { get; } + /// public override string ToString() => $"CHANGE_RETENTION = {RetentionPeriod} {RetentionPeriodUnit}, AUTO_CLEANUP = {(AutoCleanup ? "ON" : "OFF")}"; } } diff --git a/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs b/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs index 2c7b1ba..1f1a432 100644 --- a/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs +++ b/src/CoreSync.SqlServerCT/SqlServerCTProvider.cs @@ -556,6 +556,36 @@ [ID] ASC _initialized = true; } + /// + /// Applies the configured change tracking retention to the database and reports the settings + /// actually in effect afterwards, without provisioning anything else. + /// + /// A token to cancel the operation. + /// + /// The settings read back from sys.change_tracking_databases, or null when change + /// tracking is not enabled on the database. + /// + /// + /// Retention is normally reconciled as part of . This exists for + /// callers that let an operator edit the retention on its own: changing the stored setting has no + /// effect until an ALTER DATABASE is actually issued, and a setting that looks saved but was + /// never applied is precisely how a database ends up quietly keeping a week of history when it was + /// configured for a month. The read-back lets the caller show what the database really holds rather + /// than what was requested. + /// + /// + /// The statement failed - most often because the account lacks the ALTER DATABASE permission. + /// + public async Task ApplyChangeRetentionAsync(CancellationToken cancellationToken = default) + { + using var connection = new SqlConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + await ReconcileChangeTrackingRetentionAsync(connection, cancellationToken); + + return await connection.GetChangeTrackingOptionsAsync(cancellationToken); + } + public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) { await InitializeStoreAsync(cancellationToken); diff --git a/src/CoreSync.Sqlite/SqliteSyncProvider.cs b/src/CoreSync.Sqlite/SqliteSyncProvider.cs index 4777b88..543cfd4 100644 --- a/src/CoreSync.Sqlite/SqliteSyncProvider.cs +++ b/src/CoreSync.Sqlite/SqliteSyncProvider.cs @@ -1,1011 +1,1011 @@ -using JetBrains.Annotations; -using Microsoft.Data.Sqlite; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; - -namespace CoreSync.Sqlite -{ - public class SqliteSyncProvider : ISyncProvider - { - private bool _initialized = false; - private Guid _storeId; - private readonly ISyncLogger? _logger; - - public SqliteSyncProvider(SqliteSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) - { - Configuration = configuration; - ProviderMode = providerMode; - _logger = logger; - - if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && - providerMode == ProviderMode.Bidirectional) - { - throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); - } - } - - public SqliteSyncConfiguration Configuration { get; } - public ProviderMode ProviderMode { get; } - public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); - - public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func? onConflictFunc = null, CancellationToken cancellationToken = default) - { - Validate.NotNull(changeSet, nameof(changeSet)); - Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); - Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); - - await InitializeStoreAsync(cancellationToken); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); - - using var c = new SqliteConnection(Configuration.ConnectionString); // +";Foreign Keys=False")) - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - // Reusable prepared commands, cached by (change type + table + column shape). The - // per-item write loop then only binds parameter values instead of rebuilding the SQL - // text and re-allocating parameters for each of (potentially) 100K+ items; SQLite keeps - // the compiled statement as long as the command text is unchanged. - var preparedCommands = new Dictionary(); - var existsCommands = new Dictionary(); - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - var remainingItems = changeSet.Items.ToList(); - - PreparedItemCommand GetItemCommand(SqliteSyncTable itemTable, ChangeType changeType, List> validColumns) - { - var columns = new string[validColumns.Count]; - for (int i = 0; i < validColumns.Count; i++) - { - columns[i] = validColumns[i].Key; - } - - var cacheKey = $"{(int)changeType}|{itemTable.Name}|{string.Join(",", columns)}"; - if (preparedCommands.TryGetValue(cacheKey, out var cached)) - { - return cached; - } - - var command = new SqliteCommand { Connection = c, Transaction = tr }; - var valueParameters = new SqliteParameter[columns.Length]; - - void AddValueParameters() - { - for (int i = 0; i < columns.Length; i++) - { - var valueParameter = new SqliteParameter($"@p{i}", DBNull.Value); - command.Parameters.Add(valueParameter); - valueParameters[i] = valueParameter; - } - } - - SqliteParameter? primaryKeyParameter = null; - SqliteParameter? lastSyncVersionParameter = null; - SqliteParameter? forceWriteParameter = null; - - switch (changeType) - { - case ChangeType.Insert: - command.CommandText = itemTable.BuildInsertCommandText(columns); - AddValueParameters(); - break; - - case ChangeType.Update: - command.CommandText = itemTable.BuildUpdateCommandText(columns); - primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); - command.Parameters.Add(primaryKeyParameter); - AddValueParameters(); - lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); - command.Parameters.Add(lastSyncVersionParameter); - forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); - command.Parameters.Add(forceWriteParameter); - break; - - case ChangeType.Delete: - command.CommandText = itemTable.BuildDeleteCommandText(); - valueParameters = Array.Empty(); - primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); - command.Parameters.Add(primaryKeyParameter); - lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); - command.Parameters.Add(lastSyncVersionParameter); - forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); - command.Parameters.Add(forceWriteParameter); - break; - } - - var preparedCommand = new PreparedItemCommand(command, valueParameters, primaryKeyParameter, lastSyncVersionParameter, forceWriteParameter); - preparedCommands[cacheKey] = preparedCommand; - return preparedCommand; - } - - SqliteCommand GetExistsCommand(SqliteSyncTable itemTable) - { - if (!existsCommands.TryGetValue(itemTable.Name, out var existsCommand)) - { - existsCommand = new SqliteCommand { Connection = c, Transaction = tr }; - existsCommand.CommandText = itemTable.SelectExistingQuery; - existsCommand.Parameters.Add(new SqliteParameter("@PrimaryColumnParameter", DBNull.Value)); - existsCommands[itemTable.Name] = existsCommand; - } - - return existsCommand; - } - int pass = 0; - while (remainingItems.Count > 0) - { - pass++; - var failedItems = new List(); - int appliedInPass = 0; - - foreach (var item in remainingItems) - { - var table = (SqliteSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); - if (table == null) - { - continue; - } - - bool syncForceWrite = false; - var itemChangeType = item.ChangeType; - bool itemHandled = false; - - retryWrite: - var validColumns = table.GetValuesForValidColumns(item.Values); - var prepared = GetItemCommand(table, itemChangeType, validColumns); - var itemCommand = prepared.Command; - - // ValueParameters is empty for deletes (which bind only the primary key); - // for inserts/updates it is parallel to validColumns. - for (int i = 0; i < prepared.ValueParameters.Length; i++) - { - prepared.ValueParameters[i].Value = validColumns[i].Value.Value ?? DBNull.Value; - } - - if (prepared.PrimaryKeyParameter != null) - { - prepared.PrimaryKeyParameter.Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; - } - - if (prepared.LastSyncVersionParameter != null) - { - prepared.LastSyncVersionParameter.Value = changeSet.TargetAnchor.Version; - } - - if (prepared.ForceWriteParameter != null) - { - prepared.ForceWriteParameter.Value = syncForceWrite; - } - - int affectedRows = 0; - - try - { - affectedRows = await itemCommand.ExecuteNonQueryAsync(cancellationToken); - - if (affectedRows > 0) - { - _logger?.Trace($"[{_storeId}] Successfully applied {item}"); - } - else - { - _logger?.Trace($"[{_storeId}] No row affected on insert for {item} (maybe it's a conflict? or a different table schema between client and server):{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); - } - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - //throw new SynchronizationException($"Unable to {item} item to store for table {table}", ex); - _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}{ex}{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); - } - - if (affectedRows == 0) - { - if (itemChangeType == ChangeType.Insert) - { - //If we can't apply an insert means that we already - //applied the insert or another record with same values (see primary key) - //is already present in table. - //In any case we can't proceed - var existsCommand = GetExistsCommand(table); - existsCommand.Parameters[0].Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; - if (1 == (long)await existsCommand.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) - { - itemChangeType = ChangeType.Update; - goto retryWrite; - } - else - { - _logger?.Warning($"Unable to {item}: much probably there is a foreign key constraint issue logged before (pass {pass})"); - } - } - else if (itemChangeType == ChangeType.Update || - itemChangeType == ChangeType.Delete) - { - if (syncForceWrite) - { - if (itemChangeType == ChangeType.Delete) - { - //item is already deleted in data store - //so this means that we're going to delete a already deleted record - //i.e. nothing to do - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemHandled = true; - } - else - { - //if user wants to update forcely a deleted record means that - //he asctually wants to insert it again in store - _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); - itemChangeType = ChangeType.Insert; - goto retryWrite; - } - } - else - { - //conflict detected - var res = onConflictFunc?.Invoke(item); - if (res.HasValue && res.Value == ConflictResolution.ForceWrite) - { - _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); - - syncForceWrite = true; - goto retryWrite; - } - else - { - _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); - itemHandled = true; - } - } - } - } - - if (affectedRows > 0) - { - itemHandled = true; - appliedInPass++; - } - - if (!itemHandled) - { - failedItems.Add(item); - } - } - - if (failedItems.Count == 0 || appliedInPass == 0) - { - if (failedItems.Count > 0) - { - _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); - } - break; - } - - _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); - remainingItems = failedItems; - } - - // Stamp every change-tracking row produced while applying this change set as - // originating from the source store, so those changes are not echoed back on the - // next GetChanges (IncrementalAddOrUpdatesQuery filters on SRC != @sourceId). - // The AFTER INSERT/UPDATE/DELETE triggers create these rows with SRC = NULL and - // IDs strictly greater than the version captured before applying, so a single bulk - // UPDATE here replaces the former per-item SELECT MAX(ID) + UPDATE (which executed - // two extra statements for every applied item). - cmd.CommandText = "UPDATE [__CORE_SYNC_CT] SET [SRC] = @sourceId WHERE [ID] > @version AND [SRC] IS NULL"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@sourceId", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); - cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - var resAnchor = new SyncAnchor(_storeId, version); - - _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resAnchor; - } - catch (Exception ex) - { - _logger?.Error($"[{_storeId}] Unable to complete the apply changes:{Environment.NewLine}{ex}"); - tr.Rollback(); - throw; - } - finally - { - foreach (var preparedCommand in preparedCommands.Values) - { - preparedCommand.Command.Dispose(); - } - - foreach (var existsCommand in existsCommands.Values) - { - existsCommand.Dispose(); - } - } - } - - /// - /// A reusable command for applying a single change type against a table with a fixed column - /// shape. The command text is built once; each item only rebinds the parameter values. - /// - private sealed class PreparedItemCommand - { - public PreparedItemCommand( - SqliteCommand command, - SqliteParameter[] valueParameters, - SqliteParameter? primaryKeyParameter, - SqliteParameter? lastSyncVersionParameter, - SqliteParameter? forceWriteParameter) - { - Command = command; - ValueParameters = valueParameters; - PrimaryKeyParameter = primaryKeyParameter; - LastSyncVersionParameter = lastSyncVersionParameter; - ForceWriteParameter = forceWriteParameter; - } - - public SqliteCommand Command { get; } - - public SqliteParameter[] ValueParameters { get; } - - public SqliteParameter? PrimaryKeyParameter { get; } - - public SqliteParameter? LastSyncVersionParameter { get; } - - public SqliteParameter? ForceWriteParameter { get; } - } - - public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) - { - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; - cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); - cmd.Parameters.AddWithValue("@version", version); - - if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; - - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - tr.Commit(); - - _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - - public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters = null, SyncDirection syncDirection = SyncDirection.UploadAndDownload, string[]? tables = null, CancellationToken cancellationToken = default) - { - syncFilterParameters ??= []; - - var tablesToSync = Configuration.ResolveTableFilter(tables); - - var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); - - //if fromVersion is 0 means that client needs actually to initialize its local datastore - - //await InitializeStoreAsync(); - - var now = DateTime.Now; - - _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - var items = new List(); - - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - cmd.Parameters.Clear(); - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be - // served is the one immediately before it. - if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) - { - // Traced before throwing: this runs ahead of the per-table loop below, so without - // it a session that fails here leaves no account of why it stopped. - var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); - _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); - throw anchorTooOld; - } - - foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) - { - if (table.SyncDirection != SyncDirection.UploadAndDownload && - table.SyncDirection != syncDirection) - continue; - - //var snapshotItems = new HashSet(); - - if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) - { - if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) - { - throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); - } - - cmd.CommandText = table.InitialSnapshotQuery; - cmd.Parameters.Clear(); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - using var r = await cmd.ExecuteReaderAsync(cancellationToken); - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount) - .ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - items.Add(new SqliteSyncItem(table, ChangeType.Insert, values)); - //snapshotItems.Add(values[table.PrimaryColumnName]); - _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); - } - } - - if (!fromAnchor.IsNull()) - { - cmd.CommandText = table.IncrementalAddOrUpdatesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); - foreach (var syncFilterParameter in syncFilterParameters) - { - cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); - } - - using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) - { - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - //if (snapshotItems.Contains(values[table.PrimaryColumnName])) - // continue; - - items.Add(new SqliteSyncItem(table, DetectChangeType(values), - values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); - _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); - } - } - - cmd.CommandText = table.IncrementalDeletesQuery; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@version", fromAnchor.Version); - cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); - using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) - { - while (await r.ReadAsync(cancellationToken)) - { - var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); - items.Add(new SqliteSyncItem(table, ChangeType.Delete, values)); - _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); - } - } - } - } - - tr.Commit(); - - var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); - - _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); - - return resChangeSet; - - } - catch (Exception) - { - tr.Rollback(); - throw; - } - - } - - private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(_storeId, (long)version); - } - - private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) - { - //await InitializeAsync(); - - using var c = new SqliteConnection(Configuration.ConnectionString); - await c.OpenAsync(cancellationToken); - - using var cmd = c.CreateCommand(); - cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; - cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); - - var version = await cmd.ExecuteScalarAsync(cancellationToken); - - if (version == null || version == DBNull.Value) - return SyncAnchor.Null; - - return new SyncAnchor(otherStoreId, (long)version); - } - - public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - return _storeId; - } - - private static ChangeType DetectChangeType(Dictionary values) - { - if (values.TryGetValue("__OP", out var syncChangeOperation)) - { - switch (syncChangeOperation?.ToString()) - { - case "I": - return ChangeType.Insert; - - case "U": - return ChangeType.Update; - - case "D": - return ChangeType.Delete; - - default: - throw new NotSupportedException(); - } - } - - return ChangeType.Insert; - } - - private static object? GetValueFromRecord(SqliteSyncTable table, string columnName, int columnOrdinal, SqliteDataReader r) - { - if (r.IsDBNull(columnOrdinal)) - return null; - - if (table.RecordType == null) - return r.GetValue(columnOrdinal); - - var property = table.RecordType.GetProperty(columnName); - if (property != null) - return GetValueFromRecord(r, columnOrdinal, property.PropertyType); - - property = table.RecordType.GetProperties().FirstOrDefault(_ => - { - var columnAttribute = _.GetCustomAttribute(false); - if (columnAttribute != null) - { - return columnAttribute.Name == columnName; - } - - return false; - }); - - if (property != null) - return GetValueFromRecord(r, columnOrdinal, property.PropertyType); - - //fallback to getvalue - return r.GetValue(columnOrdinal); - } - - private static object GetValueFromRecord(SqliteDataReader r, int columnOrdinal, Type propertyType) - { - if (propertyType == typeof(string)) - return r.GetString(columnOrdinal); - if (propertyType == typeof(DateTime)) - return r.GetDateTime(columnOrdinal); - if (propertyType == typeof(int)) - return r.GetInt32(columnOrdinal); - if (propertyType == typeof(bool)) - return r.GetBoolean(columnOrdinal); - if (propertyType == typeof(byte)) - return r.GetByte(columnOrdinal); - if (propertyType == typeof(char)) - return r.GetChar(columnOrdinal); - if (propertyType == typeof(short)) - return r.GetInt16(columnOrdinal); - if (propertyType == typeof(long)) - return r.GetInt64(columnOrdinal); - if (propertyType == typeof(decimal)) - return r.GetDecimal(columnOrdinal); - if (propertyType == typeof(double)) - return r.GetDouble(columnOrdinal); - if (propertyType == typeof(float)) - return r.GetFloat(columnOrdinal); - - //fallback to getvalue - return r.GetValue(columnOrdinal); - } - - private static string TypeToPrintFormat(string type) - { - if (type == "INTEGER") - return "%d"; - if (type == "TEXT") - return "%s"; - - return "%s"; - } - - private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) - { - if (_initialized) - return; - - using (var connection = new SqliteConnection(Configuration.ConnectionString)) - { - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - cmd.CommandText = $@"CREATE TABLE IF NOT EXISTS __CORE_SYNC_CT -(ID INTEGER PRIMARY KEY, TBL TEXT NOT NULL COLLATE NOCASE, OP CHAR NOT NULL, PK_{SqlitePrimaryColumnType.Int} INTEGER NULL, PK_{SqlitePrimaryColumnType.Text} TEXT NULL COLLATE NOCASE, PK_{SqlitePrimaryColumnType.Blob} BLOB NULL, SRC TEXT NULL COLLATE NOCASE)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Int}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Int})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Text}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Text})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Blob}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Blob})"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_REMOTE_ANCHOR (ID TEXT NOT NULL PRIMARY KEY, LOCAL_VERSION LONG NULL, REMOTE_VERSION LONG NULL)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_LOCAL_ID (ID TEXT NOT NULL PRIMARY KEY)"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = $"SELECT ID FROM __CORE_SYNC_LOCAL_ID LIMIT 1"; - var localId = await cmd.ExecuteScalarAsync(cancellationToken); - if (localId == null) - { - localId = Guid.NewGuid().ToString(); - cmd.CommandText = $"INSERT INTO __CORE_SYNC_LOCAL_ID (ID) VALUES (@localId)"; - cmd.Parameters.Add(new SqliteParameter("@localId", localId)); - if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) - { - throw new InvalidOperationException(); - } - cmd.Parameters.Clear(); - } - - _storeId = Guid.Parse((string)localId); - - foreach (SqliteSyncTable table in Configuration.Tables) - { - cmd.CommandText = $"PRAGMA table_info('{table.Name}')"; - using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) - { - /* - cid name type notnull dflt_value pk - ---------- ---------- ---------- ---------- ---------- ---------- - */ - while (await reader.ReadAsync(cancellationToken)) - { - var colName = reader.GetString(1); - var colType = reader.GetString(2); - var pk = reader.GetBoolean(5); - - if (string.CompareOrdinal(colName, "__OP") == 0) - { - throw new NotSupportedException($"Unable to synchronize table '{table.Name}': one column has a reserved name '__OP'"); - } - - table.Columns.Add(colName, new SqliteColumn(colName, colType, pk)); - } - } - - if (table.Columns.Count == 0) - { - throw new InvalidOperationException($"Unable to configure table '{table}': does it exist with at least one column?"); - } - - if (table.Columns.Count(_ => _.Value.IsPrimaryKey) == 0) - { - throw new NotSupportedException($"Unable to configure table '{table}': no primary key defined"); - } - - if (table.Columns.Count(_ => _.Value.IsPrimaryKey) > 1) - { - throw new NotSupportedException($"Unable to configure table '{table}': it has more than one column as primary key"); - } - } - } - - _initialized = true; - } - - public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) - { - //if (_initialized) - // return; - - await InitializeStoreAsync(cancellationToken); - - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - foreach (var table in Configuration.Tables.Cast().Where(_ => _.Columns.Any())) - { - //var primaryKeyColumns = table.Columns.Where(_ => _.IsPrimaryKey).ToArray(); - //var tableColumns = table.Columns.Where(_ => !_.IsPrimaryKey).ToArray(); - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - //_initialized = true; - } - - private async Task SetupTableForFullChangeDetection(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) - { - var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] - AFTER {op} ON [{table.Name}] - FOR EACH ROW - BEGIN - INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); - END"); - cmd.CommandText = commandTextBase("INSERT"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - private async Task SetupTableForUpdatesOrDeletesOnly(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) - { - var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] - AFTER {op} ON [{table.Name}] - FOR EACH ROW - BEGIN - INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); - END"); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(cancellationToken); - } - - public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) - { - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - - //1. drop ct table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_CT"; - await cmd.ExecuteNonQueryAsync(); - - //2. drop remote anchor table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_REMOTE_ANCHOR"; - await cmd.ExecuteNonQueryAsync(); - - //3. drop local anchor table - cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_LOCAL_ID"; - await cmd.ExecuteNonQueryAsync(); - - //4. drop triggers — iterate table names directly; DROP TRIGGER IF EXISTS is safe even if never provisioned - foreach (SqliteSyncTable table in Configuration.Tables) - { - await DisableChangeTrackingForTable(cmd, table.Name, cancellationToken); - } - } - - public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(cancellationToken); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - return new SyncVersion(version, minVersion); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); - } - } - - public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) - { - await InitializeStoreAsync(cancellationToken); - - using var c = new SqliteConnection(Configuration.ConnectionString); - try - { - await c.OpenAsync(); - - using var cmd = new SqliteCommand(); - using var tr = c.BeginTransaction(); - cmd.Connection = c; - cmd.Transaction = tr; - - try - { - cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; - await cmd.ExecuteNonQueryAsync(cancellationToken); - - cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; - var version = await cmd.ExecuteLongScalarAsync(cancellationToken); - - cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; - var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); - - tr.Commit(); - - return new SyncVersion(version, newMinVersion); - } - catch (Exception) - { - tr.Rollback(); - throw; - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); - } - } - - public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - await InitializeStoreAsync(cancellationToken); - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); - - if (table == null) - { - throw new InvalidOperationException($"Unable to find table with name '{name}'"); - } - - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - - if (table.SyncDirection == SyncDirection.UploadAndDownload || - (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || - (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) - { - await SetupTableForFullChangeDetection(table, cmd, cancellationToken); - } - else - { - await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); - } - } - - public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); - } - - var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name) ?? throw new InvalidOperationException($"Unable to find table with name '{name}'"); - using var connection = new SqliteConnection(Configuration.ConnectionString); - await connection.OpenAsync(cancellationToken); - - using var cmd = connection.CreateCommand(); - await DisableChangeTrackingForTable(cmd, name, cancellationToken); - } - - private async Task DisableChangeTrackingForTable(SqliteCommand cmd, string tableName, CancellationToken cancellationToken) - { - var commandTextBase = new Func((op) => $@"DROP TRIGGER IF EXISTS [__{tableName}_ct-{op}__]"); - cmd.CommandText = commandTextBase("INSERT"); - await cmd.ExecuteNonQueryAsync(); - - cmd.CommandText = commandTextBase("UPDATE"); - await cmd.ExecuteNonQueryAsync(); - - cmd.CommandText = commandTextBase("DELETE"); - await cmd.ExecuteNonQueryAsync(); - } - - } +using JetBrains.Annotations; +using Microsoft.Data.Sqlite; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +namespace CoreSync.Sqlite +{ + public class SqliteSyncProvider : ISyncProvider + { + private bool _initialized = false; + private Guid _storeId; + private readonly ISyncLogger? _logger; + + public SqliteSyncProvider(SqliteSyncConfiguration configuration, ProviderMode providerMode = ProviderMode.Bidirectional, ISyncLogger? logger = null) + { + Configuration = configuration; + ProviderMode = providerMode; + _logger = logger; + + if (configuration.Tables.Any(_ => _.SyncDirection != SyncDirection.UploadAndDownload) && + providerMode == ProviderMode.Bidirectional) + { + throw new InvalidOperationException("One or more table with sync direction different from Bidirectional: please must specify the provider mode to Local or Remote"); + } + } + + public SqliteSyncConfiguration Configuration { get; } + public ProviderMode ProviderMode { get; } + public string[]? SyncTableNames => Configuration.Tables.Select(_ => _.Name).ToArray(); + + public async Task ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func? onConflictFunc = null, CancellationToken cancellationToken = default) + { + Validate.NotNull(changeSet, nameof(changeSet)); + Validate.NotNullAnchor(changeSet.SourceAnchor, nameof(changeSet), nameof(changeSet.SourceAnchor)); + Validate.NotNull(changeSet.TargetAnchor, nameof(changeSet), nameof(changeSet.TargetAnchor)); + + await InitializeStoreAsync(cancellationToken); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin ApplyChanges(source={changeSet.SourceAnchor}, target={changeSet.TargetAnchor}, {changeSet.Items.Count} items)"); + + using var c = new SqliteConnection(Configuration.ConnectionString); // +";Foreign Keys=False")) + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + // Reusable prepared commands, cached by (change type + table + column shape). The + // per-item write loop then only binds parameter values instead of rebuilding the SQL + // text and re-allocating parameters for each of (potentially) 100K+ items; SQLite keeps + // the compiled statement as long as the command text is unchanged. + var preparedCommands = new Dictionary(); + var existsCommands = new Dictionary(); + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + var remainingItems = changeSet.Items.ToList(); + + PreparedItemCommand GetItemCommand(SqliteSyncTable itemTable, ChangeType changeType, List> validColumns) + { + var columns = new string[validColumns.Count]; + for (int i = 0; i < validColumns.Count; i++) + { + columns[i] = validColumns[i].Key; + } + + var cacheKey = $"{(int)changeType}|{itemTable.Name}|{string.Join(",", columns)}"; + if (preparedCommands.TryGetValue(cacheKey, out var cached)) + { + return cached; + } + + var command = new SqliteCommand { Connection = c, Transaction = tr }; + var valueParameters = new SqliteParameter[columns.Length]; + + void AddValueParameters() + { + for (int i = 0; i < columns.Length; i++) + { + var valueParameter = new SqliteParameter($"@p{i}", DBNull.Value); + command.Parameters.Add(valueParameter); + valueParameters[i] = valueParameter; + } + } + + SqliteParameter? primaryKeyParameter = null; + SqliteParameter? lastSyncVersionParameter = null; + SqliteParameter? forceWriteParameter = null; + + switch (changeType) + { + case ChangeType.Insert: + command.CommandText = itemTable.BuildInsertCommandText(columns); + AddValueParameters(); + break; + + case ChangeType.Update: + command.CommandText = itemTable.BuildUpdateCommandText(columns); + primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); + command.Parameters.Add(primaryKeyParameter); + AddValueParameters(); + lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); + command.Parameters.Add(lastSyncVersionParameter); + forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); + command.Parameters.Add(forceWriteParameter); + break; + + case ChangeType.Delete: + command.CommandText = itemTable.BuildDeleteCommandText(); + valueParameters = Array.Empty(); + primaryKeyParameter = new SqliteParameter("@PrimaryColumnParameter", DBNull.Value); + command.Parameters.Add(primaryKeyParameter); + lastSyncVersionParameter = new SqliteParameter("@last_sync_version", DBNull.Value); + command.Parameters.Add(lastSyncVersionParameter); + forceWriteParameter = new SqliteParameter("@sync_force_write", DBNull.Value); + command.Parameters.Add(forceWriteParameter); + break; + } + + var preparedCommand = new PreparedItemCommand(command, valueParameters, primaryKeyParameter, lastSyncVersionParameter, forceWriteParameter); + preparedCommands[cacheKey] = preparedCommand; + return preparedCommand; + } + + SqliteCommand GetExistsCommand(SqliteSyncTable itemTable) + { + if (!existsCommands.TryGetValue(itemTable.Name, out var existsCommand)) + { + existsCommand = new SqliteCommand { Connection = c, Transaction = tr }; + existsCommand.CommandText = itemTable.SelectExistingQuery; + existsCommand.Parameters.Add(new SqliteParameter("@PrimaryColumnParameter", DBNull.Value)); + existsCommands[itemTable.Name] = existsCommand; + } + + return existsCommand; + } + int pass = 0; + while (remainingItems.Count > 0) + { + pass++; + var failedItems = new List(); + int appliedInPass = 0; + + foreach (var item in remainingItems) + { + var table = (SqliteSyncTable)Configuration.Tables.FirstOrDefault(_ => _.Name == item.TableName); + if (table == null) + { + continue; + } + + bool syncForceWrite = false; + var itemChangeType = item.ChangeType; + bool itemHandled = false; + + retryWrite: + var validColumns = table.GetValuesForValidColumns(item.Values); + var prepared = GetItemCommand(table, itemChangeType, validColumns); + var itemCommand = prepared.Command; + + // ValueParameters is empty for deletes (which bind only the primary key); + // for inserts/updates it is parallel to validColumns. + for (int i = 0; i < prepared.ValueParameters.Length; i++) + { + prepared.ValueParameters[i].Value = validColumns[i].Value.Value ?? DBNull.Value; + } + + if (prepared.PrimaryKeyParameter != null) + { + prepared.PrimaryKeyParameter.Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; + } + + if (prepared.LastSyncVersionParameter != null) + { + prepared.LastSyncVersionParameter.Value = changeSet.TargetAnchor.Version; + } + + if (prepared.ForceWriteParameter != null) + { + prepared.ForceWriteParameter.Value = syncForceWrite; + } + + int affectedRows = 0; + + try + { + affectedRows = await itemCommand.ExecuteNonQueryAsync(cancellationToken); + + if (affectedRows > 0) + { + _logger?.Trace($"[{_storeId}] Successfully applied {item}"); + } + else + { + _logger?.Trace($"[{_storeId}] No row affected on insert for {item} (maybe it's a conflict? or a different table schema between client and server):{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + //throw new SynchronizationException($"Unable to {item} item to store for table {table}", ex); + _logger?.Error($"Unable to {itemChangeType} item {item} to store for table {table}.{Environment.NewLine}{ex}{Environment.NewLine}Generated SQL:{Environment.NewLine}{itemCommand.CommandText}"); + } + + if (affectedRows == 0) + { + if (itemChangeType == ChangeType.Insert) + { + //If we can't apply an insert means that we already + //applied the insert or another record with same values (see primary key) + //is already present in table. + //In any case we can't proceed + var existsCommand = GetExistsCommand(table); + existsCommand.Parameters[0].Value = item.Values[table.PrimaryColumnName].Value ?? DBNull.Value; + if (1 == (long)await existsCommand.ExecuteScalarAsync(cancellationToken) && !syncForceWrite) + { + itemChangeType = ChangeType.Update; + goto retryWrite; + } + else + { + _logger?.Warning($"Unable to {item}: much probably there is a foreign key constraint issue logged before (pass {pass})"); + } + } + else if (itemChangeType == ChangeType.Update || + itemChangeType == ChangeType.Delete) + { + if (syncForceWrite) + { + if (itemChangeType == ChangeType.Delete) + { + //item is already deleted in data store + //so this means that we're going to delete a already deleted record + //i.e. nothing to do + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemHandled = true; + } + else + { + //if user wants to update forcely a deleted record means that + //he asctually wants to insert it again in store + _logger?.Trace($"[{_storeId}] Insert on delete conflict occurred for {item}"); + itemChangeType = ChangeType.Insert; + goto retryWrite; + } + } + else + { + //conflict detected + var res = onConflictFunc?.Invoke(item); + if (res.HasValue && res.Value == ConflictResolution.ForceWrite) + { + _logger?.Trace($"[{_storeId}] Force write on conflict occurred for {item}"); + + syncForceWrite = true; + goto retryWrite; + } + else + { + _logger?.Warning($"[{_storeId}] Skip conflict for {item}"); + itemHandled = true; + } + } + } + } + + if (affectedRows > 0) + { + itemHandled = true; + appliedInPass++; + } + + if (!itemHandled) + { + failedItems.Add(item); + } + } + + if (failedItems.Count == 0 || appliedInPass == 0) + { + if (failedItems.Count > 0) + { + _logger?.Warning($"[{_storeId}] {failedItems.Count} item(s) could not be applied after {pass} pass(es) (possible unresolvable foreign key constraint)"); + } + break; + } + + _logger?.Info($"[{_storeId}] Pass {pass}: applied {appliedInPass} item(s), retrying {failedItems.Count} remaining item(s)"); + remainingItems = failedItems; + } + + // Stamp every change-tracking row produced while applying this change set as + // originating from the source store, so those changes are not echoed back on the + // next GetChanges (IncrementalAddOrUpdatesQuery filters on SRC != @sourceId). + // The AFTER INSERT/UPDATE/DELETE triggers create these rows with SRC = NULL and + // IDs strictly greater than the version captured before applying, so a single bulk + // UPDATE here replaces the former per-item SELECT MAX(ID) + UPDATE (which executed + // two extra statements for every applied item). + cmd.CommandText = "UPDATE [__CORE_SYNC_CT] SET [SRC] = @sourceId WHERE [ID] > @version AND [SRC] IS NULL"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@sourceId", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [REMOTE_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@id", changeSet.SourceAnchor.StoreId.ToString()); + cmd.Parameters.AddWithValue("@version", changeSet.SourceAnchor.Version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [REMOTE_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + var resAnchor = new SyncAnchor(_storeId, version); + + _logger?.Info($"[{_storeId}] Completed ApplyChanges(resAnchor={resAnchor}) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resAnchor; + } + catch (Exception ex) + { + _logger?.Error($"[{_storeId}] Unable to complete the apply changes:{Environment.NewLine}{ex}"); + tr.Rollback(); + throw; + } + finally + { + foreach (var preparedCommand in preparedCommands.Values) + { + preparedCommand.Command.Dispose(); + } + + foreach (var existsCommand in existsCommands.Values) + { + existsCommand.Dispose(); + } + } + } + + /// + /// A reusable command for applying a single change type against a table with a fixed column + /// shape. The command text is built once; each item only rebinds the parameter values. + /// + private sealed class PreparedItemCommand + { + public PreparedItemCommand( + SqliteCommand command, + SqliteParameter[] valueParameters, + SqliteParameter? primaryKeyParameter, + SqliteParameter? lastSyncVersionParameter, + SqliteParameter? forceWriteParameter) + { + Command = command; + ValueParameters = valueParameters; + PrimaryKeyParameter = primaryKeyParameter; + LastSyncVersionParameter = lastSyncVersionParameter; + ForceWriteParameter = forceWriteParameter; + } + + public SqliteCommand Command { get; } + + public SqliteParameter[] ValueParameters { get; } + + public SqliteParameter? PrimaryKeyParameter { get; } + + public SqliteParameter? LastSyncVersionParameter { get; } + + public SqliteParameter? ForceWriteParameter { get; } + } + + public async Task SaveVersionForStoreAsync(Guid otherStoreId, long version, CancellationToken cancellationToken = default) + { + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"UPDATE [__CORE_SYNC_REMOTE_ANCHOR] SET [LOCAL_VERSION] = @version WHERE [ID] = @id"; + cmd.Parameters.AddWithValue("@id", otherStoreId.ToString()); + cmd.Parameters.AddWithValue("@version", version); + + if (0 == await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + cmd.CommandText = "INSERT INTO [__CORE_SYNC_REMOTE_ANCHOR] ([ID], [LOCAL_VERSION]) VALUES (@id, @version)"; + + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + tr.Commit(); + + _logger?.Trace($"[{_storeId}] Save version {version} for store {otherStoreId}"); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + + public async Task GetChangesAsync(Guid otherStoreId, SyncFilterParameter[]? syncFilterParameters = null, SyncDirection syncDirection = SyncDirection.UploadAndDownload, string[]? tables = null, CancellationToken cancellationToken = default) + { + syncFilterParameters ??= []; + + var tablesToSync = Configuration.ResolveTableFilter(tables); + + var fromAnchor = (await GetLastLocalAnchorForStoreAsync(otherStoreId, cancellationToken)); + + //if fromVersion is 0 means that client needs actually to initialize its local datastore + + //await InitializeStoreAsync(); + + var now = DateTime.Now; + + _logger?.Info($"[{_storeId}] Begin GetChanges(from={otherStoreId}, syncDirection={syncDirection}, fromVersion={fromAnchor})"); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + var items = new List(); + + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + cmd.Parameters.Clear(); + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be + // served is the one immediately before it. + if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1) + { + // Traced before throwing: this runs ahead of the per-table loop below, so without + // it a session that fails here leaves no account of why it stopped. + var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1); + _logger?.Error($"[{_storeId}] {anchorTooOld.Message}"); + throw anchorTooOld; + } + + foreach (var table in tablesToSync.Cast().Where(_ => _.Columns.Any())) + { + if (table.SyncDirection != SyncDirection.UploadAndDownload && + table.SyncDirection != syncDirection) + continue; + + //var snapshotItems = new HashSet(); + + if (fromAnchor.IsNull() && !table.SkipInitialSnapshot) + { + if (string.IsNullOrWhiteSpace(table.InitialSnapshotQuery)) + { + throw new InvalidOperationException($"InitialSnapshotQuery not specified for table"); + } + + cmd.CommandText = table.InitialSnapshotQuery; + cmd.Parameters.Clear(); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + using var r = await cmd.ExecuteReaderAsync(cancellationToken); + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount) + .ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + items.Add(new SqliteSyncItem(table, ChangeType.Insert, values)); + //snapshotItems.Add(values[table.PrimaryColumnName]); + _logger?.Trace($"[{_storeId}] Initial snapshot {items.Last()}"); + } + } + + if (!fromAnchor.IsNull()) + { + cmd.CommandText = table.IncrementalAddOrUpdatesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); + foreach (var syncFilterParameter in syncFilterParameters) + { + cmd.Parameters.AddWithValue(syncFilterParameter.Name, syncFilterParameter.Value); + } + + using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) + { + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + //if (snapshotItems.Contains(values[table.PrimaryColumnName])) + // continue; + + items.Add(new SqliteSyncItem(table, DetectChangeType(values), + values.Where(_ => _.Key != "__OP").ToDictionary(_ => _.Key, _ => _.Value == DBNull.Value ? null : _.Value))); + _logger?.Trace($"[{_storeId}] Incremental add or update {items.Last()}"); + } + } + + cmd.CommandText = table.IncrementalDeletesQuery; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@version", fromAnchor.Version); + cmd.Parameters.AddWithValue("@sourceId", otherStoreId.ToString()); + using (var r = await cmd.ExecuteReaderAsync(cancellationToken)) + { + while (await r.ReadAsync(cancellationToken)) + { + var values = Enumerable.Range(0, r.FieldCount).ToDictionary(_ => r.GetName(_), _ => GetValueFromRecord(table, r.GetName(_), _, r)); + items.Add(new SqliteSyncItem(table, ChangeType.Delete, values)); + _logger?.Trace($"[{_storeId}] Incremental delete {items.Last()}"); + } + } + } + } + + tr.Commit(); + + var resChangeSet = new SyncChangeSet(new SyncAnchor(_storeId, version), await GetLastRemoteAnchorForStoreAsync(otherStoreId, cancellationToken), items); + + _logger?.Info($"[{_storeId}] Completed GetChanges(to={version}, {items.Count} items) in {(DateTime.Now - now).TotalMilliseconds}ms"); + + return resChangeSet; + + } + catch (Exception) + { + tr.Rollback(); + throw; + } + + } + + private async Task GetLastLocalAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [LOCAL_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(_storeId, (long)version); + } + + private async Task GetLastRemoteAnchorForStoreAsync(Guid otherStoreId, CancellationToken cancellationToken = default) + { + //await InitializeAsync(); + + using var c = new SqliteConnection(Configuration.ConnectionString); + await c.OpenAsync(cancellationToken); + + using var cmd = c.CreateCommand(); + cmd.CommandText = "SELECT [REMOTE_VERSION] FROM [__CORE_SYNC_REMOTE_ANCHOR] WHERE [ID] = @storeId"; + cmd.Parameters.AddWithValue("@storeId", otherStoreId.ToString()); + + var version = await cmd.ExecuteScalarAsync(cancellationToken); + + if (version == null || version == DBNull.Value) + return SyncAnchor.Null; + + return new SyncAnchor(otherStoreId, (long)version); + } + + public async Task GetStoreIdAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + return _storeId; + } + + private static ChangeType DetectChangeType(Dictionary values) + { + if (values.TryGetValue("__OP", out var syncChangeOperation)) + { + switch (syncChangeOperation?.ToString()) + { + case "I": + return ChangeType.Insert; + + case "U": + return ChangeType.Update; + + case "D": + return ChangeType.Delete; + + default: + throw new NotSupportedException(); + } + } + + return ChangeType.Insert; + } + + private static object? GetValueFromRecord(SqliteSyncTable table, string columnName, int columnOrdinal, SqliteDataReader r) + { + if (r.IsDBNull(columnOrdinal)) + return null; + + if (table.RecordType == null) + return r.GetValue(columnOrdinal); + + var property = table.RecordType.GetProperty(columnName); + if (property != null) + return GetValueFromRecord(r, columnOrdinal, property.PropertyType); + + property = table.RecordType.GetProperties().FirstOrDefault(_ => + { + var columnAttribute = _.GetCustomAttribute(false); + if (columnAttribute != null) + { + return columnAttribute.Name == columnName; + } + + return false; + }); + + if (property != null) + return GetValueFromRecord(r, columnOrdinal, property.PropertyType); + + //fallback to getvalue + return r.GetValue(columnOrdinal); + } + + private static object GetValueFromRecord(SqliteDataReader r, int columnOrdinal, Type propertyType) + { + if (propertyType == typeof(string)) + return r.GetString(columnOrdinal); + if (propertyType == typeof(DateTime)) + return r.GetDateTime(columnOrdinal); + if (propertyType == typeof(int)) + return r.GetInt32(columnOrdinal); + if (propertyType == typeof(bool)) + return r.GetBoolean(columnOrdinal); + if (propertyType == typeof(byte)) + return r.GetByte(columnOrdinal); + if (propertyType == typeof(char)) + return r.GetChar(columnOrdinal); + if (propertyType == typeof(short)) + return r.GetInt16(columnOrdinal); + if (propertyType == typeof(long)) + return r.GetInt64(columnOrdinal); + if (propertyType == typeof(decimal)) + return r.GetDecimal(columnOrdinal); + if (propertyType == typeof(double)) + return r.GetDouble(columnOrdinal); + if (propertyType == typeof(float)) + return r.GetFloat(columnOrdinal); + + //fallback to getvalue + return r.GetValue(columnOrdinal); + } + + private static string TypeToPrintFormat(string type) + { + if (type == "INTEGER") + return "%d"; + if (type == "TEXT") + return "%s"; + + return "%s"; + } + + private async Task InitializeStoreAsync(CancellationToken cancellationToken = default) + { + if (_initialized) + return; + + using (var connection = new SqliteConnection(Configuration.ConnectionString)) + { + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"CREATE TABLE IF NOT EXISTS __CORE_SYNC_CT +(ID INTEGER PRIMARY KEY, TBL TEXT NOT NULL COLLATE NOCASE, OP CHAR NOT NULL, PK_{SqlitePrimaryColumnType.Int} INTEGER NULL, PK_{SqlitePrimaryColumnType.Text} TEXT NULL COLLATE NOCASE, PK_{SqlitePrimaryColumnType.Blob} BLOB NULL, SRC TEXT NULL COLLATE NOCASE)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Int}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Int})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Text}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Text})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + cmd.CommandText = $"CREATE INDEX IF NOT EXISTS __CORE_SYNC_CT_PK_{SqlitePrimaryColumnType.Blob}_INDEX ON __CORE_SYNC_CT(PK_{SqlitePrimaryColumnType.Blob})"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_REMOTE_ANCHOR (ID TEXT NOT NULL PRIMARY KEY, LOCAL_VERSION LONG NULL, REMOTE_VERSION LONG NULL)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"CREATE TABLE IF NOT EXISTS __CORE_SYNC_LOCAL_ID (ID TEXT NOT NULL PRIMARY KEY)"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = $"SELECT ID FROM __CORE_SYNC_LOCAL_ID LIMIT 1"; + var localId = await cmd.ExecuteScalarAsync(cancellationToken); + if (localId == null) + { + localId = Guid.NewGuid().ToString(); + cmd.CommandText = $"INSERT INTO __CORE_SYNC_LOCAL_ID (ID) VALUES (@localId)"; + cmd.Parameters.Add(new SqliteParameter("@localId", localId)); + if (1 != await cmd.ExecuteNonQueryAsync(cancellationToken)) + { + throw new InvalidOperationException(); + } + cmd.Parameters.Clear(); + } + + _storeId = Guid.Parse((string)localId); + + foreach (SqliteSyncTable table in Configuration.Tables) + { + cmd.CommandText = $"PRAGMA table_info('{table.Name}')"; + using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) + { + /* + cid name type notnull dflt_value pk + ---------- ---------- ---------- ---------- ---------- ---------- + */ + while (await reader.ReadAsync(cancellationToken)) + { + var colName = reader.GetString(1); + var colType = reader.GetString(2); + var pk = reader.GetBoolean(5); + + if (string.CompareOrdinal(colName, "__OP") == 0) + { + throw new NotSupportedException($"Unable to synchronize table '{table.Name}': one column has a reserved name '__OP'"); + } + + table.Columns.Add(colName, new SqliteColumn(colName, colType, pk)); + } + } + + if (table.Columns.Count == 0) + { + throw new InvalidOperationException($"Unable to configure table '{table}': does it exist with at least one column?"); + } + + if (table.Columns.Count(_ => _.Value.IsPrimaryKey) == 0) + { + throw new NotSupportedException($"Unable to configure table '{table}': no primary key defined"); + } + + if (table.Columns.Count(_ => _.Value.IsPrimaryKey) > 1) + { + throw new NotSupportedException($"Unable to configure table '{table}': it has more than one column as primary key"); + } + } + } + + _initialized = true; + } + + public async Task ApplyProvisionAsync(CancellationToken cancellationToken = default) + { + //if (_initialized) + // return; + + await InitializeStoreAsync(cancellationToken); + + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + foreach (var table in Configuration.Tables.Cast().Where(_ => _.Columns.Any())) + { + //var primaryKeyColumns = table.Columns.Where(_ => _.IsPrimaryKey).ToArray(); + //var tableColumns = table.Columns.Where(_ => !_.IsPrimaryKey).ToArray(); + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + //_initialized = true; + } + + private async Task SetupTableForFullChangeDetection(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) + { + var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] + AFTER {op} ON [{table.Name}] + FOR EACH ROW + BEGIN + INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); + END"); + cmd.CommandText = commandTextBase("INSERT"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + private async Task SetupTableForUpdatesOrDeletesOnly(SqliteSyncTable table, SqliteCommand cmd, CancellationToken cancellationToken = default) + { + var commandTextBase = new Func((op) => $@"CREATE TRIGGER IF NOT EXISTS [__{table.Name}_ct-{op}__] + AFTER {op} ON [{table.Name}] + FOR EACH ROW + BEGIN + INSERT INTO [__CORE_SYNC_CT] (TBL, OP, PK_{table.PrimaryColumnType}) VALUES ('{table.Name}', '{op[0]}', {(op == "DELETE" ? "OLD" : "NEW")}.[{table.PrimaryColumnName}]); + END"); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(cancellationToken); + } + + public async Task RemoveProvisionAsync(CancellationToken cancellationToken = default) + { + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + + //1. drop ct table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_CT"; + await cmd.ExecuteNonQueryAsync(); + + //2. drop remote anchor table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_REMOTE_ANCHOR"; + await cmd.ExecuteNonQueryAsync(); + + //3. drop local anchor table + cmd.CommandText = $"DROP TABLE IF EXISTS __CORE_SYNC_LOCAL_ID"; + await cmd.ExecuteNonQueryAsync(); + + //4. drop triggers — iterate table names directly; DROP TRIGGER IF EXISTS is safe even if never provisioned + foreach (SqliteSyncTable table in Configuration.Tables) + { + await DisableChangeTrackingForTable(cmd, table.Name, cancellationToken); + } + } + + public async Task GetSyncVersionAsync(CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(cancellationToken); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + return new SyncVersion(version, minVersion); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to get current/minimum version from store", ex); + } + } + + public async Task ApplyRetentionPolicyAsync(int minVersion, CancellationToken cancellationToken = default) + { + await InitializeStoreAsync(cancellationToken); + + using var c = new SqliteConnection(Configuration.ConnectionString); + try + { + await c.OpenAsync(); + + using var cmd = new SqliteCommand(); + using var tr = c.BeginTransaction(); + cmd.Connection = c; + cmd.Transaction = tr; + + try + { + cmd.CommandText = $"DELETE FROM __CORE_SYNC_CT WHERE ID < {minVersion}"; + await cmd.ExecuteNonQueryAsync(cancellationToken); + + cmd.CommandText = "SELECT MAX(ID) FROM __CORE_SYNC_CT"; + var version = await cmd.ExecuteLongScalarAsync(cancellationToken); + + cmd.CommandText = "SELECT MIN(ID) FROM __CORE_SYNC_CT"; + var newMinVersion = await cmd.ExecuteLongScalarAsync(cancellationToken); + + tr.Commit(); + + return new SyncVersion(version, newMinVersion); + } + catch (Exception) + { + tr.Rollback(); + throw; + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to apply version {minVersion} to tracking table of the store", ex); + } + } + + public async Task EnableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + await InitializeStoreAsync(cancellationToken); + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name); + + if (table == null) + { + throw new InvalidOperationException($"Unable to find table with name '{name}'"); + } + + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + + if (table.SyncDirection == SyncDirection.UploadAndDownload || + (table.SyncDirection == SyncDirection.UploadOnly && ProviderMode == ProviderMode.Local) || + (table.SyncDirection == SyncDirection.DownloadOnly && ProviderMode == ProviderMode.Remote)) + { + await SetupTableForFullChangeDetection(table, cmd, cancellationToken); + } + else + { + await SetupTableForUpdatesOrDeletesOnly(table, cmd, cancellationToken); + } + } + + public async Task DisableChangeTrackingForTable(string name, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException($"'{nameof(name)}' cannot be null or whitespace", nameof(name)); + } + + var table = Configuration.Tables.Cast().FirstOrDefault(_ => _.Name == name) ?? throw new InvalidOperationException($"Unable to find table with name '{name}'"); + using var connection = new SqliteConnection(Configuration.ConnectionString); + await connection.OpenAsync(cancellationToken); + + using var cmd = connection.CreateCommand(); + await DisableChangeTrackingForTable(cmd, name, cancellationToken); + } + + private async Task DisableChangeTrackingForTable(SqliteCommand cmd, string tableName, CancellationToken cancellationToken) + { + var commandTextBase = new Func((op) => $@"DROP TRIGGER IF EXISTS [__{tableName}_ct-{op}__]"); + cmd.CommandText = commandTextBase("INSERT"); + await cmd.ExecuteNonQueryAsync(); + + cmd.CommandText = commandTextBase("UPDATE"); + await cmd.ExecuteNonQueryAsync(); + + cmd.CommandText = commandTextBase("DELETE"); + await cmd.ExecuteNonQueryAsync(); + } + + } } \ No newline at end of file From 76c397ac17167b4fad17467d50bdee375767f413 Mon Sep 17 00:00:00 2001 From: Adolfo Marinucci Date: Wed, 26 Aug 2026 12:27:28 +0200 Subject: [PATCH 4/4] Keep SynchronizeAsync wrapping every failure, including a stale anchor SyncAnchorTooOldException was rethrown unwrapped so callers could branch on it without digging through inner exceptions. That reads well until you count who it breaks: callers have caught SynchronizationException around SynchronizeAsync since before the typed exception existed, and an escaping subclass gives them nothing to catch - at runtime, with no compile-time warning. When only SqlServerCT could raise it the blast radius was small. Now that every provider does, it covers Sqlite, which is what most clients run locally. Wrap it like everything else and let callers branch on the inner exception: catch (SynchronizationException ex) when (ex.InnerException is SyncAnchorTooOldException tooOld) Documented on SynchronizeAsync, and pinned by a test so it cannot regress quietly. The provider-level rethrow stays: it keeps the typed exception out of SyncErrorException so it still arrives intact as the inner exception. --- src/CoreSync.Tests/AnchorTooOldTests.cs | 40 +++++++++++++++++++++++ src/CoreSync/SyncAgent.cs | 31 +++++++++++++----- src/CoreSync/SyncAnchorTooOldException.cs | 5 +++ 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/CoreSync.Tests/AnchorTooOldTests.cs b/src/CoreSync.Tests/AnchorTooOldTests.cs index 34dd170..c7f6a34 100644 --- a/src/CoreSync.Tests/AnchorTooOldTests.cs +++ b/src/CoreSync.Tests/AnchorTooOldTests.cs @@ -123,6 +123,46 @@ public async Task GetChangesAsync_SucceedsWhenAnchorIsStillWithinRetention() changeSet.ShouldNotBeNull(); } + [TestMethod] + public async Task SyncAgent_WrapsTheTypedExceptionSoExistingCatchesStillFire() + { + var localFile = CreateDatabase(nameof(SyncAgent_WrapsTheTypedExceptionSoExistingCatchesStillFire) + "_local"); + var remoteFile = CreateDatabase(nameof(SyncAgent_WrapsTheTypedExceptionSoExistingCatchesStillFire) + "_remote"); + + var local = CreateProvider(localFile); + var remote = CreateProvider(remoteFile); + + await local.ApplyProvisionAsync(); + await remote.ApplyProvisionAsync(); + + using (var db = new SQLiteConnection(remoteFile)) + { + for (var i = 0; i < 10; i++) + { + db.Insert(new Stock { Id = Guid.NewGuid(), Symbol = $"SYM{i}" }); + } + } + + SqliteConnection.ClearAllPools(); + + //the remote store believes this client is at version 1, then loses everything below 5 + await remote.SaveVersionForStoreAsync(await local.GetStoreIdAsync(), 1); + await remote.ApplyRetentionPolicyAsync(5); + + var agent = new SyncAgent(local, remote); + + // Callers have caught SynchronizationException around SynchronizeAsync since before the typed + // exception existed. Letting this one case escape unwrapped would break them at runtime with + // nothing to catch it at compile time, so every failure still leaves here wrapped. + var ex = await Should.ThrowAsync(() => agent.SynchronizeAsync()); + + ex.InnerException.ShouldBeOfType(); + + var tooOld = (SyncAnchorTooOldException)ex.InnerException!; + tooOld.RequestedVersion.ShouldBe(1); + tooOld.MinValidVersion.ShouldBe(4); + } + [TestMethod] public async Task HttpClient_RebuildsTypedExceptionFromServerSignal() { diff --git a/src/CoreSync/SyncAgent.cs b/src/CoreSync/SyncAgent.cs index 1994a2c..d3bb615 100644 --- a/src/CoreSync/SyncAgent.cs +++ b/src/CoreSync/SyncAgent.cs @@ -61,7 +61,23 @@ public SyncAgent(ISyncProviderBase localSyncProvider, ISyncProviderBase remoteSy /// Optional filter parameters applied when retrieving changes from the local provider. /// /// A task that represents the asynchronous synchronization operation. - /// Thrown when synchronization fails. + /// + /// Thrown when synchronization fails, with the underlying failure as + /// . + /// + /// An inner means one of the stores can no longer + /// resolve changes from the other's anchor, because the change history covering it has been + /// discarded by its retention policy. That is permanent: the affected store has to be + /// reinitialized from a fresh snapshot, and retrying will not help. + /// + /// catch (SynchronizationException ex) + /// when (ex.InnerException is SyncAnchorTooOldException tooOld) + /// { + /// // tooOld.TableName, tooOld.RequestedVersion, tooOld.MinValidVersion + /// } + /// + /// + /// public async Task SynchronizeAsync( Func? remoteConflictResolutionFunc = null, Func? localConflictResolutionFunc = null, @@ -88,15 +104,14 @@ public async Task SynchronizeAsync( //await LocalSyncProvider.ApplyProvisionAsync(cancellationToken: cancellationToken); //await RemoteSyncProvider.ApplyProvisionAsync(cancellationToken: cancellationToken); } - catch (SyncAnchorTooOldException) - { - // Deliberately not wrapped: this one is not retryable, the client has to be - // reinitialized from a fresh snapshot, and callers need to see that without digging - // through inner exceptions. - throw; - } catch (Exception ex) { + // Every failure leaves here as a SynchronizationException, including + // SyncAnchorTooOldException: callers have caught this one type around + // SynchronizeAsync since before the typed exception existed, and letting one case + // escape unwrapped would break them at runtime with nothing to catch it at compile + // time. The typed exception stays reachable as the inner exception, which is what + // callers branch on to tell "reinitialize this client" from a transient failure. throw new SynchronizationException("Unable to synchronize stores", ex); } } diff --git a/src/CoreSync/SyncAnchorTooOldException.cs b/src/CoreSync/SyncAnchorTooOldException.cs index 8b714e4..b105954 100644 --- a/src/CoreSync/SyncAnchorTooOldException.cs +++ b/src/CoreSync/SyncAnchorTooOldException.cs @@ -19,6 +19,11 @@ namespace CoreSync /// It derives from so that callers written against the /// previous, untyped behaviour keep working. /// + /// + /// Providers raise it directly. wraps it, like every + /// other failure, in a - so callers at that level branch + /// on rather than catching this type directly. + /// /// public class SyncAnchorTooOldException : InvalidOperationException {