Report an aged-out sync anchor as a distinguishable condition

src/CoreSync.Http.Client/Implementation/SyncProviderHttpClient.cs

@@ -32,6 +32,43 @@ public SyncProviderHttpClient(IHttpClientFactory httpClientProvider, SyncProvide
         _options = options;
     }
 
+    /// <summary>
+    /// Throws for an unsuccessful response, rebuilding <see cref="SyncAnchorTooOldException"/> from the
+    /// server's 410 Gone signal so that callers see the same typed error they would from a local provider.
+    /// </summary>
+    /// <remarks>
+    /// Used in place of <see cref="HttpResponseMessage.EnsureSuccessStatusCode"/> on every sync endpoint:
+    /// without it an aged-out anchor arrives as a bare <see cref="HttpRequestException"/> and the only
+    /// thing an application can tell the user is that synchronization failed.
+    /// </remarks>
+    private static async Task EnsureSyncSuccessAsync(HttpResponseMessage response, CancellationToken cancellationToken)
+    {
+        if (response.StatusCode == HttpStatusCode.Gone &&
+            response.Headers.TryGetValues(SyncHttpHeaders.ErrorCode, out var errorCodes) &&
+            errorCodes.Contains(SyncHttpErrorCodes.AnchorTooOld))
+        {
+            SyncAnchorTooOldError? error = null;
+            try
+            {
+                error = await response.Content.ReadFromJsonAsync<SyncAnchorTooOldError>(cancellationToken);
+            }
+            catch (JsonException)
+            {
+                // The header alone identifies the condition; a body we cannot read must not downgrade
+                // this into an ordinary transport failure.
+            }
+
+            var requestedVersion = error?.RequestedVersion ?? -1;
+            var minValidVersion = error?.MinValidVersion ?? -1;
+
+            throw error?.TableName is { } tableName
+                ? new SyncAnchorTooOldException(tableName, requestedVersion, minValidVersion)
+                : new SyncAnchorTooOldException(requestedVersion, minValidVersion);
+        }
+
+        response.EnsureSuccessStatusCode();
+    }
+
     public async Task<SyncAnchor> ApplyChangesAsync([NotNull] SyncChangeSet changeSet, [CanBeNull] Func<SyncItem, ConflictResolution>? onConflictFunc = null, CancellationToken cancellationToken = default)
     {
         var httpClient = _httpClientFactory.CreateClient(_options.HttpClientName ?? Options.DefaultName);
@@ -48,8 +85,9 @@ public async Task<SyncAnchor> ApplyChangesAsync([NotNull] SyncChangeSet changeSe
 
         SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ComputingLocalChanges));
 
-        (await httpClient.PostAsJsonAsync($"/{_options.SyncControllerRoute}/changes-bulk-begin", bulkChangeSet, cancellationToken))
-            .EnsureSuccessStatusCode();
+        await EnsureSyncSuccessAsync(
+            await httpClient.PostAsJsonAsync($"/{_options.SyncControllerRoute}/changes-bulk-begin", bulkChangeSet, cancellationToken),
+            cancellationToken);
 
         var listOfItemsToUpload = new List<SyncItem>();
 
@@ -74,8 +112,9 @@ public async Task<SyncAnchor> ApplyChangesAsync([NotNull] SyncChangeSet changeSe
 
                 beginUploadItemContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-msgpack");
 
-                (await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary", beginUploadItemContent, cancellationToken))
-                    .EnsureSuccessStatusCode();
+                await EnsureSyncSuccessAsync(
+                    await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary", beginUploadItemContent, cancellationToken),
+                    cancellationToken);
             }
             else
             {
@@ -85,16 +124,17 @@ public async Task<SyncAnchor> ApplyChangesAsync([NotNull] SyncChangeSet changeSe
                     Items = listOfItemsToUpload
                 }), Encoding.UTF8, "application/json");
 
-                (await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item", beginUploadItemContent, cancellationToken))
-                    .EnsureSuccessStatusCode();
+                await EnsureSyncSuccessAsync(
+                    await httpClient.PostAsync($"{_options.SyncControllerRoute}/changes-bulk-item", beginUploadItemContent, cancellationToken),
+                    cancellationToken);
             }
 
             SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ApplyChanges, skip / (double)changeSet.Items.Count));
         }
 
         var remoteChangeSetResponse = await httpClient.PostAsync(
             $"{_options.SyncControllerRoute}/changes-bulk-complete{(_options.UseBinaryFormat ? "-binary" : string.Empty)}/{sessionId}", null, cancellationToken);
-        remoteChangeSetResponse.EnsureSuccessStatusCode();
+        await EnsureSyncSuccessAsync(remoteChangeSetResponse, cancellationToken);
 
         SyncProgress?.Invoke(this, new SyncProgressEventArgs(SyncStage.ApplyChanges, 1.0));
 
@@ -115,7 +155,7 @@ public async Task<SyncChangeSet> GetChangesAsync(Guid otherStoreId, SyncFilterPa
         }
 
         var response = await httpClient.SendAsync(request, cancellationToken);
-        response.EnsureSuccessStatusCode();
+        await EnsureSyncSuccessAsync(response, cancellationToken);
 
         var bulkSyncChangeSet = await response.Content.ReadFromJsonAsync<BulkSyncChangeSet>(cancellationToken)
             ?? throw new InvalidOperationException();
@@ -181,14 +221,20 @@ private async Task<SyncChangeSet> DownloadBulkChangeSetBinary(BulkSyncChangeSet
                 {
                     var response = await httpClient.GetAsync($"{_options.SyncControllerRoute}/changes-bulk-item-binary/{bulkSyncChangeSet.SessionId}/{skip}/{_options.BulkItemSize}", cancellationToken);
 
-                    response.EnsureSuccessStatusCode();
+                    await EnsureSyncSuccessAsync(response, cancellationToken);
 
                     using var s = await response.Content.ReadAsStreamAsync(cancellationToken);
                     var downloadedItems = await CoreSyncMessagePackSerializer.DeserializeAsync<object>(s, cancellationToken);
                     var bulkItems = (List<SyncItem>?)(downloadedItems) ?? throw new InvalidOperationException();
                     items.AddRange(bulkItems);
                     break;
                 }
+                catch (SyncAnchorTooOldException)
+                {
+                    // Permanent for this client: the change history it asked for is gone. Retrying
+                    // burns three round trips and fifteen seconds to reach the same answer.
+                    throw;
+                }
                 catch (Exception)
                 {
                     if (retry == 0)

src/CoreSync.Http.Server/WebApplicationExtensions.cs

@@ -50,6 +50,9 @@ public static void MapCoreSyncHttpServerEndpoints(this IEndpointRouteBuilder web
 
         void ConfigureEndpoint<T>(T endpoint, Action<T>? specificAction = null) where T : IEndpointConventionBuilder
         {
+            // Added before the caller's own conventions so that an aged-out anchor reaches the client
+            // as a documented 410 rather than whatever the host makes of an unhandled exception.
+            endpoint.AddSyncErrorEndpointFilter();
             options.AllEndpoints?.Invoke(endpoint);
             specificAction?.Invoke(endpoint);
         }
@@ -131,6 +134,45 @@ void ConfigureEndpoint<T>(T endpoint, Action<T>? specificAction = null) where T
 }
 
 
+internal static class SyncErrorFilterEndpointExtensions
+{
+    /// <summary>
+    /// Translates <see cref="SyncAnchorTooOldException"/> into an HTTP 410 Gone carrying the
+    /// <see cref="SyncHttpErrorCodes.AnchorTooOld"/> header and a <see cref="SyncAnchorTooOldError"/> body.
+    /// </summary>
+    /// <remarks>
+    /// 410 is the honest code here: the change history the client asked for existed once and is
+    /// permanently gone, so unlike a 5xx there is nothing to retry. Without this the condition
+    /// surfaced as an unhandled exception, which the client could only report as a generic
+    /// "unable to synchronize".
+    /// </remarks>
+    public static TBuilder AddSyncErrorEndpointFilter<TBuilder>(this TBuilder builder) where TBuilder : IEndpointConventionBuilder
+    {
+        builder.AddEndpointFilter(async (context, next) =>
+        {
+            try
+            {
+                return await next(context);
+            }
+            catch (SyncAnchorTooOldException ex)
+            {
+                context.HttpContext.Response.Headers[SyncHttpHeaders.ErrorCode] = SyncHttpErrorCodes.AnchorTooOld;
+
+                return Results.Json(
+                    new SyncAnchorTooOldError
+                    {
+                        TableName = ex.TableName,
+                        RequestedVersion = ex.RequestedVersion,
+                        MinValidVersion = ex.MinValidVersion
+                    },
+                    statusCode: StatusCodes.Status410Gone);
+            }
+        });
+
+        return builder;
+    }
+}
+
 internal static class EndpointMessagePackFilterEndpointExtensions
 {
     public static TBuilder AddMessagePackEndpointFilter<TBuilder>(this TBuilder builder) where TBuilder : IEndpointConventionBuilder

src/CoreSync.Http/SyncAnchorTooOldError.cs

@@ -0,0 +1,30 @@
+namespace CoreSync.Http;
+
+/// <summary>
+/// Body of the HTTP 410 Gone response returned when a client's sync anchor has fallen outside the
+/// store's change retention window.
+/// </summary>
+/// <remarks>
+/// The wire form of <c>CoreSync.SyncAnchorTooOldException</c>: the client rebuilds that exception
+/// from these values, so the condition reaches application code typed rather than as a generic
+/// transport failure.
+/// </remarks>
+public class SyncAnchorTooOldError
+{
+    /// <summary>
+    /// Gets or sets the table whose change history no longer covers <see cref="RequestedVersion"/>,
+    /// or <c>null</c> when the store keeps a single shared change journal.
+    /// </summary>
+    public string? TableName { get; set; }
+
+    /// <summary>
+    /// Gets or sets the version the client asked for. A negative value means the server holds no
+    /// anchor at all for the client.
+    /// </summary>
+    public long RequestedVersion { get; set; }
+
+    /// <summary>
+    /// Gets or sets the oldest version the server can still resolve changes from.
+    /// </summary>
+    public long MinValidVersion { get; set; }
+}

src/CoreSync.Http/SyncHttpErrorCodes.cs

@@ -0,0 +1,23 @@
+namespace CoreSync.Http;
+
+/// <summary>
+/// Machine-readable values carried by the <see cref="SyncHttpHeaders.ErrorCode"/> response header.
+/// </summary>
+/// <remarks>
+/// The header lets a client branch on a failure without parsing the response body or matching on
+/// message text, and stays readable even when the body is a message pack payload the client did not
+/// expect.
+/// </remarks>
+public static class SyncHttpErrorCodes
+{
+    /// <summary>
+    /// The client asked for changes relative to a version the server can no longer resolve, because
+    /// the change history covering it has been discarded by the store's retention policy.
+    /// Accompanied by HTTP 410 Gone and a <see cref="SyncAnchorTooOldError"/> body.
+    /// </summary>
+    /// <remarks>
+    /// This is permanent for the client involved: retrying with the same anchor can never succeed.
+    /// The client has to be reinitialized from a fresh snapshot.
+    /// </remarks>
+    public const string AnchorTooOld = "anchor-too-old";
+}

src/CoreSync.Http/SyncHttpHeaders.cs

@@ -16,4 +16,11 @@ public static class SyncHttpHeaders
     /// The server uses this to detect truncated headers.
     /// </summary>
     public const string TablesCount = "X-CoreSync-Tables-Count";
+
+    /// <summary>
+    /// A machine-readable identifier for a failure the client is expected to handle specifically,
+    /// set on the response alongside the corresponding status code.
+    /// See <see cref="SyncHttpErrorCodes"/> for the defined values.
+    /// </summary>
+    public const string ErrorCode = "X-CoreSync-Error";
 }

src/CoreSync.MySql/MySqlSyncProvider.cs

@@ -1,4 +1,4 @@
-using JetBrains.Annotations;
+using JetBrains.Annotations;
 using MySqlConnector;
 using System;
 using System.Collections.Generic;
@@ -299,9 +299,15 @@ public async Task<SyncChangeSet> GetChangesAsync(Guid otherStoreId, SyncFilterPa
                 cmd.Parameters.Clear();
                 var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken);
 
+                // MIN(`id`) is the oldest change still journalled, so the oldest anchor that can still be
+                // served is the one immediately before it.
                 if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1)
                 {
-                    throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})");
+                    // Traced before throwing: this runs ahead of the per-table loop below, so without
+                    // it a session that fails here leaves no account of why it stopped.
+                    var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1);
+                    _logger?.Error($"[{_storeId}] {anchorTooOld.Message}");
+                    throw anchorTooOld;
                 }
 
                 foreach (var table in tablesToSync.Cast<MySqlSyncTable>().Where(_ => _.Columns.Any()))

src/CoreSync.PostgreSQL/PostgreSQLSyncProvider.cs

@@ -1,4 +1,4 @@
-using JetBrains.Annotations;
+using JetBrains.Annotations;
 using Npgsql;
 using System;
 using System.Collections.Generic;
@@ -315,8 +315,16 @@ public async Task<SyncChangeSet> GetChangesAsync(Guid otherStoreId, SyncFilterPa
                 cmd.Parameters.Clear();
                 var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken);
 
+                // MIN(id) is the oldest change still journalled, so the oldest anchor that can still be
+                // served is the one immediately before it.
                 if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1)
-                    throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})");
+                {
+                    // Traced before throwing: this runs ahead of the per-table loop below, so without
+                    // it a session that fails here leaves no account of why it stopped.
+                    var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1);
+                    _logger?.Error($"[{_storeId}] {anchorTooOld.Message}");
+                    throw anchorTooOld;
+                }
 
                 foreach (var table in tablesToSync.Cast<PostgreSQLSyncTable>().Where(_ => _.Columns.Any()))
                 {

src/CoreSync.SqlServer/SqlSyncProvider.cs

@@ -730,8 +730,16 @@ public async Task<SyncChangeSet> GetChangesAsync(Guid otherStoreId, SyncFilterPa
                 cmd.Parameters.Clear();
                 var minVersion = await cmd.ExecuteLongScalarAsync(cancellationToken);
 
+                // MIN(ID) is the oldest change still journalled, so the oldest anchor that can still be
+                // served is the one immediately before it.
                 if (!fromAnchor.IsNull() && fromAnchor.Version < minVersion - 1)
-                    throw new InvalidOperationException($"Unable to get changes, version of data requested ({fromAnchor}) is too old (min valid version {minVersion})");
+                {
+                    // Traced before throwing: this runs ahead of the per-table loop below, so without
+                    // it a session that fails here leaves no account of why it stopped.
+                    var anchorTooOld = new SyncAnchorTooOldException(fromAnchor.Version, minVersion - 1);
+                    _logger?.Error($"[{_storeId}] {anchorTooOld.Message}");
+                    throw anchorTooOld;
+                }
 
                 foreach (SqlSyncTable table in tablesToSync.Cast<SqlSyncTable>())
                 {

src/CoreSync.SqlServerCT/ChangeTrackingDatabaseOptions.cs

@@ -1,23 +1,39 @@
-namespace CoreSync.SqlServerCT
+namespace CoreSync.SqlServerCT
 {
     /// <summary>
     /// The database-level change tracking settings as reported by <c>sys.change_tracking_databases</c>.
     /// </summary>
-    internal sealed class ChangeTrackingDatabaseOptions
+    public sealed class ChangeTrackingDatabaseOptions
     {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ChangeTrackingDatabaseOptions"/> class.
+        /// </summary>
+        /// <param name="retentionPeriod">The retention period, expressed in <paramref name="retentionPeriodUnit"/>.</param>
+        /// <param name="retentionPeriodUnit">The unit <paramref name="retentionPeriod"/> is expressed in.</param>
+        /// <param name="autoCleanup">Whether SQL Server automatically removes expired change tracking data.</param>
         public ChangeTrackingDatabaseOptions(int retentionPeriod, ChangeRetentionUnit retentionPeriodUnit, bool autoCleanup)
         {
             RetentionPeriod = retentionPeriod;
             RetentionPeriodUnit = retentionPeriodUnit;
             AutoCleanup = autoCleanup;
         }
 
+        /// <summary>
+        /// Gets how long change history is kept, expressed in <see cref="RetentionPeriodUnit"/>.
+        /// </summary>
         public int RetentionPeriod { get; }
 
+        /// <summary>
+        /// Gets the unit <see cref="RetentionPeriod"/> is expressed in.
+        /// </summary>
         public ChangeRetentionUnit RetentionPeriodUnit { get; }
 
+        /// <summary>
+        /// Gets a value indicating whether SQL Server automatically removes expired change tracking data.
+        /// </summary>
         public bool AutoCleanup { get; }
 
+        /// <inheritdoc />
         public override string ToString() => $"CHANGE_RETENTION = {RetentionPeriod} {RetentionPeriodUnit}, AUTO_CLEANUP = {(AutoCleanup ? "ON" : "OFF")}";
     }
 }