From e67ad34ab4ae0b6b55f7147238fc10d4f80c3f18 Mon Sep 17 00:00:00 2001
From: Lucas Carlson <lucas@carlson.net>
Date: Mon, 10 Aug 2026 09:15:05 -0700
Subject: [PATCH 1/3] feat: run retention on the supervisor

Retention was configurable with sensible defaults and bounded pruners existed, but nothing invoked them: the supervisor pruned dead process records only. Every actor call writes a durable message row, including queries, so history grew without bound until an application scheduled its own job, which every adopter had to discover independently. The monitor now prunes expired messages and process history on retention_interval, defaulting to one hour, with zero disabling it.
---
 CHANGELOG.md                                  |   7 +
 docs/roadmap.md                               |  13 +-
 lib/solid_objects/configuration.rb            |   7 +
 lib/solid_objects/supervisor.rb               |  18 +++
 .../lib/solid_objects/configuration.rbs       |  10 +-
 .../lib/solid_objects/supervisor.rbs          |   9 ++
 test/integration/scheduled_retention_test.rb  | 150 ++++++++++++++++++
 7 files changed, 205 insertions(+), 9 deletions(-)
 create mode 100644 test/integration/scheduled_retention_test.rb

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8a7e0d1..d8811ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+- Run retention on the supervisor rather than leaving it configured but
+  unscheduled. Every actor call writes a durable message row, so a policy that
+  nothing invokes let history grow without bound until an application scheduled
+  its own job. `retention_interval` defaults to one hour, and zero disables it.
+
+## Unreleased
+
 - Add Ruby 4.0 to the compatibility matrix, which now covers Ruby 3.3, 3.4, and
   4.0 against Rails 8.0 and 8.1.
 
diff --git a/docs/roadmap.md b/docs/roadmap.md
index 8392dea..0720e27 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -35,7 +35,9 @@
   graceful caller shutdown, committed state snapshots, and an opt-in Minitest
   helper
 - Supervisor role replacement: a role whose thread dies is restarted until
-  shutdown is requested, and dead process records are pruned on an interval
+  shutdown is requested, and dead process records plus expired message and
+  process history are pruned on their own intervals without an application
+  scheduling its own job
 - SQLite, PostgreSQL, and MySQL integration suites
 - Opt-in cross-process wake-up on PostgreSQL through `WakeUpAdapters.for`, with
   a listening connection per waiting thread and release on supervisor shutdown
@@ -82,13 +84,12 @@
 ## Next milestones
 
 1. Add result lookup by request ID and broader deadlock retry classification.
-2. Add scheduled retention and stale-process maintenance.
-3. Add Turbo append intents and expand reconnect coverage in a full browser.
-4. Add distributed rate limits, global admission hooks, and cache-capacity
+2. Add Turbo append intents and expand reconnect coverage in a full browser.
+3. Add distributed rate limits, global admission hooks, and cache-capacity
    eviction.
-5. Expand security scanning beyond the Brakeman scan, such as dependency
+4. Expand security scanning beyond the Brakeman scan, such as dependency
    auditing and secret scanning.
-6. Benchmark all workloads under documented hardware/database settings and
+5. Benchmark all workloads under documented hardware/database settings and
    publish adapter-specific adoption measurements. Throughput, synchronous
    latency, query counts, and the three reactive delivery paths are measured on
    SQLite; adapter-specific and end-to-end browser measurements are not.
diff --git a/lib/solid_objects/configuration.rb b/lib/solid_objects/configuration.rb
index 670bf7a..6cfb349 100644
--- a/lib/solid_objects/configuration.rb
+++ b/lib/solid_objects/configuration.rb
@@ -22,6 +22,7 @@ class Configuration
     # @rbs @process_alive_threshold: Float
     # @rbs @shutdown_timeout: Float
     # @rbs @supervisor_monitor_interval: Float
+    # @rbs @retention_interval: Float
     # @rbs @dead_process_cleanup_interval: Float
     # @rbs @message_retention: Numeric
     # @rbs @message_retention_by_actor_type: Hash[String, Numeric]
@@ -65,6 +66,7 @@ class Configuration
       :process_alive_threshold,
       :shutdown_timeout,
       :supervisor_monitor_interval,
+      :retention_interval,
       :dead_process_cleanup_interval,
       :message_retention,
       :message_retention_by_actor_type,
@@ -108,6 +110,7 @@ def initialize
       @lock_retry_attempts = 10
       @supervisor_monitor_interval = 1.0
       @dead_process_cleanup_interval = 60.0
+      @retention_interval = 3600.0
       @process_heartbeat_interval = 15.0
       @process_alive_threshold = 60.0
       @shutdown_timeout = 15.0
@@ -144,6 +147,10 @@ def validate!
         raise ArgumentError, "table_name_prefix must contain lowercase letters, digits, and underscores"
       end
 
+      if retention_interval.negative?
+        raise ArgumentError, "retention_interval must not be negative"
+      end
+
       unless supervisor_monitor_interval.positive?
         raise ArgumentError, "supervisor_monitor_interval must be positive"
       end
diff --git a/lib/solid_objects/supervisor.rb b/lib/solid_objects/supervisor.rb
index 19ba16d..f3a6c99 100644
--- a/lib/solid_objects/supervisor.rb
+++ b/lib/solid_objects/supervisor.rb
@@ -7,6 +7,7 @@ class Supervisor
     # @rbs @monitor: Thread?
     # @rbs @started: bool
     # @rbs @cleaned_up_at: Float
+    # @rbs @pruned_at: Float
     # @rbs @lifecycle: Thread::Mutex
 
     # @rbs (?worker_count: Integer, ?effect_worker_count: Integer, ?broadcast_worker_count: Integer, ?reminder_scheduler_count: Integer) -> void
@@ -26,6 +27,7 @@ def initialize(
       @monitor = nil
       @started = false
       @cleaned_up_at = nil
+      @pruned_at = nil
       @lifecycle = Thread::Mutex.new
     end
 
@@ -84,6 +86,7 @@ def monitor_loop
         begin
           replace_dead_roles
           cleanup_dead_processes
+          prune_expired_records
         rescue => error
           SolidObjects.instrument(
             :"supervisor.monitor_failed",
@@ -133,6 +136,21 @@ def thread_error(thread)
       error.class.name
     end
 
+    # Every actor call writes a durable message row, so retention that is only
+    # configured and never run leaves those rows to grow without bound. The
+    # supervisor runs it rather than requiring every application to schedule
+    # its own job.
+    # @rbs () -> void
+    def prune_expired_records
+      interval = SolidObjects.configuration.retention_interval
+      return unless interval.positive?
+      return if @pruned_at && monotonic_now - @pruned_at < interval
+
+      @pruned_at = monotonic_now
+      MessagePruner.new.prune
+      ProcessPruner.new.prune
+    end
+
     # @rbs () -> void
     def cleanup_dead_processes
       interval = SolidObjects.configuration.dead_process_cleanup_interval
diff --git a/sig/generated/lib/solid_objects/configuration.rbs b/sig/generated/lib/solid_objects/configuration.rbs
index 0dbf707..0f5a91a 100644
--- a/sig/generated/lib/solid_objects/configuration.rbs
+++ b/sig/generated/lib/solid_objects/configuration.rbs
@@ -2,10 +2,12 @@
 
 module SolidObjects
   class Configuration
-    @shutdown_timeout: Float
+    @table_name_prefix: String
 
     @supervisor_monitor_interval: Float
 
+    @retention_interval: Float
+
     @dead_process_cleanup_interval: Float
 
     @message_retention: Numeric
@@ -50,8 +52,6 @@ module SolidObjects
 
     @authorize_administration: Proc
 
-    @table_name_prefix: String
-
     @polling_interval: Float
 
     @sync_polling_interval: Float
@@ -86,6 +86,8 @@ module SolidObjects
 
     @process_alive_threshold: Float
 
+    @shutdown_timeout: Float
+
     attr_accessor table_name_prefix: untyped
 
     attr_accessor polling_interval: untyped
@@ -126,6 +128,8 @@ module SolidObjects
 
     attr_accessor supervisor_monitor_interval: untyped
 
+    attr_accessor retention_interval: untyped
+
     attr_accessor dead_process_cleanup_interval: untyped
 
     attr_accessor message_retention: untyped
diff --git a/sig/generated/lib/solid_objects/supervisor.rbs b/sig/generated/lib/solid_objects/supervisor.rbs
index 12d7ac4..60e4004 100644
--- a/sig/generated/lib/solid_objects/supervisor.rbs
+++ b/sig/generated/lib/solid_objects/supervisor.rbs
@@ -4,6 +4,8 @@ module SolidObjects
   class Supervisor
     @lifecycle: Thread::Mutex
 
+    @pruned_at: Float
+
     @cleaned_up_at: Float
 
     @started: bool
@@ -52,6 +54,13 @@ module SolidObjects
     # @rbs (Thread?) -> String?
     def thread_error: (Thread?) -> String?
 
+    # Every actor call writes a durable message row, so retention that is only
+    # configured and never run leaves those rows to grow without bound. The
+    # supervisor runs it rather than requiring every application to schedule
+    # its own job.
+    # @rbs () -> void
+    def prune_expired_records: () -> void
+
     # @rbs () -> void
     def cleanup_dead_processes: () -> void
 
diff --git a/test/integration/scheduled_retention_test.rb b/test/integration/scheduled_retention_test.rb
new file mode 100644
index 0000000..29f909b
--- /dev/null
+++ b/test/integration/scheduled_retention_test.rb
@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "database_test_helper"
+require "timeout"
+
+class ScheduledRetentionTest < ActiveSupport::TestCase
+  class IdleRole
+    # An endless method definition cannot take a modifier keyword: the modifier
+    # would apply to the definition itself and loop the class body.
+    def run
+      sleep(0.01) until @shutdown
+    end
+
+    def request_shutdown = @shutdown = true
+    def stop = @stopped = true
+    def stopped? = @stopped
+  end
+
+  setup do
+    SolidObjects.configuration.supervisor_monitor_interval = 0.02
+    SolidObjects.configuration.dead_process_cleanup_interval = 0
+    SolidObjects.configuration.retention_interval = 0.05
+    SolidObjects.configuration.message_retention = 0
+  end
+
+  teardown do
+    @supervisor&.stop
+    SolidObjects.configuration.retention_interval = 3600.0
+    SolidObjects.configuration.dead_process_cleanup_interval = 60.0
+  end
+
+  test "the supervisor prunes expired messages without being asked" do
+    expired_message
+
+    @supervisor = supervisor
+    @supervisor.start
+
+    Timeout.timeout(10) { sleep 0.02 until SolidObjects::Message.count.zero? }
+    assert_equal 0, SolidObjects::Message.count
+  end
+
+  test "retention can be disabled" do
+    SolidObjects.configuration.retention_interval = 0
+    expired_message
+
+    @supervisor = supervisor
+    @supervisor.start
+    sleep 0.3
+
+    assert_equal 1, SolidObjects::Message.count,
+      "a zero interval should disable scheduled retention"
+  end
+
+  # Retention runs once at startup and then on its interval, matching how dead
+  # process records are cleaned up.
+  test "retention waits for its interval after the first pass" do
+    SolidObjects.configuration.retention_interval = 60
+    expired_message
+
+    @supervisor = supervisor
+    @supervisor.start
+    Timeout.timeout(10) { sleep 0.02 until SolidObjects::Message.count.zero? }
+    expired_message
+    sleep 0.2
+
+    assert_equal 1, SolidObjects::Message.count,
+      "a second pass should wait for the interval rather than run every monitor tick"
+  end
+
+  test "instruments what it pruned" do
+    counts = []
+    subscription = ActiveSupport::Notifications.subscribe("solid_objects.messages.pruned") do |event|
+      counts << event.payload[:count]
+    end
+    expired_message
+
+    @supervisor = supervisor
+    @supervisor.start
+    Timeout.timeout(10) { sleep 0.02 until counts.any? { |count| count.positive? } }
+
+    assert_includes counts, 1
+  ensure
+    ActiveSupport::Notifications.unsubscribe(subscription) if subscription
+  end
+
+  test "a failing retention pass does not stop the supervisor" do
+    failures = []
+    subscription = ActiveSupport::Notifications.subscribe("solid_objects.supervisor.monitor_failed") do
+      failures << true
+    end
+    role = IdleRole.new
+    @supervisor = supervisor(role)
+    @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" }
+
+    @supervisor.start
+    Timeout.timeout(10) { sleep 0.02 until failures.any? }
+    sleep 0.1
+
+    assert @supervisor.instance_variable_get(:@monitor).alive?,
+      "the monitor should survive a failing retention pass"
+  ensure
+    ActiveSupport::Notifications.unsubscribe(subscription) if subscription
+  end
+
+  test "rejects a negative retention interval" do
+    SolidObjects.configuration.retention_interval = -1
+
+    assert_raises ArgumentError do
+      SolidObjects.configuration.validate!
+    end
+  end
+
+  private
+
+  def supervisor(role = IdleRole.new)
+    SolidObjects::Supervisor.new(
+      worker_count: 0,
+      effect_worker_count: 0,
+      broadcast_worker_count: 0,
+      reminder_scheduler_count: 0
+    ).tap { |supervisor| supervisor.instance_variable_set(:@components, [ role ]) }
+  end
+
+  # A message old enough that the configured retention has already expired it.
+  def expired_message(actor_id = SecureRandom.hex(4))
+    instance = SolidObjects::Instance.create!(
+      actor_type: "retention-actor",
+      actor_id:,
+      state: {},
+      state_version: 1,
+      next_message_sequence: 2
+    )
+    SolidObjects::Message.create!(
+      instance:,
+      actor_type: "retention-actor",
+      actor_id:,
+      message_name: "noop",
+      message_kind: "async",
+      arguments: {},
+      sequence: 1,
+      max_attempts: 1,
+      attempt_count: 1,
+      request_id: SecureRandom.uuid,
+      enqueued_at: 1.hour.ago,
+      completed_at: 1.hour.ago,
+      created_at: 1.hour.ago,
+      updated_at: 1.hour.ago
+    )
+  end
+end

From cc2cbb935856ec7e8053cc3c03ad4e95aa3de56b Mon Sep 17 00:00:00 2001
From: Lucas Carlson <lucas@carlson.net>
Date: Mon, 10 Aug 2026 09:38:30 -0700
Subject: [PATCH 2/3] fix: run retention on its own thread

Greptile flagged two defects in the scheduled retention pass.

A failed pass suppressed retries: the timestamp was recorded before
pruning ran, so a transient lock or connection error deferred retention
for the full interval, up to an hour by default. Retention now retries on
the next tick, and the failure is instrumented as supervisor.retention_failed.

Retention also shared the monitor thread with role replacement. Pruning
is unbounded work against a table that only grows, so a slow pass held
back the supervision that keeps a crashed role alive. It now runs on a
dedicated thread. The pause between passes is taken in monitor-interval
steps so shutdown does not wait out an hour-long sleep.
---
 lib/solid_objects/supervisor.rb               | 61 ++++++++++++++++---
 .../lib/solid_objects/supervisor.rbs          | 20 +++++-
 test/integration/scheduled_retention_test.rb  | 49 +++++++++++++--
 3 files changed, 118 insertions(+), 12 deletions(-)

diff --git a/lib/solid_objects/supervisor.rb b/lib/solid_objects/supervisor.rb
index f3a6c99..0fa3e01 100644
--- a/lib/solid_objects/supervisor.rb
+++ b/lib/solid_objects/supervisor.rb
@@ -7,7 +7,7 @@ class Supervisor
     # @rbs @monitor: Thread?
     # @rbs @started: bool
     # @rbs @cleaned_up_at: Float
-    # @rbs @pruned_at: Float
+    # @rbs @retention: Thread?
     # @rbs @lifecycle: Thread::Mutex
 
     # @rbs (?worker_count: Integer, ?effect_worker_count: Integer, ?broadcast_worker_count: Integer, ?reminder_scheduler_count: Integer) -> void
@@ -27,7 +27,7 @@ def initialize(
       @monitor = nil
       @started = false
       @cleaned_up_at = nil
-      @pruned_at = nil
+      @retention = nil
       @lifecycle = Thread::Mutex.new
     end
 
@@ -46,6 +46,7 @@ def start
       @started = true
       @threads = components.map { |component| supervise(component) }
       @monitor = Thread.new { monitor_loop }
+      @retention = Thread.new { retention_loop }
       SolidObjects.instrument(:"supervisor.started", component_count: components.length)
     end
 
@@ -59,6 +60,7 @@ def stop
         # list, or never starts.
         @lifecycle.synchronize { @started = false }
         stop_monitor
+        stop_retention
         components.each(&:request_shutdown)
         join_until_timeout
         components.reject(&:stopped?).each(&:stop)
@@ -86,7 +88,6 @@ def monitor_loop
         begin
           replace_dead_roles
           cleanup_dead_processes
-          prune_expired_records
         rescue => error
           SolidObjects.instrument(
             :"supervisor.monitor_failed",
@@ -136,21 +137,67 @@ def thread_error(thread)
       error.class.name
     end
 
+    # Retention gets its own thread rather than sharing the monitor's. A large
+    # backlog or a lock wait can make a pass slow, and role replacement must not
+    # wait behind housekeeping. A failed pass retries on the next tick instead of
+    # deferring for the whole interval.
+    # @rbs () -> void
+    def retention_loop
+      while @started
+        begin
+          prune_expired_records
+        rescue => error
+          SolidObjects.instrument(
+            :"supervisor.retention_failed",
+            error_class: error.class.name,
+            error_message: error.message
+          )
+        end
+        wait_for_next_retention
+      end
+    end
+
+    # Sleeping the whole interval would make shutdown wait out an hour-long
+    # nap, so the pause is taken in short steps that notice a stop request.
+    # @rbs () -> void
+    def wait_for_next_retention
+      deadline = monotonic_now + retention_pause
+      step = SolidObjects.configuration.supervisor_monitor_interval
+      while @started && monotonic_now < deadline
+        sleep [ step, deadline - monotonic_now ].min
+      end
+    end
+
     # Every actor call writes a durable message row, so retention that is only
     # configured and never run leaves those rows to grow without bound. The
     # supervisor runs it rather than requiring every application to schedule
     # its own job.
     # @rbs () -> void
     def prune_expired_records
-      interval = SolidObjects.configuration.retention_interval
-      return unless interval.positive?
-      return if @pruned_at && monotonic_now - @pruned_at < interval
+      return unless SolidObjects.configuration.retention_interval.positive?
 
-      @pruned_at = monotonic_now
       MessagePruner.new.prune
       ProcessPruner.new.prune
     end
 
+    # @rbs () -> Float
+    def retention_pause
+      interval = SolidObjects.configuration.retention_interval
+      return interval if interval.positive?
+
+      SolidObjects.configuration.supervisor_monitor_interval
+    end
+
+    # @rbs () -> void
+    def stop_retention
+      retention = @retention
+      @retention = nil
+      return unless retention
+
+      retention.join(SolidObjects.configuration.shutdown_timeout)
+      retention.kill if retention.alive?
+    end
+
     # @rbs () -> void
     def cleanup_dead_processes
       interval = SolidObjects.configuration.dead_process_cleanup_interval
diff --git a/sig/generated/lib/solid_objects/supervisor.rbs b/sig/generated/lib/solid_objects/supervisor.rbs
index 60e4004..17b1c01 100644
--- a/sig/generated/lib/solid_objects/supervisor.rbs
+++ b/sig/generated/lib/solid_objects/supervisor.rbs
@@ -4,7 +4,7 @@ module SolidObjects
   class Supervisor
     @lifecycle: Thread::Mutex
 
-    @pruned_at: Float
+    @retention: Thread?
 
     @cleaned_up_at: Float
 
@@ -54,6 +54,18 @@ module SolidObjects
     # @rbs (Thread?) -> String?
     def thread_error: (Thread?) -> String?
 
+    # Retention gets its own thread rather than sharing the monitor's. A large
+    # backlog or a lock wait can make a pass slow, and role replacement must not
+    # wait behind housekeeping. A failed pass retries on the next tick instead of
+    # deferring for the whole interval.
+    # @rbs () -> void
+    def retention_loop: () -> void
+
+    # Sleeping the whole interval would make shutdown wait out an hour-long
+    # nap, so the pause is taken in short steps that notice a stop request.
+    # @rbs () -> void
+    def wait_for_next_retention: () -> void
+
     # Every actor call writes a durable message row, so retention that is only
     # configured and never run leaves those rows to grow without bound. The
     # supervisor runs it rather than requiring every application to schedule
@@ -61,6 +73,12 @@ module SolidObjects
     # @rbs () -> void
     def prune_expired_records: () -> void
 
+    # @rbs () -> Float
+    def retention_pause: () -> Float
+
+    # @rbs () -> void
+    def stop_retention: () -> void
+
     # @rbs () -> void
     def cleanup_dead_processes: () -> void
 
diff --git a/test/integration/scheduled_retention_test.rb b/test/integration/scheduled_retention_test.rb
index 29f909b..b081858 100644
--- a/test/integration/scheduled_retention_test.rb
+++ b/test/integration/scheduled_retention_test.rb
@@ -16,7 +16,32 @@ def stop = @stopped = true
     def stopped? = @stopped
   end
 
+  class CrashOnceRole < IdleRole
+    class << self
+      attr_accessor :shared_runs
+    end
+
+    def initialize
+      self.class.shared_runs ||= 0
+    end
+
+    def runs = self.class.shared_runs
+
+    # Crashes only after the first monitor pass has begun, so replacement is
+    # needed while a slow retention pass is already running.
+    def run
+      self.class.shared_runs += 1
+      if self.class.shared_runs == 1
+        sleep 0.1
+        raise "role crashed"
+      end
+
+      super
+    end
+  end
+
   setup do
+    CrashOnceRole.shared_runs = nil
     SolidObjects.configuration.supervisor_monitor_interval = 0.02
     SolidObjects.configuration.dead_process_cleanup_interval = 0
     SolidObjects.configuration.retention_interval = 0.05
@@ -83,9 +108,9 @@ def stopped? = @stopped
     ActiveSupport::Notifications.unsubscribe(subscription) if subscription
   end
 
-  test "a failing retention pass does not stop the supervisor" do
+  test "a failing retention pass retries rather than deferring for the interval" do
     failures = []
-    subscription = ActiveSupport::Notifications.subscribe("solid_objects.supervisor.monitor_failed") do
+    subscription = ActiveSupport::Notifications.subscribe("solid_objects.supervisor.retention_failed") do
       failures << true
     end
     role = IdleRole.new
@@ -93,15 +118,31 @@ def stopped? = @stopped
     @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" }
 
     @supervisor.start
-    Timeout.timeout(10) { sleep 0.02 until failures.any? }
-    sleep 0.1
+    Timeout.timeout(10) { sleep 0.02 until failures.length >= 2 }
 
     assert @supervisor.instance_variable_get(:@monitor).alive?,
       "the monitor should survive a failing retention pass"
+    assert_operator failures.length, :>=, 2,
+      "a failed pass should retry on the next tick, not wait out the interval"
   ensure
     ActiveSupport::Notifications.unsubscribe(subscription) if subscription
   end
 
+  # Role replacement must not wait behind housekeeping.
+  test "a slow retention pass does not block role replacement" do
+    SolidObjects.configuration.retention_interval = 0.02
+    role = CrashOnceRole.new
+    @supervisor = supervisor(role)
+    @supervisor.define_singleton_method(:prune_expired_records) { sleep 10 }
+
+    @supervisor.start
+
+    # Shorter than the pruning pass, so sharing a thread with it fails here.
+    Timeout.timeout(3) { sleep 0.02 until role.runs >= 2 }
+    assert_operator role.runs, :>=, 2,
+      "a crashed role should be replaced while retention is still running"
+  end
+
   test "rejects a negative retention interval" do
     SolidObjects.configuration.retention_interval = -1
 

From 15f0fdbdd0367157bd0da51fd2280d6f6db1cd7f Mon Sep 17 00:00:00 2001
From: Lucas Carlson <lucas@carlson.net>
Date: Mon, 10 Aug 2026 09:50:36 -0700
Subject: [PATCH 3/3] fix: back off failed retention passes

The previous fix removed the timestamp that suppressed retries but left
the failure path waiting the configured interval, so a transient lock or
connection error still deferred retention for up to an hour. Greptile
caught that the reply overstated what the change did.

A failed pass now retries at monitor cadence and doubles the pause per
consecutive failure, capped by the retention interval, so recovery is
prompt without polling a database that stays down once a second forever.
The retry test now configures a 600 second interval, which the previous
behaviour would have waited out.

Also merge the duplicate Unreleased changelog heading left by the rebase.
---
 CHANGELOG.md                                  |  6 +--
 lib/solid_objects/supervisor.rb               | 31 ++++++++++-----
 .../lib/solid_objects/supervisor.rbs          | 17 ++++++---
 test/integration/scheduled_retention_test.rb  | 38 +++++++++++++++++--
 4 files changed, 70 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d8811ff..06ef394 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,9 +6,9 @@
   unscheduled. Every actor call writes a durable message row, so a policy that
   nothing invokes let history grow without bound until an application scheduled
   its own job. `retention_interval` defaults to one hour, and zero disables it.
-
-## Unreleased
-
+  Retention runs on its own thread, so a slow pass cannot delay replacing a
+  crashed role, and a failed pass retries at monitor cadence with a doubling
+  backoff rather than deferring for the whole interval.
 - Add Ruby 4.0 to the compatibility matrix, which now covers Ruby 3.3, 3.4, and
   4.0 against Rails 8.0 and 8.1.
 
diff --git a/lib/solid_objects/supervisor.rb b/lib/solid_objects/supervisor.rb
index 0fa3e01..9203aef 100644
--- a/lib/solid_objects/supervisor.rb
+++ b/lib/solid_objects/supervisor.rb
@@ -2,6 +2,8 @@
 
 module SolidObjects
   class Supervisor
+    MAXIMUM_RETENTION_BACKOFF_DOUBLINGS = 16
+
     # @rbs @components: Array[Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor]
     # @rbs @threads: Array[Thread]
     # @rbs @monitor: Thread?
@@ -139,29 +141,31 @@ def thread_error(thread)
 
     # Retention gets its own thread rather than sharing the monitor's. A large
     # backlog or a lock wait can make a pass slow, and role replacement must not
-    # wait behind housekeeping. A failed pass retries on the next tick instead of
-    # deferring for the whole interval.
+    # wait behind housekeeping.
     # @rbs () -> void
     def retention_loop
+      failures = 0
       while @started
         begin
           prune_expired_records
+          failures = 0
         rescue => error
+          failures += 1
           SolidObjects.instrument(
             :"supervisor.retention_failed",
             error_class: error.class.name,
             error_message: error.message
           )
         end
-        wait_for_next_retention
+        wait_for_next_retention(failures)
       end
     end
 
     # Sleeping the whole interval would make shutdown wait out an hour-long
     # nap, so the pause is taken in short steps that notice a stop request.
-    # @rbs () -> void
-    def wait_for_next_retention
-      deadline = monotonic_now + retention_pause
+    # @rbs (Integer) -> void
+    def wait_for_next_retention(failures)
+      deadline = monotonic_now + retention_pause(failures)
       step = SolidObjects.configuration.supervisor_monitor_interval
       while @started && monotonic_now < deadline
         sleep [ step, deadline - monotonic_now ].min
@@ -180,12 +184,19 @@ def prune_expired_records
       ProcessPruner.new.prune
     end
 
-    # @rbs () -> Float
-    def retention_pause
+    # A transient lock or connection error must not defer retention for the
+    # whole interval, so a failed pass retries at monitor cadence. The pause
+    # then doubles per consecutive failure, capped by the interval, so a
+    # database that stays down is not polled once a second forever.
+    # @rbs (Integer) -> Float
+    def retention_pause(failures)
       interval = SolidObjects.configuration.retention_interval
-      return interval if interval.positive?
+      interval = SolidObjects.configuration.supervisor_monitor_interval unless interval.positive?
+      return interval if failures.zero?
 
-      SolidObjects.configuration.supervisor_monitor_interval
+      backoff = SolidObjects.configuration.supervisor_monitor_interval *
+        (2**[ failures - 1, MAXIMUM_RETENTION_BACKOFF_DOUBLINGS ].min)
+      [ backoff, interval ].min
     end
 
     # @rbs () -> void
diff --git a/sig/generated/lib/solid_objects/supervisor.rbs b/sig/generated/lib/solid_objects/supervisor.rbs
index 17b1c01..b25c460 100644
--- a/sig/generated/lib/solid_objects/supervisor.rbs
+++ b/sig/generated/lib/solid_objects/supervisor.rbs
@@ -2,6 +2,8 @@
 
 module SolidObjects
   class Supervisor
+    MAXIMUM_RETENTION_BACKOFF_DOUBLINGS: ::Integer
+
     @lifecycle: Thread::Mutex
 
     @retention: Thread?
@@ -56,15 +58,14 @@ module SolidObjects
 
     # Retention gets its own thread rather than sharing the monitor's. A large
     # backlog or a lock wait can make a pass slow, and role replacement must not
-    # wait behind housekeeping. A failed pass retries on the next tick instead of
-    # deferring for the whole interval.
+    # wait behind housekeeping.
     # @rbs () -> void
     def retention_loop: () -> void
 
     # Sleeping the whole interval would make shutdown wait out an hour-long
     # nap, so the pause is taken in short steps that notice a stop request.
-    # @rbs () -> void
-    def wait_for_next_retention: () -> void
+    # @rbs (Integer) -> void
+    def wait_for_next_retention: (Integer) -> void
 
     # Every actor call writes a durable message row, so retention that is only
     # configured and never run leaves those rows to grow without bound. The
@@ -73,8 +74,12 @@ module SolidObjects
     # @rbs () -> void
     def prune_expired_records: () -> void
 
-    # @rbs () -> Float
-    def retention_pause: () -> Float
+    # A transient lock or connection error must not defer retention for the
+    # whole interval, so a failed pass retries at monitor cadence. The pause
+    # then doubles per consecutive failure, capped by the interval, so a
+    # database that stays down is not polled once a second forever.
+    # @rbs (Integer) -> Float
+    def retention_pause: (Integer) -> Float
 
     # @rbs () -> void
     def stop_retention: () -> void
diff --git a/test/integration/scheduled_retention_test.rb b/test/integration/scheduled_retention_test.rb
index b081858..ed86e95 100644
--- a/test/integration/scheduled_retention_test.rb
+++ b/test/integration/scheduled_retention_test.rb
@@ -32,6 +32,8 @@ def runs = self.class.shared_runs
     def run
       self.class.shared_runs += 1
       if self.class.shared_runs == 1
+        # The crash is the point of the test, so its backtrace is not news.
+        Thread.current.report_on_exception = false
         sleep 0.1
         raise "role crashed"
       end
@@ -46,12 +48,17 @@ def run
     SolidObjects.configuration.dead_process_cleanup_interval = 0
     SolidObjects.configuration.retention_interval = 0.05
     SolidObjects.configuration.message_retention = 0
+    # A test that stalls a retention pass on purpose should not pay the
+    # production join before the thread is killed.
+    SolidObjects.configuration.shutdown_timeout = 0.2
   end
 
   teardown do
     @supervisor&.stop
     SolidObjects.configuration.retention_interval = 3600.0
     SolidObjects.configuration.dead_process_cleanup_interval = 60.0
+    SolidObjects.configuration.supervisor_monitor_interval = 1.0
+    SolidObjects.configuration.shutdown_timeout = 15.0
   end
 
   test "the supervisor prunes expired messages without being asked" do
@@ -108,7 +115,11 @@ def run
     ActiveSupport::Notifications.unsubscribe(subscription) if subscription
   end
 
+  # The interval here is far longer than the assertion window, so a retry that
+  # waited it out would fail. A transient lock or connection error must not
+  # defer retention for the whole hour.
   test "a failing retention pass retries rather than deferring for the interval" do
+    SolidObjects.configuration.retention_interval = 600
     failures = []
     subscription = ActiveSupport::Notifications.subscribe("solid_objects.supervisor.retention_failed") do
       failures << true
@@ -118,16 +129,37 @@ def run
     @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" }
 
     @supervisor.start
-    Timeout.timeout(10) { sleep 0.02 until failures.length >= 2 }
+    Timeout.timeout(5) { sleep 0.02 until failures.length >= 3 }
 
     assert @supervisor.instance_variable_get(:@monitor).alive?,
       "the monitor should survive a failing retention pass"
-    assert_operator failures.length, :>=, 2,
-      "a failed pass should retry on the next tick, not wait out the interval"
+    assert_operator failures.length, :>=, 3
   ensure
     ActiveSupport::Notifications.unsubscribe(subscription) if subscription
   end
 
+  # Retrying at monitor cadence forever would hammer a database that stays
+  # down, so the pause grows and is capped by the configured interval.
+  test "repeated retention failures back off" do
+    SolidObjects.configuration.retention_interval = 600
+    SolidObjects.configuration.supervisor_monitor_interval = 0.05
+    @supervisor = supervisor
+    @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" }
+
+    pauses = (1..5).map { |failures| @supervisor.send(:retention_pause, failures) }
+
+    assert_equal pauses.sort, pauses, "each failure should wait at least as long"
+    assert_operator pauses.last, :>, pauses.first
+    assert_operator pauses.max, :<=, 600
+  end
+
+  test "a recovered retention pass returns to its interval" do
+    SolidObjects.configuration.retention_interval = 600
+    @supervisor = supervisor
+
+    assert_equal 600, @supervisor.send(:retention_pause, 0)
+  end
+
   # Role replacement must not wait behind housekeeping.
   test "a slow retention pass does not block role replacement" do
     SolidObjects.configuration.retention_interval = 0.02