diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a7e0d1..06ef394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- Run retention on the supervisor rather than leaving it configured but + unscheduled. Every actor call writes a durable message row, so a policy that + nothing invokes let history grow without bound until an application scheduled + its own job. `retention_interval` defaults to one hour, and zero disables it. + Retention runs on its own thread, so a slow pass cannot delay replacing a + crashed role, and a failed pass retries at monitor cadence with a doubling + backoff rather than deferring for the whole interval. - Add Ruby 4.0 to the compatibility matrix, which now covers Ruby 3.3, 3.4, and 4.0 against Rails 8.0 and 8.1. diff --git a/docs/roadmap.md b/docs/roadmap.md index 8392dea..0720e27 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -35,7 +35,9 @@ graceful caller shutdown, committed state snapshots, and an opt-in Minitest helper - Supervisor role replacement: a role whose thread dies is restarted until - shutdown is requested, and dead process records are pruned on an interval + shutdown is requested, and dead process records plus expired message and + process history are pruned on their own intervals without an application + scheduling its own job - SQLite, PostgreSQL, and MySQL integration suites - Opt-in cross-process wake-up on PostgreSQL through `WakeUpAdapters.for`, with a listening connection per waiting thread and release on supervisor shutdown @@ -82,13 +84,12 @@ ## Next milestones 1. Add result lookup by request ID and broader deadlock retry classification. -2. Add scheduled retention and stale-process maintenance. -3. Add Turbo append intents and expand reconnect coverage in a full browser. -4. Add distributed rate limits, global admission hooks, and cache-capacity +2. Add Turbo append intents and expand reconnect coverage in a full browser. +3. Add distributed rate limits, global admission hooks, and cache-capacity eviction. -5. Expand security scanning beyond the Brakeman scan, such as dependency +4. Expand security scanning beyond the Brakeman scan, such as dependency auditing and secret scanning. -6. Benchmark all workloads under documented hardware/database settings and +5. Benchmark all workloads under documented hardware/database settings and publish adapter-specific adoption measurements. Throughput, synchronous latency, query counts, and the three reactive delivery paths are measured on SQLite; adapter-specific and end-to-end browser measurements are not. diff --git a/lib/solid_objects/configuration.rb b/lib/solid_objects/configuration.rb index 670bf7a..6cfb349 100644 --- a/lib/solid_objects/configuration.rb +++ b/lib/solid_objects/configuration.rb @@ -22,6 +22,7 @@ class Configuration # @rbs @process_alive_threshold: Float # @rbs @shutdown_timeout: Float # @rbs @supervisor_monitor_interval: Float + # @rbs @retention_interval: Float # @rbs @dead_process_cleanup_interval: Float # @rbs @message_retention: Numeric # @rbs @message_retention_by_actor_type: Hash[String, Numeric] @@ -65,6 +66,7 @@ class Configuration :process_alive_threshold, :shutdown_timeout, :supervisor_monitor_interval, + :retention_interval, :dead_process_cleanup_interval, :message_retention, :message_retention_by_actor_type, @@ -108,6 +110,7 @@ def initialize @lock_retry_attempts = 10 @supervisor_monitor_interval = 1.0 @dead_process_cleanup_interval = 60.0 + @retention_interval = 3600.0 @process_heartbeat_interval = 15.0 @process_alive_threshold = 60.0 @shutdown_timeout = 15.0 @@ -144,6 +147,10 @@ def validate! raise ArgumentError, "table_name_prefix must contain lowercase letters, digits, and underscores" end + if retention_interval.negative? + raise ArgumentError, "retention_interval must not be negative" + end + unless supervisor_monitor_interval.positive? raise ArgumentError, "supervisor_monitor_interval must be positive" end diff --git a/lib/solid_objects/supervisor.rb b/lib/solid_objects/supervisor.rb index 19ba16d..9203aef 100644 --- a/lib/solid_objects/supervisor.rb +++ b/lib/solid_objects/supervisor.rb @@ -2,11 +2,14 @@ module SolidObjects class Supervisor + MAXIMUM_RETENTION_BACKOFF_DOUBLINGS = 16 + # @rbs @components: Array[Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor] # @rbs @threads: Array[Thread] # @rbs @monitor: Thread? # @rbs @started: bool # @rbs @cleaned_up_at: Float + # @rbs @retention: Thread? # @rbs @lifecycle: Thread::Mutex # @rbs (?worker_count: Integer, ?effect_worker_count: Integer, ?broadcast_worker_count: Integer, ?reminder_scheduler_count: Integer) -> void @@ -26,6 +29,7 @@ def initialize( @monitor = nil @started = false @cleaned_up_at = nil + @retention = nil @lifecycle = Thread::Mutex.new end @@ -44,6 +48,7 @@ def start @started = true @threads = components.map { |component| supervise(component) } @monitor = Thread.new { monitor_loop } + @retention = Thread.new { retention_loop } SolidObjects.instrument(:"supervisor.started", component_count: components.length) end @@ -57,6 +62,7 @@ def stop # list, or never starts. @lifecycle.synchronize { @started = false } stop_monitor + stop_retention components.each(&:request_shutdown) join_until_timeout components.reject(&:stopped?).each(&:stop) @@ -133,6 +139,76 @@ def thread_error(thread) error.class.name end + # Retention gets its own thread rather than sharing the monitor's. A large + # backlog or a lock wait can make a pass slow, and role replacement must not + # wait behind housekeeping. + # @rbs () -> void + def retention_loop + failures = 0 + while @started + begin + prune_expired_records + failures = 0 + rescue => error + failures += 1 + SolidObjects.instrument( + :"supervisor.retention_failed", + error_class: error.class.name, + error_message: error.message + ) + end + wait_for_next_retention(failures) + end + end + + # Sleeping the whole interval would make shutdown wait out an hour-long + # nap, so the pause is taken in short steps that notice a stop request. + # @rbs (Integer) -> void + def wait_for_next_retention(failures) + deadline = monotonic_now + retention_pause(failures) + step = SolidObjects.configuration.supervisor_monitor_interval + while @started && monotonic_now < deadline + sleep [ step, deadline - monotonic_now ].min + end + end + + # Every actor call writes a durable message row, so retention that is only + # configured and never run leaves those rows to grow without bound. The + # supervisor runs it rather than requiring every application to schedule + # its own job. + # @rbs () -> void + def prune_expired_records + return unless SolidObjects.configuration.retention_interval.positive? + + MessagePruner.new.prune + ProcessPruner.new.prune + end + + # A transient lock or connection error must not defer retention for the + # whole interval, so a failed pass retries at monitor cadence. The pause + # then doubles per consecutive failure, capped by the interval, so a + # database that stays down is not polled once a second forever. + # @rbs (Integer) -> Float + def retention_pause(failures) + interval = SolidObjects.configuration.retention_interval + interval = SolidObjects.configuration.supervisor_monitor_interval unless interval.positive? + return interval if failures.zero? + + backoff = SolidObjects.configuration.supervisor_monitor_interval * + (2**[ failures - 1, MAXIMUM_RETENTION_BACKOFF_DOUBLINGS ].min) + [ backoff, interval ].min + end + + # @rbs () -> void + def stop_retention + retention = @retention + @retention = nil + return unless retention + + retention.join(SolidObjects.configuration.shutdown_timeout) + retention.kill if retention.alive? + end + # @rbs () -> void def cleanup_dead_processes interval = SolidObjects.configuration.dead_process_cleanup_interval diff --git a/sig/generated/lib/solid_objects/configuration.rbs b/sig/generated/lib/solid_objects/configuration.rbs index 0dbf707..0f5a91a 100644 --- a/sig/generated/lib/solid_objects/configuration.rbs +++ b/sig/generated/lib/solid_objects/configuration.rbs @@ -2,10 +2,12 @@ module SolidObjects class Configuration - @shutdown_timeout: Float + @table_name_prefix: String @supervisor_monitor_interval: Float + @retention_interval: Float + @dead_process_cleanup_interval: Float @message_retention: Numeric @@ -50,8 +52,6 @@ module SolidObjects @authorize_administration: Proc - @table_name_prefix: String - @polling_interval: Float @sync_polling_interval: Float @@ -86,6 +86,8 @@ module SolidObjects @process_alive_threshold: Float + @shutdown_timeout: Float + attr_accessor table_name_prefix: untyped attr_accessor polling_interval: untyped @@ -126,6 +128,8 @@ module SolidObjects attr_accessor supervisor_monitor_interval: untyped + attr_accessor retention_interval: untyped + attr_accessor dead_process_cleanup_interval: untyped attr_accessor message_retention: untyped diff --git a/sig/generated/lib/solid_objects/supervisor.rbs b/sig/generated/lib/solid_objects/supervisor.rbs index 12d7ac4..b25c460 100644 --- a/sig/generated/lib/solid_objects/supervisor.rbs +++ b/sig/generated/lib/solid_objects/supervisor.rbs @@ -2,8 +2,12 @@ module SolidObjects class Supervisor + MAXIMUM_RETENTION_BACKOFF_DOUBLINGS: ::Integer + @lifecycle: Thread::Mutex + @retention: Thread? + @cleaned_up_at: Float @started: bool @@ -52,6 +56,34 @@ module SolidObjects # @rbs (Thread?) -> String? def thread_error: (Thread?) -> String? + # Retention gets its own thread rather than sharing the monitor's. A large + # backlog or a lock wait can make a pass slow, and role replacement must not + # wait behind housekeeping. + # @rbs () -> void + def retention_loop: () -> void + + # Sleeping the whole interval would make shutdown wait out an hour-long + # nap, so the pause is taken in short steps that notice a stop request. + # @rbs (Integer) -> void + def wait_for_next_retention: (Integer) -> void + + # Every actor call writes a durable message row, so retention that is only + # configured and never run leaves those rows to grow without bound. The + # supervisor runs it rather than requiring every application to schedule + # its own job. + # @rbs () -> void + def prune_expired_records: () -> void + + # A transient lock or connection error must not defer retention for the + # whole interval, so a failed pass retries at monitor cadence. The pause + # then doubles per consecutive failure, capped by the interval, so a + # database that stays down is not polled once a second forever. + # @rbs (Integer) -> Float + def retention_pause: (Integer) -> Float + + # @rbs () -> void + def stop_retention: () -> void + # @rbs () -> void def cleanup_dead_processes: () -> void diff --git a/test/integration/scheduled_retention_test.rb b/test/integration/scheduled_retention_test.rb new file mode 100644 index 0000000..ed86e95 --- /dev/null +++ b/test/integration/scheduled_retention_test.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: true + +require "database_test_helper" +require "timeout" + +class ScheduledRetentionTest < ActiveSupport::TestCase + class IdleRole + # An endless method definition cannot take a modifier keyword: the modifier + # would apply to the definition itself and loop the class body. + def run + sleep(0.01) until @shutdown + end + + def request_shutdown = @shutdown = true + def stop = @stopped = true + def stopped? = @stopped + end + + class CrashOnceRole < IdleRole + class << self + attr_accessor :shared_runs + end + + def initialize + self.class.shared_runs ||= 0 + end + + def runs = self.class.shared_runs + + # Crashes only after the first monitor pass has begun, so replacement is + # needed while a slow retention pass is already running. + def run + self.class.shared_runs += 1 + if self.class.shared_runs == 1 + # The crash is the point of the test, so its backtrace is not news. + Thread.current.report_on_exception = false + sleep 0.1 + raise "role crashed" + end + + super + end + end + + setup do + CrashOnceRole.shared_runs = nil + SolidObjects.configuration.supervisor_monitor_interval = 0.02 + SolidObjects.configuration.dead_process_cleanup_interval = 0 + SolidObjects.configuration.retention_interval = 0.05 + SolidObjects.configuration.message_retention = 0 + # A test that stalls a retention pass on purpose should not pay the + # production join before the thread is killed. + SolidObjects.configuration.shutdown_timeout = 0.2 + end + + teardown do + @supervisor&.stop + SolidObjects.configuration.retention_interval = 3600.0 + SolidObjects.configuration.dead_process_cleanup_interval = 60.0 + SolidObjects.configuration.supervisor_monitor_interval = 1.0 + SolidObjects.configuration.shutdown_timeout = 15.0 + end + + test "the supervisor prunes expired messages without being asked" do + expired_message + + @supervisor = supervisor + @supervisor.start + + Timeout.timeout(10) { sleep 0.02 until SolidObjects::Message.count.zero? } + assert_equal 0, SolidObjects::Message.count + end + + test "retention can be disabled" do + SolidObjects.configuration.retention_interval = 0 + expired_message + + @supervisor = supervisor + @supervisor.start + sleep 0.3 + + assert_equal 1, SolidObjects::Message.count, + "a zero interval should disable scheduled retention" + end + + # Retention runs once at startup and then on its interval, matching how dead + # process records are cleaned up. + test "retention waits for its interval after the first pass" do + SolidObjects.configuration.retention_interval = 60 + expired_message + + @supervisor = supervisor + @supervisor.start + Timeout.timeout(10) { sleep 0.02 until SolidObjects::Message.count.zero? } + expired_message + sleep 0.2 + + assert_equal 1, SolidObjects::Message.count, + "a second pass should wait for the interval rather than run every monitor tick" + end + + test "instruments what it pruned" do + counts = [] + subscription = ActiveSupport::Notifications.subscribe("solid_objects.messages.pruned") do |event| + counts << event.payload[:count] + end + expired_message + + @supervisor = supervisor + @supervisor.start + Timeout.timeout(10) { sleep 0.02 until counts.any? { |count| count.positive? } } + + assert_includes counts, 1 + ensure + ActiveSupport::Notifications.unsubscribe(subscription) if subscription + end + + # The interval here is far longer than the assertion window, so a retry that + # waited it out would fail. A transient lock or connection error must not + # defer retention for the whole hour. + test "a failing retention pass retries rather than deferring for the interval" do + SolidObjects.configuration.retention_interval = 600 + failures = [] + subscription = ActiveSupport::Notifications.subscribe("solid_objects.supervisor.retention_failed") do + failures << true + end + role = IdleRole.new + @supervisor = supervisor(role) + @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" } + + @supervisor.start + Timeout.timeout(5) { sleep 0.02 until failures.length >= 3 } + + assert @supervisor.instance_variable_get(:@monitor).alive?, + "the monitor should survive a failing retention pass" + assert_operator failures.length, :>=, 3 + ensure + ActiveSupport::Notifications.unsubscribe(subscription) if subscription + end + + # Retrying at monitor cadence forever would hammer a database that stays + # down, so the pause grows and is capped by the configured interval. + test "repeated retention failures back off" do + SolidObjects.configuration.retention_interval = 600 + SolidObjects.configuration.supervisor_monitor_interval = 0.05 + @supervisor = supervisor + @supervisor.define_singleton_method(:prune_expired_records) { raise "boom" } + + pauses = (1..5).map { |failures| @supervisor.send(:retention_pause, failures) } + + assert_equal pauses.sort, pauses, "each failure should wait at least as long" + assert_operator pauses.last, :>, pauses.first + assert_operator pauses.max, :<=, 600 + end + + test "a recovered retention pass returns to its interval" do + SolidObjects.configuration.retention_interval = 600 + @supervisor = supervisor + + assert_equal 600, @supervisor.send(:retention_pause, 0) + end + + # Role replacement must not wait behind housekeeping. + test "a slow retention pass does not block role replacement" do + SolidObjects.configuration.retention_interval = 0.02 + role = CrashOnceRole.new + @supervisor = supervisor(role) + @supervisor.define_singleton_method(:prune_expired_records) { sleep 10 } + + @supervisor.start + + # Shorter than the pruning pass, so sharing a thread with it fails here. + Timeout.timeout(3) { sleep 0.02 until role.runs >= 2 } + assert_operator role.runs, :>=, 2, + "a crashed role should be replaced while retention is still running" + end + + test "rejects a negative retention interval" do + SolidObjects.configuration.retention_interval = -1 + + assert_raises ArgumentError do + SolidObjects.configuration.validate! + end + end + + private + + def supervisor(role = IdleRole.new) + SolidObjects::Supervisor.new( + worker_count: 0, + effect_worker_count: 0, + broadcast_worker_count: 0, + reminder_scheduler_count: 0 + ).tap { |supervisor| supervisor.instance_variable_set(:@components, [ role ]) } + end + + # A message old enough that the configured retention has already expired it. + def expired_message(actor_id = SecureRandom.hex(4)) + instance = SolidObjects::Instance.create!( + actor_type: "retention-actor", + actor_id:, + state: {}, + state_version: 1, + next_message_sequence: 2 + ) + SolidObjects::Message.create!( + instance:, + actor_type: "retention-actor", + actor_id:, + message_name: "noop", + message_kind: "async", + arguments: {}, + sequence: 1, + max_attempts: 1, + attempt_count: 1, + request_id: SecureRandom.uuid, + enqueued_at: 1.hour.ago, + completed_at: 1.hour.ago, + created_at: 1.hour.ago, + updated_at: 1.hour.ago + ) + end +end