diff --git a/Algorithm.CSharp/AutomaticIndicatorDeregistrationRegressionAlgorithm.cs b/Algorithm.CSharp/AutomaticIndicatorDeregistrationRegressionAlgorithm.cs new file mode 100644 index 000000000000..e67b1df2fefa --- /dev/null +++ b/Algorithm.CSharp/AutomaticIndicatorDeregistrationRegressionAlgorithm.cs @@ -0,0 +1,145 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Collections.Generic; +using QuantConnect.Data; +using QuantConnect.Indicators; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Example and regression algorithm asserting the behavior of : + /// when enabled, helper-created indicators are automatically deregistered when their security is removed from the + /// algorithm, without any explicit cleanup call + /// + public class AutomaticIndicatorDeregistrationRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Symbol _spy; + private Symbol _ibm; + private RelativeStrengthIndex _ibmRsi; + private SimpleMovingAverage _ibmSma; + private bool _removed; + + /// + /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 07); + SetEndDate(2013, 10, 11); + + Settings.AutomaticIndicatorDeregistration = true; + + _spy = AddEquity("SPY").Symbol; + _ibm = AddEquity("IBM").Symbol; + + _ibmRsi = RSI(_ibm, 14, resolution: Resolution.Minute); + _ibmSma = SMA(_ibm, 10, Resolution.Minute); + } + + /// + /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. + /// + /// Slice object keyed by symbol containing the stock data + public override void OnData(Slice slice) + { + if (!_removed && Time.Day == 9) + { + _removed = true; + RemoveSecurity(_ibm); + } + + if (!Portfolio.Invested) + { + SetHoldings(_spy, 0.5m); + } + } + + public override void OnEndOfAlgorithm() + { + if (!_removed) + { + throw new RegressionTestException("The security should have been removed"); + } + // the indicators were deregistered by the engine when the security was completely removed, + // no explicit cleanup call needed + if (_ibmRsi.Consolidators.Count != 0 || _ibmSma.Consolidators.Count != 0) + { + throw new RegressionTestException("The removed security indicators should have been automatically deregistered"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 5506; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "1"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "93.262%"}, + {"Drawdown", "1.100%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100845.96"}, + {"Net Profit", "0.846%"}, + {"Sharpe Ratio", "6.447"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "67.235%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "-0.268"}, + {"Beta", "0.496"}, + {"Annual Standard Deviation", "0.11"}, + {"Annual Variance", "0.012"}, + {"Information Ratio", "-11.27"}, + {"Tracking Error", "0.112"}, + {"Treynor Ratio", "1.435"}, + {"Total Fees", "$1.72"}, + {"Estimated Strategy Capacity", "$87000000.00"}, + {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"}, + {"Portfolio Turnover", "9.96%"}, + {"Drawdown Recovery", "3"}, + {"OrderListHash", "d17dbd01fd291aab1eb04cf714ceba93"} + }; + } +} diff --git a/Algorithm.CSharp/DeregisterAllRegressionAlgorithm.cs b/Algorithm.CSharp/DeregisterAllRegressionAlgorithm.cs new file mode 100644 index 000000000000..604947227826 --- /dev/null +++ b/Algorithm.CSharp/DeregisterAllRegressionAlgorithm.cs @@ -0,0 +1,186 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Collections.Generic; +using QuantConnect.Data; +using QuantConnect.Data.Market; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Indicators; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Example and regression algorithm asserting the behavior of : + /// on security removal, a single call disposes all the indicators and consolidators created for it through the + /// algorithm helper methods, so add/remove churn doesn't leak consolidators, and re-adding the security with + /// fresh indicators keeps working + /// + public class DeregisterAllRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Symbol _spy; + private Symbol _ibm; + private RelativeStrengthIndex _ibmRsi; + private SimpleMovingAverage _ibmSma; + private SimpleMovingAverage _newIbmSma; + private int _ibmConsolidatedCount; + private int _ibmConsolidatedCountAtRemoval; + private long _ibmRsiSamplesAtRemoval; + private bool _removed; + private bool _readded; + + /// + /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 07); + SetEndDate(2013, 10, 11); + + _spy = AddEquity("SPY").Symbol; + _ibm = AddEquity("IBM").Symbol; + + // per symbol state created through the helper methods, tracked by the engine + _ibmRsi = RSI(_ibm, 14, resolution: Resolution.Minute); + _ibmSma = SMA(_ibm, 10, Resolution.Minute); + Consolidate(_ibm, Resolution.Hour, (TradeBar bar) => _ibmConsolidatedCount++); + } + + /// + /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. + /// + /// Slice object keyed by symbol containing the stock data + public override void OnData(Slice slice) + { + if (!_removed && Time.Day == 8) + { + _removed = true; + RemoveSecurity(_ibm); + } + else if (_removed && !_readded && Time.Day == 10) + { + _readded = true; + // re-adding the security after cleanup works: the helpers create fresh consolidators + AddEquity("IBM"); + _newIbmSma = SMA(_ibm, 10, Resolution.Minute); + } + + if (!Portfolio.Invested) + { + SetHoldings(_spy, 0.5m); + } + } + + /// + /// Event fired each time the we add/remove securities from the data feed + /// + /// Security additions/removals for this time step + public override void OnSecuritiesChanged(SecurityChanges changes) + { + foreach (var security in changes.RemovedSecurities) + { + // single call cleanup of every helper-created indicator and consolidator of the removed security + DeregisterAll(security.Symbol); + + if (security.Symbol == _ibm) + { + _ibmRsiSamplesAtRemoval = _ibmRsi.Samples; + _ibmConsolidatedCountAtRemoval = _ibmConsolidatedCount; + + if (_ibmRsi.Consolidators.Count != 0 || _ibmSma.Consolidators.Count != 0) + { + throw new RegressionTestException("The removed security indicators should have no consolidators after DeregisterAll"); + } + } + } + } + + public override void OnEndOfAlgorithm() + { + if (!_removed || _ibmRsiSamplesAtRemoval == 0) + { + throw new RegressionTestException("The security should have been removed and its indicators deregistered"); + } + if (_ibmRsi.Samples != _ibmRsiSamplesAtRemoval || _ibmConsolidatedCount != _ibmConsolidatedCountAtRemoval) + { + throw new RegressionTestException("Deregistered indicators and consolidators should have stopped getting updates"); + } + if (_newIbmSma == null || !_newIbmSma.IsReady) + { + throw new RegressionTestException("Indicators created after re-adding the security should be getting updates"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6285; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "1"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "93.262%"}, + {"Drawdown", "1.100%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100845.96"}, + {"Net Profit", "0.846%"}, + {"Sharpe Ratio", "6.447"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "67.235%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "-0.268"}, + {"Beta", "0.496"}, + {"Annual Standard Deviation", "0.11"}, + {"Annual Variance", "0.012"}, + {"Information Ratio", "-11.27"}, + {"Tracking Error", "0.112"}, + {"Treynor Ratio", "1.435"}, + {"Total Fees", "$1.72"}, + {"Estimated Strategy Capacity", "$87000000.00"}, + {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"}, + {"Portfolio Turnover", "9.96%"}, + {"Drawdown Recovery", "3"}, + {"OrderListHash", "d17dbd01fd291aab1eb04cf714ceba93"} + }; + } +} diff --git a/Algorithm.Python/DeregisterAllRegressionAlgorithm.py b/Algorithm.Python/DeregisterAllRegressionAlgorithm.py new file mode 100644 index 000000000000..26f4cff4e52e --- /dev/null +++ b/Algorithm.Python/DeregisterAllRegressionAlgorithm.py @@ -0,0 +1,76 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * + +### +### Example and regression algorithm asserting the behavior of deregister_all: on security removal, a single call +### disposes all the indicators and consolidators created for it through the algorithm helper methods, so add/remove +### churn doesn't leak consolidators, and re-adding the security with fresh indicators keeps working +### +class DeregisterAllRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2013, 10, 7) + self.set_end_date(2013, 10, 11) + + self._spy = self.add_equity("SPY").symbol + self._ibm = self.add_equity("IBM").symbol + + # per symbol state created through the helper methods, tracked by the engine + self._ibm_rsi = self.rsi(self._ibm, 14, resolution=Resolution.MINUTE) + self._ibm_sma = self.sma(self._ibm, 10, Resolution.MINUTE) + self._ibm_consolidated_count = 0 + self.consolidate(self._ibm, Resolution.HOUR, self._on_hour_bar) + + self._new_ibm_sma = None + self._ibm_rsi_samples_at_removal = 0 + self._ibm_consolidated_count_at_removal = 0 + self._removed = False + self._readded = False + + def _on_hour_bar(self, bar): + self._ibm_consolidated_count += 1 + + def on_data(self, data): + if not self._removed and self.time.day == 8: + self._removed = True + self.remove_security(self._ibm) + elif self._removed and not self._readded and self.time.day == 10: + self._readded = True + # re-adding the security after cleanup works: the helpers create fresh consolidators + self.add_equity("IBM") + self._new_ibm_sma = self.sma(self._ibm, 10, Resolution.MINUTE) + + if not self.portfolio.invested: + self.set_holdings(self._spy, 0.5) + + def on_securities_changed(self, changes): + for security in changes.removed_securities: + # single call cleanup of every helper-created indicator and consolidator of the removed security + self.deregister_all(security.symbol) + + if security.symbol == self._ibm: + self._ibm_rsi_samples_at_removal = self._ibm_rsi.samples + self._ibm_consolidated_count_at_removal = self._ibm_consolidated_count + + if self._ibm_rsi.consolidators.count != 0 or self._ibm_sma.consolidators.count != 0: + raise Exception("The removed security indicators should have no consolidators after deregister_all") + + def on_end_of_algorithm(self): + if not self._removed or self._ibm_rsi_samples_at_removal == 0: + raise Exception("The security should have been removed and its indicators deregistered") + if self._ibm_rsi.samples != self._ibm_rsi_samples_at_removal or self._ibm_consolidated_count != self._ibm_consolidated_count_at_removal: + raise Exception("Deregistered indicators and consolidators should have stopped getting updates") + if self._new_ibm_sma is None or not self._new_ibm_sma.is_ready: + raise Exception("Indicators created after re-adding the security should be getting updates") diff --git a/Algorithm/QCAlgorithm.Indicators.cs b/Algorithm/QCAlgorithm.Indicators.cs index e86ffd96b864..383d91dcbe1f 100644 --- a/Algorithm/QCAlgorithm.Indicators.cs +++ b/Algorithm/QCAlgorithm.Indicators.cs @@ -56,6 +56,13 @@ public partial class QCAlgorithm "Period" }; + // Per symbol tracking of the indicators and consolidators created through the algorithm helper methods, + // so they can be deregistered in bulk when a security is removed, see 'DeregisterAll'. Without cleanup, + // universe churn accumulates consolidators that are scanned forever, leaking until out of memory + private readonly Dictionary> _registeredIndicators = new(); + private readonly Dictionary> _registeredConsolidators = new(); + private readonly object _registrationsLock = new(); + /// /// Gets whether or not WarmUpIndicator is allowed to warm up indicators /// @@ -3372,6 +3379,61 @@ public void DeregisterIndicator(IndicatorBase indicator) } indicator.Consolidators.Clear(); + + // drop it from the per symbol tracking so 'DeregisterAll' doesn't hold on to it. An indicator can be + // tracked under multiple symbols (e.g. Beta), so we sweep all entries + lock (_registrationsLock) + { + foreach (var indicators in _registeredIndicators.Values) + { + indicators.Remove(indicator); + } + } + } + + /// + /// Deregisters all indicators and consolidators for the given symbol that were created through the algorithm + /// helper methods, e.g. , + /// or + /// , so they stop receiving data updates and + /// no longer consume resources. Indicators created for multiple symbols, e.g. + /// , are completely + /// deregistered when any of their symbols is deregistered. Typically called from + /// for each removed security, which avoids leaking + /// consolidators on universe churn; alternatively see + /// . + /// Indicators registered through the RegisterIndicator overloads are also deregistered. Consolidators added + /// directly through without an indicator are not tracked and are unaffected, + /// intentionally allowing them to be kept across universe removals. + /// + /// The symbol whose helper-created indicators and consolidators will be deregistered + [DocumentationAttribute(ConsolidatingData)] + [DocumentationAttribute(Indicators)] + public void DeregisterAll(Symbol symbol) + { + HashSet indicators; + HashSet consolidators; + lock (_registrationsLock) + { + _registeredIndicators.Remove(symbol, out indicators); + _registeredConsolidators.Remove(symbol, out consolidators); + } + + if (indicators != null) + { + foreach (var indicator in indicators) + { + DeregisterIndicator(indicator); + } + } + + if (consolidators != null) + { + foreach (var consolidator in consolidators) + { + SubscriptionManager.RemoveConsolidator(symbol, consolidator); + } + } } /// @@ -4454,6 +4516,27 @@ private void RegisterConsolidator(Symbol symbol, IDataConsolidator consolidator, // register the consolidator for automatic updates via SubscriptionManager SubscriptionManager.AddConsolidator(symbol, consolidator, tickType); + + // track helper-created indicators and consolidators per symbol so 'DeregisterAll' can dispose them in bulk + lock (_registrationsLock) + { + if (indicatorBase != null) + { + if (!_registeredIndicators.TryGetValue(symbol, out var indicators)) + { + _registeredIndicators[symbol] = indicators = new(); + } + indicators.Add(indicatorBase); + } + else + { + if (!_registeredConsolidators.TryGetValue(symbol, out var consolidators)) + { + _registeredConsolidators[symbol] = consolidators = new(); + } + consolidators.Add(consolidator); + } + } } private DateTime GetIndicatorAdjustedHistoryStart(IndicatorBase indicator, IEnumerable symbols, DateTime start, DateTime end, Resolution? resolution = null) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index daf195a7a1e1..81d32e6f0698 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.Linq; using System.Linq.Expressions; using System.Globalization; @@ -205,6 +206,19 @@ public QCAlgorithm() SubscriptionManager = new SubscriptionManager(_timeKeeper); Securities = new SecurityManager(_timeKeeper); + Securities.CollectionChanged += (sender, args) => + { + // automatically dispose helper-created indicators and consolidators when a security is completely + // removed from the algorithm, so universe churn doesn't leak consolidators. Opt-in via settings + // because some algorithms intentionally keep indicators alive across removals to reuse them on re-add + if (Settings.AutomaticIndicatorDeregistration && args.Action == NotifyCollectionChangedAction.Remove) + { + foreach (Security security in args.OldItems) + { + DeregisterAll(security.Symbol); + } + } + }; Transactions = new SecurityTransactionManager(this, Securities); Portfolio = new SecurityPortfolioManager(Securities, Transactions, Settings, DefaultOrderProperties); SignalExport = new SignalExportManager(this); diff --git a/Common/AlgorithmSettings.cs b/Common/AlgorithmSettings.cs index 2cf382305795..77ae12da9b1b 100644 --- a/Common/AlgorithmSettings.cs +++ b/Common/AlgorithmSettings.cs @@ -40,6 +40,13 @@ public class AlgorithmSettings : IAlgorithmSettings /// public bool AutomaticIndicatorWarmUp { get; set; } + /// + /// Gets whether indicators and consolidators created through the algorithm helper methods are automatically + /// deregistered when their security is removed from the algorithm, e.g. when it leaves the universe. + /// Disabled by default + /// + public bool AutomaticIndicatorDeregistration { get; set; } + /// /// True if should rebalance portfolio on security changes. True by default /// diff --git a/Common/Data/SubscriptionManager.cs b/Common/Data/SubscriptionManager.cs index 33238d08b49c..4f970726012a 100644 --- a/Common/Data/SubscriptionManager.cs +++ b/Common/Data/SubscriptionManager.cs @@ -238,11 +238,15 @@ public void RemoveConsolidator(Symbol symbol, IDataConsolidator consolidator) foreach (var subscription in _subscriptionManager.GetSubscriptionDataConfigs(symbol)) { subscription.Consolidators.Remove(consolidator); + } - if (_consolidators.Remove(consolidator, out var consolidatorsToScan)) - { - consolidatorsToScan.Dispose(); - } + // dispose the scan wrapper even if the symbol has no subscriptions left (e.g. the security was + // already removed from the universe): 'ScanPastConsolidators' only drops disposed wrappers, so a + // wrapper that isn't disposed here would be re-enqueued and re-scanned forever, leaking the + // consolidator and anything its event handlers reference + if (_consolidators.Remove(consolidator, out var consolidatorsToScan)) + { + consolidatorsToScan.Dispose(); } // dispose of the consolidator to remove any remaining event handlers diff --git a/Common/Interfaces/IAlgorithmSettings.cs b/Common/Interfaces/IAlgorithmSettings.cs index 14667144c819..ca1b806aa84d 100644 --- a/Common/Interfaces/IAlgorithmSettings.cs +++ b/Common/Interfaces/IAlgorithmSettings.cs @@ -28,6 +28,13 @@ public interface IAlgorithmSettings /// bool AutomaticIndicatorWarmUp { get; set; } + /// + /// Gets whether indicators and consolidators created through the algorithm helper methods are automatically + /// deregistered when their security is removed from the algorithm, e.g. when it leaves the universe. + /// Disabled by default + /// + bool AutomaticIndicatorDeregistration { get; set; } + /// /// True if should rebalance portfolio on security changes. True by default /// diff --git a/Tests/Algorithm/AlgorithmDeregisterAllTests.cs b/Tests/Algorithm/AlgorithmDeregisterAllTests.cs new file mode 100644 index 000000000000..53913c94e771 --- /dev/null +++ b/Tests/Algorithm/AlgorithmDeregisterAllTests.cs @@ -0,0 +1,156 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Linq; +using NUnit.Framework; +using QuantConnect.Algorithm; +using QuantConnect.Data.Consolidators; +using QuantConnect.Data.Market; +using QuantConnect.Tests.Engine.DataFeeds; + +namespace QuantConnect.Tests.Algorithm +{ + [TestFixture] + public class AlgorithmDeregisterAllTests + { + private QCAlgorithm _algorithm; + private Symbol _spy; + private Symbol _ibm; + + [SetUp] + public void Setup() + { + _algorithm = new QCAlgorithm(); + _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); + _spy = _algorithm.AddEquity("SPY").Symbol; + _ibm = _algorithm.AddEquity("IBM").Symbol; + } + + private int ConsolidatorCount(Symbol symbol) + { + return _algorithm.SubscriptionManager.Subscriptions + .Where(config => config.Symbol == symbol) + .Sum(config => config.Consolidators.Count); + } + + [Test] + public void DeregisterAllDisposesHelperCreatedIndicators() + { + var rsi = _algorithm.RSI(_spy, 14, resolution: Resolution.Minute); + var sma = _algorithm.SMA(_spy, 10, Resolution.Minute); + var ibmSma = _algorithm.SMA(_ibm, 10, Resolution.Minute); + + Assert.AreEqual(2, ConsolidatorCount(_spy)); + Assert.AreEqual(1, ConsolidatorCount(_ibm)); + + _algorithm.DeregisterAll(_spy); + + Assert.IsEmpty(rsi.Consolidators); + Assert.IsEmpty(sma.Consolidators); + Assert.AreEqual(0, ConsolidatorCount(_spy)); + + // other symbols are untouched + Assert.AreEqual(1, ibmSma.Consolidators.Count); + Assert.AreEqual(1, ConsolidatorCount(_ibm)); + } + + [Test] + public void DeregisterAllDisposesHelperCreatedConsolidators() + { + _algorithm.Consolidate(_spy, Resolution.Hour, (TradeBar bar) => { }); + + Assert.AreEqual(1, ConsolidatorCount(_spy)); + + _algorithm.DeregisterAll(_spy); + + Assert.AreEqual(0, ConsolidatorCount(_spy)); + } + + [Test] + public void DeregisterAllDisposesMultiSymbolIndicatorsThroughAnyOfTheirSymbols() + { + var beta = _algorithm.B(_spy, _ibm, 10, Resolution.Daily); + + Assert.AreEqual(2, beta.Consolidators.Count); + + _algorithm.DeregisterAll(_spy); + + // the indicator can't work without one of its legs, so it's completely deregistered + Assert.IsEmpty(beta.Consolidators); + Assert.AreEqual(0, ConsolidatorCount(_spy)); + Assert.AreEqual(0, ConsolidatorCount(_ibm)); + + // deregistering the other leg is a no-op + Assert.DoesNotThrow(() => _algorithm.DeregisterAll(_ibm)); + } + + [Test] + public void DeregisterAllIsANoOpAfterManualDeregistration() + { + var sma = _algorithm.SMA(_spy, 10, Resolution.Minute); + + _algorithm.DeregisterIndicator(sma); + Assert.AreEqual(0, ConsolidatorCount(_spy)); + + Assert.DoesNotThrow(() => _algorithm.DeregisterAll(_spy)); + Assert.AreEqual(0, ConsolidatorCount(_spy)); + } + + [Test] + public void DeregistersManuallyRegisteredIndicatorsButKeepsRawConsolidators() + { + // RegisterIndicator creates and manages the consolidator internally, so it's tracked too + var sma = new QuantConnect.Indicators.SimpleMovingAverage(10); + _algorithm.RegisterIndicator(_spy, sma, Resolution.Minute); + + // consolidators added directly through the subscription manager are not tracked and are + // intentionally kept, so users can hold on to them across universe removals + using var rawConsolidator = new TradeBarConsolidator(System.TimeSpan.FromHours(1)); + _algorithm.SubscriptionManager.AddConsolidator(_spy, rawConsolidator); + + _algorithm.DeregisterAll(_spy); + + Assert.IsEmpty(sma.Consolidators); + Assert.AreEqual(1, ConsolidatorCount(_spy)); + Assert.IsTrue(_algorithm.SubscriptionManager.Subscriptions + .Where(config => config.Symbol == _spy) + .Any(config => config.Consolidators.Contains(rawConsolidator))); + } + + [Test] + public void AutomaticallyDeregistersOnSecurityRemovalWhenEnabled() + { + _algorithm.Settings.AutomaticIndicatorDeregistration = true; + var sma = _algorithm.SMA(_spy, 10, Resolution.Minute); + var ibmSma = _algorithm.SMA(_ibm, 10, Resolution.Minute); + + _algorithm.Securities.Remove(_spy); + + Assert.IsEmpty(sma.Consolidators); + Assert.AreEqual(0, ConsolidatorCount(_spy)); + Assert.AreEqual(1, ibmSma.Consolidators.Count); + } + + [Test] + public void DoesNotAutomaticallyDeregisterByDefault() + { + var sma = _algorithm.SMA(_spy, 10, Resolution.Minute); + + _algorithm.Securities.Remove(_spy); + + Assert.AreEqual(1, sma.Consolidators.Count); + } + } +} diff --git a/Tests/Common/Data/SubscriptionManagerTests.cs b/Tests/Common/Data/SubscriptionManagerTests.cs index 33ec9a28b889..9381547115eb 100644 --- a/Tests/Common/Data/SubscriptionManagerTests.cs +++ b/Tests/Common/Data/SubscriptionManagerTests.cs @@ -709,6 +709,33 @@ def get_consolidator(): Assert.AreEqual(0, algorithm.SubscriptionManager.Subscriptions.Sum(x => x.Consolidators.Count)); } + [Test] + public void RemoveConsolidatorDisposesScanWrapperEvenWithoutSubscriptions() + { + var algorithm = new AlgorithmStub(); + var symbol = algorithm.AddEquity("SPY").Symbol; + + using var consolidator = new TradeBarConsolidator(TimeSpan.FromHours(1)); + algorithm.SubscriptionManager.AddConsolidator(symbol, consolidator); + + var consolidatorsField = typeof(SubscriptionManager) + .GetField("_consolidators", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + var wrappers = (Dictionary)consolidatorsField.GetValue(algorithm.SubscriptionManager); + Assert.AreEqual(1, wrappers.Count); + var wrapper = wrappers[consolidator]; + + // simulate the security having left the universe: its subscription data configs are gone before + // the consolidator is removed. The scan wrapper must still be disposed, otherwise + // 'ScanPastConsolidators' would re-enqueue and re-scan it forever, leaking the consolidator + algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); + Assert.IsEmpty(algorithm.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(symbol)); + + algorithm.SubscriptionManager.RemoveConsolidator(symbol, consolidator); + + Assert.AreEqual(0, wrappers.Count); + Assert.IsTrue(wrapper.Disposed); + } + [Test, Parallelizable(ParallelScope.None)] public void RunRemoveConsolidatorsRegressionAlgorithm() {