Fail loudly on canonical continuous future orders, add Future.Canonical and mapped-contract guidance

Algorithm.CSharp/ContinuousFutureCanonicalOrdersRegressionAlgorithm.cs

@@ -0,0 +1,192 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting the canonical continuous future symbol cannot be traded directly and fails loudly,
+    /// while its currently mapped contract can: <see cref="QCAlgorithm.CalculateOrderQuantity(Symbol, decimal)"/> returns
+    /// zero with an instructive error, <see cref="QCAlgorithm.SetHoldings(Symbol, double, bool, bool, string, Orders.IOrderProperties)"/>
+    /// submits no orders and direct orders produce an invalid ticket pointing to <see cref="Future.Mapped"/>.
+    /// Also asserts <see cref="Future.Canonical"/> and that <see cref="Future.Mapped"/> is null until the continuous
+    /// contract universe makes its first selection, after Initialize.
+    /// </summary>
+    public class ContinuousFutureCanonicalOrdersRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Future _continuousContract;
+        private bool _canonicalChecksDone;
+        private bool _traded;
+
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 7);
+            SetEndDate(2013, 10, 10);
+
+            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
+                dataNormalizationMode: DataNormalizationMode.BackwardsRatio,
+                dataMappingMode: DataMappingMode.OpenInterest,
+                contractDepthOffset: 0
+            );
+
+            if (_continuousContract.Mapped != null)
+            {
+                throw new RegressionTestException("Expected Future.Mapped to be null during Initialize: " +
+                    "the continuous contract universe does not make its first selection until after Initialize");
+            }
+
+            if (_continuousContract.Canonical != _continuousContract.Symbol)
+            {
+                throw new RegressionTestException("Expected Future.Canonical to be the continuous contract symbol itself");
+            }
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (_continuousContract.Mapped == null || !slice.Bars.ContainsKey(_continuousContract.Symbol))
+            {
+                return;
+            }
+
+            if (!_canonicalChecksDone)
+            {
+                _canonicalChecksDone = true;
+                var canonical = _continuousContract.Symbol;
+
+                // Continuous contract data is keyed by the canonical symbol
+                if (slice.Bars[canonical].Symbol != canonical)
+                {
+                    throw new RegressionTestException("Expected the continuous contract bar to be keyed by the canonical symbol");
+                }
+
+                // The canonical symbol is not tradable: no order quantity can be computed for it
+                if (CalculateOrderQuantity(canonical, 1m) != 0)
+                {
+                    throw new RegressionTestException("Expected CalculateOrderQuantity to return 0 for the canonical symbol");
+                }
+
+                // SetHoldings must not submit orders for the canonical symbol
+                if (SetHoldings(canonical, 0.5).Count != 0 || Portfolio.Invested)
+                {
+                    throw new RegressionTestException("Expected SetHoldings to not submit orders for the canonical symbol");
+                }
+
+                // Direct orders on the canonical symbol are rejected with an instructive message
+                var ticket = MarketOrder(canonical, 1);
+                if (ticket.Status != OrderStatus.Invalid)
+                {
+                    throw new RegressionTestException("Expected a market order on the canonical symbol to be invalid");
+                }
+                if (!ticket.SubmitRequest.Response.ErrorMessage.Contains("canonical"))
+                {
+                    throw new RegressionTestException("Expected the invalid canonical order error message to explain " +
+                        $"the symbol is canonical, but was: '{ticket.SubmitRequest.Response.ErrorMessage}'");
+                }
+            }
+
+            if (!_traded)
+            {
+                _traded = true;
+
+                // The currently mapped contract is the tradable one
+                var ticket = MarketOrder(_continuousContract.Mapped, 1);
+                if (ticket.Status == OrderStatus.Invalid)
+                {
+                    throw new RegressionTestException("Expected a market order on the mapped contract to be valid");
+                }
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_canonicalChecksDone)
+            {
+                throw new RegressionTestException("No data was received so the canonical symbol checks were not performed");
+            }
+
+            if (!Portfolio.Invested)
+            {
+                throw new RegressionTestException("Expected to hold a position in the mapped contract");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 10881;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "79.914%"},
+            {"Drawdown", "1.900%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100645.7"},
+            {"Net Profit", "0.646%"},
+            {"Sharpe Ratio", "3.958"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.372"},
+            {"Beta", "0.815"},
+            {"Annual Standard Deviation", "0.222"},
+            {"Annual Variance", "0.049"},
+            {"Information Ratio", "-12.526"},
+            {"Tracking Error", "0.052"},
+            {"Treynor Ratio", "1.077"},
+            {"Total Fees", "$2.15"},
+            {"Estimated Strategy Capacity", "$2800000000.00"},
+            {"Lowest Capacity Asset", "ES VMKLFZIH2MTD"},
+            {"Portfolio Turnover", "20.89%"},
+            {"Drawdown Recovery", "3"},
+            {"OrderListHash", "2338180a2a964389525a9f1221f97a06"}
+        };
+    }
+}

Algorithm.Python/ContinuousFutureCanonicalOrdersRegressionAlgorithm.py

@@ -0,0 +1,85 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm asserting the canonical continuous future symbol cannot be traded directly and fails loudly,
+### while its currently mapped contract can: calculate_order_quantity returns zero with an instructive error,
+### set_holdings submits no orders and direct orders produce an invalid ticket pointing to future.mapped.
+### Also asserts future.canonical and that future.mapped is None until the continuous contract universe makes
+### its first selection, after initialize.
+### </summary>
+class ContinuousFutureCanonicalOrdersRegressionAlgorithm(QCAlgorithm):
+
+    def initialize(self):
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 10)
+
+        self._continuous_contract = self.add_future(Futures.Indices.SP_500_E_MINI,
+                                                    data_normalization_mode=DataNormalizationMode.BACKWARDS_RATIO,
+                                                    data_mapping_mode=DataMappingMode.OPEN_INTEREST,
+                                                    contract_depth_offset=0)
+
+        if self._continuous_contract.mapped is not None:
+            raise AssertionError("Expected future.mapped to be None during initialize: "
+                                 "the continuous contract universe does not make its first selection until after initialize")
+
+        if self._continuous_contract.canonical != self._continuous_contract.symbol:
+            raise AssertionError("Expected future.canonical to be the continuous contract symbol itself")
+
+        self._canonical_checks_done = False
+        self._traded = False
+
+    def on_data(self, slice):
+        if self._continuous_contract.mapped is None or not slice.bars.contains_key(self._continuous_contract.symbol):
+            return
+
+        if not self._canonical_checks_done:
+            self._canonical_checks_done = True
+            canonical = self._continuous_contract.symbol
+
+            # Continuous contract data is keyed by the canonical symbol, and the future object itself can be used as the key
+            if slice.bars.get(canonical) is None or slice.bars.get(self._continuous_contract) is None:
+                raise AssertionError("Expected the continuous contract bar to be accessible through the canonical symbol and the future object")
+
+            # The canonical symbol is not tradable: no order quantity can be computed for it
+            if self.calculate_order_quantity(canonical, 1.0) != 0:
+                raise AssertionError("Expected calculate_order_quantity to return 0 for the canonical symbol")
+
+            # set_holdings must not submit orders for the canonical symbol
+            if len(self.set_holdings(canonical, 0.5)) != 0 or self.portfolio.invested:
+                raise AssertionError("Expected set_holdings to not submit orders for the canonical symbol")
+
+            # Direct orders on the canonical symbol are rejected with an instructive message
+            ticket = self.market_order(canonical, 1)
+            if ticket.status != OrderStatus.INVALID:
+                raise AssertionError("Expected a market order on the canonical symbol to be invalid")
+            if "canonical" not in ticket.submit_request.response.error_message:
+                raise AssertionError("Expected the invalid canonical order error message to explain the symbol is canonical, "
+                                     f"but was: '{ticket.submit_request.response.error_message}'")
+
+        if not self._traded:
+            self._traded = True
+
+            # The currently mapped contract is the tradable one
+            ticket = self.market_order(self._continuous_contract.mapped, 1)
+            if ticket.status == OrderStatus.INVALID:
+                raise AssertionError("Expected a market order on the mapped contract to be valid")
+
+    def on_end_of_algorithm(self):
+        if not self._canonical_checks_done:
+            raise AssertionError("No data was received so the canonical symbol checks were not performed")
+
+        if not self.portfolio.invested:
+            raise AssertionError("Expected to hold a position in the mapped contract")

Algorithm/QCAlgorithm.Trading.cs

@@ -34,6 +34,7 @@ public partial class QCAlgorithm
         private bool _isDailyResolutionMarketOrderConversionWarningSent;
         private bool _isMarketOnOpenOrderRestrictedForFuturesWarningSent;
         private bool _isGtdTfiForMooAndMocOrdersValidationWarningSent;
+        private bool _isFutureOrderPriceFarFromMarketPriceWarningSent;
         private bool _isOptionsOrderOnStockSplitWarningSent;
         private bool _liquidateSymbolNotFoundWarningSent;
 
@@ -1079,8 +1080,12 @@ private OrderResponse PreOrderChecksImpl(SubmitOrderRequest request)
 
             if (!security.IsTradable)
             {
+                // Canonical symbols (e.g. the continuous futures contract) are never tradable:
+                // point the user to the mapped contract instead of just rejecting the order
                 return OrderResponse.Error(request, OrderResponseErrorCode.NonTradableSecurity,
-                    $"The security with symbol '{request.Symbol}' is marked as non-tradable."
+                    security.Symbol.IsCanonical()
+                        ? Messages.QCAlgorithm.CanonicalSymbolNotTradable(security.Symbol)
+                        : $"The security with symbol '{request.Symbol}' is marked as non-tradable."
                 );
             }
 
@@ -1113,6 +1118,38 @@ private OrderResponse PreOrderChecksImpl(SubmitOrderRequest request)
                 return OrderResponse.Error(request, OrderResponseErrorCode.SecurityPriceZero, request.Symbol.GetZeroPriceMessage());
             }
 
+            // Futures continuous contract prices are adjusted unless DataNormalizationMode.Raw is used, so stop/limit prices
+            // computed from them can sit far away from the raw prices the mapped contract actually trades at, producing orders
+            // that fill immediately or never. Warn once when an order price deviates >10% from the contract's market price.
+            if (!_isFutureOrderPriceFarFromMarketPriceWarningSent && security.Type == SecurityType.Future)
+            {
+                var maxDeviation = 0m;
+                foreach (var orderPrice in new[] { request.StopPrice, request.LimitPrice, request.TriggerPrice })
+                {
+                    if (orderPrice != 0)
+                    {
+                        maxDeviation = Math.Max(maxDeviation, Math.Abs(orderPrice - price) / price);
+                    }
+                }
+
+                if (maxDeviation > 0.1m)
+                {
+                    var normalizationMode = SubscriptionManager.SubscriptionDataConfigService
+                        .GetSubscriptionDataConfigs(request.Symbol.Canonical)
+                        .Select(x => x.DataNormalizationMode)
+                        .FirstOrDefault(x => x != DataNormalizationMode.Raw, DataNormalizationMode.Raw);
+
+                    if (normalizationMode != DataNormalizationMode.Raw)
+                    {
+                        _isFutureOrderPriceFarFromMarketPriceWarningSent = true;
+                        Debug($"Warning: The {request.OrderType} order price(s) for '{request.Symbol.Value}' deviate more than 10% from its market price ({price.SmartRounding()}). " +
+                            $"The continuous contract '{request.Symbol.Canonical}' uses DataNormalizationMode.{normalizationMode}, whose adjusted prices can differ significantly " +
+                            "from the raw prices the mapped contract trades at. If the order price was computed from continuous contract data, use the mapped contract's " +
+                            "price instead (Securities[future.Mapped].Price) or add the future with DataNormalizationMode.Raw.");
+                    }
+                }
+            }
+
             // check quote currency existence/conversion rate on all orders
             var quoteCurrency = security.QuoteCurrency.Symbol;
             if (!Portfolio.CashBook.TryGetValue(quoteCurrency, out var quoteCash))
@@ -1268,6 +1305,13 @@ private OrderResponse PreOrderChecksImpl(SubmitOrderRequest request)
         /// </summary>
         private Security GetSecurityForOrder(Symbol symbol)
         {
+            if (symbol == null)
+            {
+                // A common source of null symbols is accessing Future.Mapped from Initialize, before the first mapping.
+                // Explain that instead of letting an NRE bubble up. See Messages.QCAlgorithm.OrderSymbolNull
+                throw new ArgumentNullException(nameof(symbol), Messages.QCAlgorithm.OrderSymbolNull());
+            }
+
             var isCanonical = symbol.IsCanonical();
             if (Securities.TryGetValue(symbol, out var security) &&
                 // Let canonical and delisted securities through instead of throwing. An invalid ticket will be returned later on when trying to submit the order.

Common/Algorithm/Framework/Portfolio/PortfolioTarget.cs

@@ -153,6 +153,16 @@ public static IPortfolioTarget Percent(IAlgorithm algorithm, Symbol symbol, deci
                 return null;
             }
 
+            // The canonical continuous futures contract is not tradable, so instead of producing a quantity that will
+            // only generate an invalid order, fail loudly here. Continuous contract data gives the canonical security
+            // a non-zero price, so without this check a plausible-looking quantity would be silently computed.
+            // Other canonical symbols (options) have no price and are already rejected by the zero-price check below.
+            if (security.Symbol.IsCanonical() && security.Symbol.SecurityType == SecurityType.Future)
+            {
+                algorithm.Error(Messages.PortfolioTarget.UnableToComputeOrderQuantityForCanonicalSymbol(security.Symbol));
+                return null;
+            }
+
             if (security.Price == 0)
             {
                 algorithm.Error(symbol.GetZeroPriceMessage());

Common/Messages/Messages.Algorithm.Framework.Portfolio.cs

@@ -63,6 +63,15 @@ public static string UnableToComputeOrderQuantityDueToNullResult(QuantConnect.Sy
                 return Invariant($"Unable to compute order quantity of {symbol}. Reason: {result.Reason} Returning null.");
             }
 
+            /// <summary>
+            /// Returns a string message saying an order quantity cannot be computed for the given canonical symbol
+            /// </summary>
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static string UnableToComputeOrderQuantityForCanonicalSymbol(QuantConnect.Symbol symbol)
+            {
+                return Invariant($"Unable to compute an order quantity for '{symbol}'. {QCAlgorithm.CanonicalSymbolNotTradable(symbol)}");
+            }
+
             /// <summary>
             /// Parses the given portfolio target into a string message containing basic information about it
             /// </summary>