diff --git a/Algorithm.CSharp/HistoryAlgorithm.cs b/Algorithm.CSharp/HistoryAlgorithm.cs index 69979d180e08..1cc86baefdd4 100644 --- a/Algorithm.CSharp/HistoryAlgorithm.cs +++ b/Algorithm.CSharp/HistoryAlgorithm.cs @@ -57,7 +57,7 @@ public override void Initialize() _dailySma = new SimpleMovingAverage(14); // get the last calendar year's worth of SPY data at the configured resolution (daily) - var tradeBarHistory = History("SPY", TimeSpan.FromDays(365)); + IEnumerable tradeBarHistory = History("SPY", TimeSpan.FromDays(365)); AssertHistoryCount("History(\"SPY\", TimeSpan.FromDays(365))", tradeBarHistory, 250, SPY); // get the last calendar day's worth of SPY data at the specified resolution diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index 077b4cbda32f..13b8a438c978 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -355,15 +355,15 @@ public IEnumerable History(Universe universe, DateTime start /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable> History(TimeSpan span, Resolution? resolution = null, bool? fillForward = null, + public DataHistory> History(TimeSpan span, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData { return History(Securities.Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, - contractDepthOffset).Memoize(); + contractDepthOffset); } /// @@ -380,15 +380,15 @@ public IEnumerable> History(TimeSpan span, Resolution? reso /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable> History(IEnumerable symbols, TimeSpan span, Resolution? resolution = null, + public DataHistory> History(IEnumerable symbols, TimeSpan span, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData { return History(symbols, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode, - dataNormalizationMode, contractDepthOffset).Memoize(); + dataNormalizationMode, contractDepthOffset); } /// @@ -406,9 +406,9 @@ public IEnumerable> History(IEnumerable symbols, Ti /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable> History(IEnumerable symbols, int periods, Resolution? resolution = null, + public DataHistory> History(IEnumerable symbols, int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData @@ -433,9 +433,9 @@ public IEnumerable> History(IEnumerable symbols, in /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable> History(IEnumerable symbols, DateTime start, DateTime end, Resolution? resolution = null, + public DataHistory> History(IEnumerable symbols, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData @@ -458,15 +458,15 @@ public IEnumerable> History(IEnumerable symbols, Da /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData { return History(symbol, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode, - dataNormalizationMode, contractDepthOffset).Memoize(); + dataNormalizationMode, contractDepthOffset); } /// @@ -482,9 +482,9 @@ public IEnumerable History(Symbol symbol, TimeSpan span, Resolution? resol /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) { @@ -514,9 +514,9 @@ public IEnumerable History(Symbol symbol, int periods, Resolution? res /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData @@ -541,9 +541,9 @@ public IEnumerable History(Symbol symbol, int periods, Resolution? resolut /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical data, which can also be converted to a pandas DataFrame through its DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) where T : IBaseData @@ -565,9 +565,9 @@ public IEnumerable History(Symbol symbol, DateTime start, DateTime end, Re /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) { @@ -588,9 +588,9 @@ public IEnumerable History(Symbol symbol, TimeSpan span, Resolution? r /// The price scaling mode to use for the securities history /// The continuous contract desired offset from the current front month. /// For example, 0 will use the front month, 1 will use the back month contract - /// An enumerable of slice containing the requested historical data + /// The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property [DocumentationAttribute(HistoricalData)] - public IEnumerable History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null, + public DataHistory History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null) { @@ -607,8 +607,11 @@ public IEnumerable History(Symbol symbol, DateTime start, DateTime end " Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API."); } - return History(new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, + // Share the memoized enumerable with the lazy data frame so accessing the + // data frame does not re-execute the history request + var tradeBars = History(new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset).Get(symbol).Memoize(); + return new DataHistory(tradeBars, GetTypedHistoryDataFrame(tradeBars)); } /// @@ -937,7 +940,7 @@ private void GetLastKnownPricesImpl(IEnumerable symbols, Dictionary<(Sym /// /// This method will check for Python custom data types in order to call the right Slice.Get dynamic method /// - private IEnumerable GetDataTypedHistory(IEnumerable requests, Symbol symbol) + private DataHistory GetDataTypedHistory(IEnumerable requests, Symbol symbol) where T : IBaseData { var type = typeof(T); @@ -974,7 +977,10 @@ private IEnumerable GetDataTypedHistory(IEnumerable reques result = slices.Get(symbol); } - return result.Memoize(); + // Share the memoized enumerable with the lazy data frame so accessing the + // data frame does not re-execute the history request + var memoizedResult = result.Memoize(); + return new DataHistory(memoizedResult, GetTypedHistoryDataFrame(memoizedResult)); } /// @@ -983,7 +989,7 @@ private IEnumerable GetDataTypedHistory(IEnumerable reques /// /// This method will check for Python custom data types in order to call the right Slice.Get dynamic method /// - protected IEnumerable> GetDataTypedHistory(IEnumerable requests) + protected DataHistory> GetDataTypedHistory(IEnumerable requests) where T : IBaseData { var historyRequests = requests.Where(x => x != null).ToList(); @@ -1013,7 +1019,11 @@ protected IEnumerable> GetDataTypedHistory(IEnumerable>(memoizedResult, + GetTypedHistoryDataFrame(memoizedResult.SelectMany(x => x.Values), typeof(T))); } private IEnumerable History(IEnumerable requests, DateTimeZone timeZone) diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index 9859cbc4094b..bd7343e68480 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -2105,6 +2105,29 @@ protected PyObject GetDataFrame(IEnumerable data, bool flatten) return flatten ? history : TryCleanupCollectionDataFrame(typeof(T), history); } + /// + /// Creates the lazy pandas data frame of a typed history result, so the conversion is only performed + /// if the data frame is actually accessed. The caller shares the memoized history enumerable with the + /// lazy conversion so that accessing the data frame does not re-execute the history request. + /// + /// The typed history data points + /// The requested history data type. + /// Used to clean up the data frame of collection types, defaults to + private Lazy GetTypedHistoryDataFrame(IEnumerable data, Type dataType = null) + where T : IBaseData + { + return new Lazy(() => + { + if (PandasConverter == null) + { + // The pandas converter is only set for Python algorithms and research (see SetPandasConverter) + throw new InvalidOperationException( + "The DataFrame property is only available when running Python algorithms or research notebooks."); + } + return TryCleanupCollectionDataFrame(dataType ?? typeof(T), PandasConverter.GetDataFrame(data)); + }, isThreadSafe: false); + } + private IEnumerable RemoveMemoizing(IEnumerable data) { var memoizingEnumerable = data as MemoizingEnumerable; diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index cde0311c42df..ec530245b48e 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -63,6 +63,91 @@ def mapper(key): return {k: mapper(v) for k, v in key.items()} return key +def _flatten_keys(key): + '''Extracts the scalar string/Symbol keys from an indexing key, which might be + a plain scalar or a list/tuple/set of keys (e.g. df[["symbol", "close"]]) + ''' + if isinstance(key, (list, tuple, set)): + keys = [] + for item in key: + keys.extend(_flatten_keys(item)) + return keys + if isinstance(key, str) or type(key) is Symbol: + return [key] + return [] + +def _format_keys(keys, limit=20): + '''Formats a list of keys for an error message, truncating long lists''' + formatted = [f"'{key}'" for key in list(keys)[:limit]] + suffix = ', ...' if len(keys) > limit else '' + return f"[{', '.join(formatted)}{suffix}]" + +def _describe_missing_keys(target, keys): + '''Builds a self-describing suffix for the KeyError raised when both the mapped and + original keys are missing: what the requested keys were, which columns/index levels the + object actually has, and hints for the common mistakes (requesting an index level as a + column, requesting a symbol that lives in the index, or a symbol with no data at all). + ''' + # .loc/.iloc/.at indexers keep the DataFrame/Series they index in 'obj'; + # DataFrame.__getitem__/Index.get_loc receive the indexed object itself + obj = getattr(target, 'obj', target) + + if isinstance(obj, (pd.DataFrame, pd.Series)): + index = obj.index + elif isinstance(obj, pd.Index): + index = obj + else: + return '' + + level_names = [name if name is not None else f'level {i}' for i, name in enumerate(index.names)] + if isinstance(index, pd.MultiIndex): + index_description = f"index levels {_format_keys(level_names)}" + elif index.names[0] is not None: + index_description = f"a '{level_names[0]}' index" + else: + index_description = f"a {type(index).__name__}" + + details = [] + if isinstance(obj, pd.DataFrame): + details.append(f"The DataFrame has columns {_format_keys([str(column) for column in obj.columns])} " + f"and {index_description}") + elif isinstance(obj, pd.Series): + details.append(f"The Series has {index_description}") + elif isinstance(index, pd.MultiIndex): + details.append(f"The index has {index_description}") + else: + details.append(f"The index is {index_description}") + + hints = [] + for key in keys: + # requesting an index level as if it were a column, e.g. df[["symbol", "close"]] + if isinstance(key, str) and key in level_names: + hints.append(f"'{key}' is an index level, not a column: read it with df.index.get_level_values('{key}') " + "or move the index levels into columns with df.reset_index()") + continue + # requesting a symbol on the wrong axis: it lives in the index, e.g. df["SPY"] on a (symbol, time) indexed frame + found_in_level = False + for level in range(index.nlevels): + try: + # index membership is symbol-mapped (see wrap_bool_function below) + found_in_level = key in index.get_level_values(level) + except TypeError: + found_in_level = False + if found_in_level: + hints.append(f"'{key}' is a value of the '{level_names[level]}' index level: " + f"select it with df.loc['{key}'] or df.xs('{key}', level='{level_names[level]}'), " + "or use df.get(key) which returns None when the key is missing") + break + if found_in_level: + continue + # a known symbol that is simply absent, e.g. df.loc[symbol] when the symbol has no data for the requested period + mapped_key = mapper(key) if isinstance(key, str) else key + if type(mapped_key) is Symbol: + hints.append(f"'{key}' is a known Symbol but is not present in this object, it might have no data " + "for the requested period: use df.get(key) which returns None when the key is missing") + + return '. ' + '. '.join(details + hints) + def wrap_keyerror_function(f): '''Wraps function f with wrapped_function, used for functions that throw KeyError when not found. wrapped_function converts the args / kwargs to use alternative index keys and then calls the function. @@ -88,9 +173,19 @@ def wrapped_function(*args, **kwargs): try: return f(*args, **kwargs) except KeyError as e: + # The requested keys, including the ones nested in list keys like df[["symbol", "close"]] + requestedKeys = [key for arg in args[1:] for key in _flatten_keys(arg)] + requestedKeys += [key for arg in kwargs.values() for key in _flatten_keys(arg)] mKey = [str(arg) for arg in newargs if isinstance(arg, str) or isinstance(arg, Symbol)] - oKey = [str(arg) for arg in args if isinstance(arg, str) or isinstance(arg, Symbol)] - raise KeyError(f"No key found for either mapped or original key. Mapped Key: {mKey}; Original Key: {oKey}") + oKey = [str(key) for key in requestedKeys] or [str(arg) for arg in args if isinstance(arg, str) or isinstance(arg, Symbol)] + message = f"No key found for either mapped or original key. Mapped Key: {mKey}; Original Key: {oKey}" + # Describe the object being indexed and hint at the usual causes so the error is actionable + # without a debugging iteration. Never let the description itself mask the KeyError. + try: + message += _describe_missing_keys(args[0], requestedKeys) if len(args) > 0 else '' + except Exception: + pass + raise KeyError(message) wrapped_function.__name__ = f.__name__ return wrapped_function @@ -139,6 +234,43 @@ def wrapped_function(*args, **kwargs): # Wrap __contains__ to support Python syntax like 'SPY' in DataFrame pd.core.indexes.base.Index.__contains__ = wrap_bool_function(pd.core.indexes.base.Index.__contains__) +# Sentinel to detect when the original DataFrame.get found nothing, so a user-provided default is never confused with a miss +_GET_MISS = object() +_original_dataframe_get = pd.core.frame.DataFrame.get + +def _dataframe_get(self, key, default=None): + '''Extends DataFrame.get to also look for symbols in the index: history data frames are + indexed by symbol (and time), so history.get(symbol) returns the symbol's sub-frame, + or the default (None) when the symbol has no data instead of raising a KeyError. + Column lookups keep the original pandas semantics. + ''' + result = _original_dataframe_get(self, key, default=_GET_MISS) + if result is not _GET_MISS: + return result + + # Only symbols/tickers get the index lookup: other keys keep pandas' column-only semantics + if isinstance(key, str) or type(key) is Symbol: + try: + if isinstance(self.index, pd.MultiIndex): + for level in range(self.index.nlevels): + # index membership is symbol-mapped (see wrap_bool_function above) + if key in self.index.get_level_values(level): + for candidate in (key, mapper(key)): + try: + return self.xs(candidate, level=level) + except (KeyError, TypeError): + continue + elif key in self.index: + # loc is symbol-mapped already (see wrap_keyerror_function above) + return self.loc[key] + except (KeyError, TypeError, ValueError, IndexError): + pass + + return default + +_dataframe_get.__name__ = 'get' +pd.core.frame.DataFrame.get = _dataframe_get + # For compatibility with PandasData.cs usage of this module (Previously wrapped classes) FrozenList = pdFrozenList Index = pd.Index diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index e6c615377946..270674695291 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -391,6 +391,57 @@ public void VerifyHistorySupportsSpecificDataTypes() Assert.AreEqual(filteredTradeBars.Count, tradeBars.Count); } + [TestCase(Language.CSharp)] + [TestCase(Language.Python)] + public void TypedHistoryResultsExposeADataFrame(Language language) + { + var algorithm = GetAlgorithm(new DateTime(2013, 10, 8)); + var spy = algorithm.AddEquity("SPY", Resolution.Minute).Symbol; + algorithm.SetPandasConverter(); + + if (language == Language.CSharp) + { + var singleSymbolHistory = algorithm.History(spy, 10, Resolution.Minute); + var multiSymbolHistory = algorithm.History(new[] { spy }, 10, Resolution.Minute); + Assert.AreEqual(10, singleSymbolHistory.Count); + Assert.AreEqual(10, multiSymbolHistory.Count); + + using (Py.GIL()) + { + dynamic singleSymbolDataFrame = singleSymbolHistory.DataFrame; + dynamic multiSymbolDataFrame = multiSymbolHistory.DataFrame; + Assert.AreEqual(10, (int)singleSymbolDataFrame.shape[0]); + Assert.AreEqual(10, (int)multiSymbolDataFrame.shape[0]); + } + + // The results are still enumerable after the data frame is built: + // the memoized data is shared with the data frame conversion + Assert.AreEqual(10, singleSymbolHistory.Count()); + Assert.AreEqual(10, multiSymbolHistory.Count()); + } + else + { + using (Py.GIL()) + { + var getTypedHistory = PyModule.FromString("testModule", + @" +from AlgorithmImports import * + +def get_typed_history(algorithm, symbol): + bars = algorithm.history[TradeBar](symbol, 10, Resolution.MINUTE) + data_frame = bars.data_frame + # the result is still enumerable after the data frame is built + return data_frame.shape[0], len(list(bars)), 'close' in data_frame.columns + ").GetAttr("get_typed_history"); + + using var result = getTypedHistory.Invoke(algorithm.ToPython(), spy.ToPython()); + Assert.AreEqual(10, result[0].As()); + Assert.AreEqual(10, result[1].As()); + Assert.IsTrue(result[2].As()); + } + } + } + [TestCase(Language.CSharp)] [TestCase(Language.Python)] public void TickResolutionPeriodBasedHistoryRequestThrowsException(Language language) diff --git a/Tests/Python/PandasIndexingTests.cs b/Tests/Python/PandasIndexingTests.cs index 0644ff1ec372..2df3419677af 100644 --- a/Tests/Python/PandasIndexingTests.cs +++ b/Tests/Python/PandasIndexingTests.cs @@ -86,6 +86,91 @@ public void ExpectedException() } } + [Test] + public void KeyErrorDescribesMissingColumn() + { + using (Py.GIL()) + { + PyObject result = _pandasDataFrameTests.test_keyerror_describes_missing_column(); + var exception = result.As(); + + // Backwards compatible legacy wording plus the new self-describing details + StringAssert.Contains("No key found for either mapped or original key.", exception); + StringAssert.Contains("'volume'", exception); + StringAssert.Contains("The DataFrame has columns", exception); + StringAssert.Contains("'lastprice'", exception); + StringAssert.Contains("index levels ['symbol', 'time']", exception); + } + } + + [Test] + public void KeyErrorDescribesIndexLevelKey() + { + using (Py.GIL()) + { + PyObject result = _pandasDataFrameTests.test_keyerror_describes_index_level_key(); + var exception = result.As(); + + StringAssert.Contains("Original Key: ['symbol', 'lastprice']", exception); + StringAssert.Contains("'symbol' is an index level, not a column", exception); + StringAssert.Contains("reset_index", exception); + } + } + + [Test] + public void KeyErrorDescribesSymbolInIndex() + { + using (Py.GIL()) + { + PyObject result = _pandasDataFrameTests.test_keyerror_describes_symbol_in_index(); + var exception = result.As(); + + StringAssert.Contains("is a value of the 'symbol' index level", exception); + StringAssert.Contains("df.loc", exception); + } + } + + [Test] + public void KeyErrorDescribesMissingSymbol() + { + using (Py.GIL()) + { + PyObject result = _pandasDataFrameTests.test_keyerror_describes_missing_symbol(); + var exception = result.As(); + + StringAssert.Contains("is a known Symbol but is not present in this object", exception); + StringAssert.Contains("df.get(key)", exception); + } + } + + [Test] + public void GetWithSymbolReturnsSubFrame() + { + using (Py.GIL()) + { + Assert.IsTrue(_pandasDataFrameTests.test_get_symbol_returns_subframe().As()); + Assert.IsTrue(_pandasDataFrameTests.test_get_ticker_returns_subframe().As()); + } + } + + [Test] + public void GetWithMissingSymbolReturnsNone() + { + using (Py.GIL()) + { + Assert.IsTrue(_pandasDataFrameTests.test_get_missing_symbol_returns_none().As()); + } + } + + [Test] + public void GetWithColumnKeepsPandasSemantics() + { + using (Py.GIL()) + { + Assert.IsTrue(_pandasDataFrameTests.test_get_column_keeps_pandas_semantics().As()); + } + } + [Test] public void ColumnEqualsOnlyMatchingString() { diff --git a/Tests/Python/PandasTests/PandasIndexingTests.py b/Tests/Python/PandasTests/PandasIndexingTests.py index 77a9100420af..3a9a1b190864 100644 --- a/Tests/Python/PandasTests/PandasIndexingTests.py +++ b/Tests/Python/PandasTests/PandasIndexingTests.py @@ -59,6 +59,56 @@ def test_expected_exception(self): except KeyError as e: return str(e) + def test_keyerror_describes_missing_column(self): + # A missing column error must describe the frame: available columns and index levels + try: + self.spydf['volume'] + except KeyError as e: + return str(e) + + def test_keyerror_describes_index_level_key(self): + # 'symbol' is an index level, not a column: the error must say so + try: + self.spydf[['symbol', 'lastprice']] + except KeyError as e: + return str(e) + + def test_keyerror_describes_symbol_in_index(self): + # The requested symbol exists but lives in the index, not in the columns + try: + self.spydf['spy'] + except KeyError as e: + return str(e) + + def test_keyerror_describes_missing_symbol(self): + # A cached symbol with no data in the frame must be called out as missing data + try: + self.spydf.loc[self.aapl] + except KeyError as e: + return str(e) + + def test_get_symbol_returns_subframe(self): + # df.get(symbol) returns the symbol sub-frame when the symbol is in the index + subframe = self.spydf.get(self.spy) + return (subframe is not None + and 'lastprice' in subframe.columns + and subframe.index.nlevels == 1 + and len(subframe) == 100) + + def test_get_ticker_returns_subframe(self): + # df.get(ticker) maps the ticker through the symbol cache before the index lookup + subframe = self.spydf.get('spy') + return subframe is not None and len(subframe) == 100 + + def test_get_missing_symbol_returns_none(self): + # df.get of a symbol with no data returns None (or the default), never raises + return self.spydf.get(self.aapl) is None and self.spydf.get(self.aapl, 'default') == 'default' + + def test_get_column_keeps_pandas_semantics(self): + # df.get of a column still returns the column, and plain missing keys still return None + column = self.spydf.get('lastprice') + return column is not None and len(column) == 100 and self.spydf.get('banana') is None + def test_contains_user_defined_columns_with_spaces(self, column_name): # Adds a column, then try accessing it. # If the colums has white spaces, it should not fail