Skip to content

Improve pandas history ergonomics: self-describing KeyErrors, DataFrame.get(symbol), lazy DataFrame on typed history #9668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jhonabreul wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Improve pandas history ergonomics: self-describing KeyErrors, DataFrame.get(symbol), lazy DataFrame on typed history #9668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8eb5cdf
Add self-describing PandasMapper KeyErrors and DataFrame.get symbol l…
jhonabreul Aug 11, 2026
9d2f069
Expose a lazy pandas DataFrame on typed history results
jhonabreul Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Algorithm.CSharp/HistoryAlgorithm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public override void Initialize()
_dailySma = new SimpleMovingAverage(14);

// get the last calendar year's worth of SPY data at the configured resolution (daily)
var tradeBarHistory = History<TradeBar>("SPY", TimeSpan.FromDays(365));
IEnumerable<TradeBar> tradeBarHistory = History<TradeBar>("SPY", TimeSpan.FromDays(365));
AssertHistoryCount("History<TradeBar>(\"SPY\", TimeSpan.FromDays(365))", tradeBarHistory, 250, SPY);

// get the last calendar day's worth of SPY data at the specified resolution
Expand Down
66 changes: 38 additions & 28 deletions Algorithm/QCAlgorithm.History.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,15 +355,15 @@ public IEnumerable<BaseDataCollection> History(Universe universe, DateTime start
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<DataDictionary<T>> History<T>(TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<DataDictionary<T>> History<T>(TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
where T : IBaseData
{
return History<T>(Securities.Keys, span, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
contractDepthOffset).Memoize();
contractDepthOffset);
}

/// <summary>
Expand All @@ -380,15 +380,15 @@ public IEnumerable<DataDictionary<T>> History<T>(TimeSpan span, Resolution? reso
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, TimeSpan span, Resolution? resolution = null,
public DataHistory<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, TimeSpan span, Resolution? resolution = null,
bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
where T : IBaseData
{
return History<T>(symbols, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
dataNormalizationMode, contractDepthOffset).Memoize();
dataNormalizationMode, contractDepthOffset);
}

/// <summary>
Expand All @@ -406,9 +406,9 @@ public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, Ti
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null,
public DataHistory<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null,
bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
where T : IBaseData
Expand All @@ -433,9 +433,9 @@ public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, in
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end, Resolution? resolution = null,
public DataHistory<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, DateTime start, DateTime end, Resolution? resolution = null,
bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
where T : IBaseData
Expand All @@ -458,15 +458,15 @@ public IEnumerable<DataDictionary<T>> History<T>(IEnumerable<Symbol> symbols, Da
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<T> History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<T> History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
where T : IBaseData
{
return History<T>(symbol, Time - span, Time, resolution, fillForward, extendedMarketHours, dataMappingMode,
dataNormalizationMode, contractDepthOffset).Memoize();
dataNormalizationMode, contractDepthOffset);
}

/// <summary>
Expand All @@ -482,9 +482,9 @@ public IEnumerable<T> History<T>(Symbol symbol, TimeSpan span, Resolution? resol
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<TradeBar> History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<TradeBar> History(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
{
Expand Down Expand Up @@ -514,9 +514,9 @@ public IEnumerable<TradeBar> History(Symbol symbol, int periods, Resolution? res
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<T> History<T>(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<T> History<T>(Symbol symbol, int periods, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
where T : IBaseData
Expand All @@ -541,9 +541,9 @@ public IEnumerable<T> History<T>(Symbol symbol, int periods, Resolution? resolut
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical data, which can also be converted to a pandas DataFrame through its DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<T> History<T>(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<T> History<T>(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
where T : IBaseData
Expand All @@ -565,9 +565,9 @@ public IEnumerable<T> History<T>(Symbol symbol, DateTime start, DateTime end, Re
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<TradeBar> History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<TradeBar> History(Symbol symbol, TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
{
Expand All @@ -588,9 +588,9 @@ public IEnumerable<TradeBar> History(Symbol symbol, TimeSpan span, Resolution? r
/// <param name="dataNormalizationMode">The price scaling mode to use for the securities history</param>
/// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
/// For example, 0 will use the front month, 1 will use the back month contract</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
/// <returns>The historical trade bars, which can also be converted to a pandas DataFrame through the result's DataFrame property</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<TradeBar> History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
public DataHistory<TradeBar> History(Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
{
Expand All @@ -607,8 +607,11 @@ public IEnumerable<TradeBar> History(Symbol symbol, DateTime start, DateTime end
" Please use the generic version with Tick type parameter or provide a list of Symbols to use the Slice history request API.");
}

return History(new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
// Share the memoized enumerable with the lazy data frame so accessing the
// data frame does not re-execute the history request
var tradeBars = History(new[] { symbol }, start, end, resolutionToUse, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode,
contractDepthOffset).Get(symbol).Memoize();
return new DataHistory<TradeBar>(tradeBars, GetTypedHistoryDataFrame(tradeBars));
}

/// <summary>
Expand Down Expand Up @@ -937,7 +940,7 @@ private void GetLastKnownPricesImpl(IEnumerable<Symbol> symbols, Dictionary<(Sym
/// <remarks>
/// This method will check for Python custom data types in order to call the right Slice.Get dynamic method
/// </remarks>
private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests, Symbol symbol)
private DataHistory<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests, Symbol symbol)
where T : IBaseData
{
var type = typeof(T);
Expand Down Expand Up @@ -974,7 +977,10 @@ private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> reques
result = slices.Get<T>(symbol);
}

return result.Memoize();
// Share the memoized enumerable with the lazy data frame so accessing the
// data frame does not re-execute the history request
var memoizedResult = result.Memoize();
return new DataHistory<T>(memoizedResult, GetTypedHistoryDataFrame(memoizedResult));
}

/// <summary>
Expand All @@ -983,7 +989,7 @@ private IEnumerable<T> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> reques
/// <remarks>
/// This method will check for Python custom data types in order to call the right Slice.Get dynamic method
/// </remarks>
protected IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
protected DataHistory<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<HistoryRequest> requests)
where T : IBaseData
{
var historyRequests = requests.Where(x => x != null).ToList();
Expand Down Expand Up @@ -1013,7 +1019,11 @@ protected IEnumerable<DataDictionary<T>> GetDataTypedHistory<T>(IEnumerable<Hist
}
}

return result.Memoize();
// Share the memoized enumerable with the lazy data frame so accessing the
// data frame does not re-execute the history request
var memoizedResult = result.Memoize();
return new DataHistory<DataDictionary<T>>(memoizedResult,
GetTypedHistoryDataFrame(memoizedResult.SelectMany(x => x.Values), typeof(T)));
}

private IEnumerable<Slice> History(IEnumerable<HistoryRequest> requests, DateTimeZone timeZone)
Expand Down
23 changes: 23 additions & 0 deletions Algorithm/QCAlgorithm.Python.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2105,6 +2105,29 @@ protected PyObject GetDataFrame<T>(IEnumerable<T> data, bool flatten)
return flatten ? history : TryCleanupCollectionDataFrame(typeof(T), history);
}

/// <summary>
/// Creates the lazy pandas data frame of a typed history result, so the conversion is only performed
/// if the data frame is actually accessed. The caller shares the memoized history enumerable with the
/// lazy conversion so that accessing the data frame does not re-execute the history request.
/// </summary>
/// <param name="data">The typed history data points</param>
/// <param name="dataType">The requested history data type.
/// Used to clean up the data frame of collection types, defaults to <typeparamref name="T"/></param>
private Lazy<PyObject> GetTypedHistoryDataFrame<T>(IEnumerable<T> data, Type dataType = null)
where T : IBaseData
{
return new Lazy<PyObject>(() =>
{
if (PandasConverter == null)
{
// The pandas converter is only set for Python algorithms and research (see SetPandasConverter)
throw new InvalidOperationException(
"The DataFrame property is only available when running Python algorithms or research notebooks.");
}
return TryCleanupCollectionDataFrame(dataType ?? typeof(T), PandasConverter.GetDataFrame(data));
}, isThreadSafe: false);
}

private IEnumerable<T> RemoveMemoizing<T>(IEnumerable<T> data)
{
var memoizingEnumerable = data as MemoizingEnumerable<T>;
Expand Down
Loading
Loading