diff --git a/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs new file mode 100644 index 000000000000..05a86aed9a37 --- /dev/null +++ b/Algorithm.CSharp/ObjectStoreErgonomicsRegressionAlgorithm.cs @@ -0,0 +1,162 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using QuantConnect.Interfaces; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the + /// key rules stated in the error, converts arbitrary names + /// into supported keys, is an alias of Save and reading a + /// missing key lists the available keys in the error. The Python version of this algorithm additionally + /// covers the tolerant save_json/read_json and save_dataframe helpers + /// + public class ObjectStoreErgonomicsRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + /// + /// Initialize the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. + /// + public override void Initialize() + { + SetStartDate(2013, 10, 7); + SetEndDate(2013, 10, 11); + + SetBenchmark(x => 0); + + AddEquity("SPY", Resolution.Daily); + } + + /// + /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation. + /// + public override void OnEndOfAlgorithm() + { + // an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix + const string invalidKey = "trade_log_ai_hardware_&_cloud.csv"; + var errorMessage = string.Empty; + try + { + ObjectStore.Save(invalidKey, "a,b\n1,2"); + } + catch (ArgumentException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains(Storage.ObjectStore.SupportedKeyRules) || !errorMessage.Contains("SanitizeKey")) + { + throw new RegressionTestException($"Expected the key rules in the unsupported key error, got: '{errorMessage}'"); + } + + // SanitizeKey converts the arbitrary name into a supported key + var sanitized = Storage.ObjectStore.SanitizeKey(invalidKey); + if (sanitized != "trade_log_ai_hardware___cloud.csv") + { + throw new RegressionTestException($"Unexpected sanitized key: '{sanitized}'"); + } + if (!Storage.ObjectStore.IsSupportedKey(sanitized) || !ObjectStore.Save(sanitized, "a,b\n1,2")) + { + throw new RegressionTestException("Expected the sanitized key to be storable"); + } + + // SaveText is an alias of Save + if (!ObjectStore.SaveText("ergonomics_report.txt", "The strategy went up") || + ObjectStore.Read("ergonomics_report.txt") != "The strategy went up") + { + throw new RegressionTestException("Expected the SaveText/Read round trip to succeed"); + } + + // reading a missing key lists the available keys in the error + errorMessage = string.Empty; + try + { + ObjectStore.ReadBytes("ergonomics_missing.json"); + } + catch (KeyNotFoundException exception) + { + errorMessage = exception.Message; + } + if (!errorMessage.Contains("Keys: [") || !errorMessage.Contains($"'{sanitized}'")) + { + throw new RegressionTestException($"Expected the available keys in the missing key error, got: '{errorMessage}'"); + } + + ObjectStore.Delete(sanitized); + ObjectStore.Delete("ergonomics_report.txt"); + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 6; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 0; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$0.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} + }; + } +} diff --git a/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py new file mode 100644 index 000000000000..fb0eaf184af2 --- /dev/null +++ b/Algorithm.Python/ObjectStoreErgonomicsRegressionAlgorithm.py @@ -0,0 +1,104 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from decimal import Decimal + +### +### Regression algorithm asserting the ObjectStore ergonomics: unsupported keys are rejected with the +### key rules stated in the error, sanitize_key converts arbitrary names into supported keys, save_text +### is an alias of save, reading a missing key lists the available keys in the error, and the tolerant +### save_json/read_json and save_dataframe helpers handle datetime/date/Decimal/Symbol and data frames +### out of the box. +### +class ObjectStoreErgonomicsRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2013, 10, 7) + self.set_end_date(2013, 10, 11) + + self.set_benchmark(lambda x: 0) + + self.spy = self.add_equity("SPY", Resolution.DAILY).symbol + + def on_end_of_algorithm(self): + # an unsupported key, e.g. built from a user-facing name, is rejected stating the rules and the fix. + # note: BaseException because AlgorithmImports shadows the builtin Exception with System.Exception + invalid_key = "trade_log_ai_hardware_&_cloud.csv" + error = None + try: + self.object_store.save(invalid_key, "a,b\n1,2") + except BaseException as e: + error = str(e) + if error is None or "keys may only contain" not in error or "SanitizeKey" not in error: + raise AssertionError(f"Expected the key rules in the unsupported key error, got: '{error}'") + + # sanitize_key converts the arbitrary name into a supported key + sanitized = self.object_store.sanitize_key(invalid_key) + if sanitized != "trade_log_ai_hardware___cloud.csv": + raise AssertionError(f"Unexpected sanitized key: '{sanitized}'") + if not self.object_store.save(sanitized, "a,b\n1,2"): + raise AssertionError("Expected the sanitized key to be storable") + + # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + trade_log = { + "time": datetime(2013, 10, 11, 16, 0, 0), + "date": date(2013, 10, 11), + "qty": Decimal("1.5"), + "symbol": self.spy, + "fills": [{"price": Decimal("167.42"), self.spy: 10}] + } + if not self.object_store.save_json("ergonomics/trade_log.json", trade_log): + raise AssertionError("Expected save_json to succeed") + data = self.object_store.read_json("ergonomics/trade_log.json") + expected = { + "time": "2013-10-11T16:00:00", + "date": "2013-10-11", + "qty": 1.5, + "symbol": "SPY", + "fills": [{"price": 167.42, "SPY": 10}] + } + if data != expected: + raise AssertionError(f"Unexpected read_json round trip result: {data}") + + # read_json returns the given default when the key is missing + if self.object_store.read_json("ergonomics/missing.json") is not None: + raise AssertionError("Expected read_json of a missing key to return None") + if self.object_store.read_json("ergonomics/missing.json", {"warmed_up": False}) != {"warmed_up": False}: + raise AssertionError("Expected read_json of a missing key to return the given default") + + # save_dataframe stores a pandas DataFrame as CSV + frame = pd.DataFrame({"close": [167.42, 168.0]}, index=pd.to_datetime(["2013-10-10", "2013-10-11"])) + if not self.object_store.save_dataframe("ergonomics/history.csv", frame): + raise AssertionError("Expected save_dataframe to succeed") + csv = self.object_store.read("ergonomics/history.csv") + if "close" not in csv or "2013-10-10" not in csv: + raise AssertionError(f"Unexpected save_dataframe content: {csv}") + + # save_text is an alias of save + if not self.object_store.save_text("ergonomics/report.txt", "The strategy went up"): + raise AssertionError("Expected save_text to succeed") + if self.object_store.read("ergonomics/report.txt") != "The strategy went up": + raise AssertionError("Expected the save_text/read round trip to succeed") + + # reading a missing key lists the available keys in the error + error = None + try: + self.object_store.read_bytes("ergonomics/missing.json") + except BaseException as e: + error = str(e) + if error is None or "Keys: [" not in error or f"'{sanitized}'" not in error: + raise AssertionError(f"Expected the available keys in the missing key error, got: '{error}'") + + for key in [sanitized, "ergonomics/trade_log.json", "ergonomics/history.csv", "ergonomics/report.txt"]: + self.object_store.delete(key) diff --git a/Common/Storage/ObjectStore.cs b/Common/Storage/ObjectStore.cs index bb4795c00d60..d2b73bb7766a 100644 --- a/Common/Storage/ObjectStore.cs +++ b/Common/Storage/ObjectStore.cs @@ -17,9 +17,12 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Xml.Serialization; using Newtonsoft.Json; +using Python.Runtime; using QuantConnect.Interfaces; using QuantConnect.Packets; @@ -30,6 +33,23 @@ namespace QuantConnect.Storage /// public class ObjectStore : IObjectStore { + // The single source of the key charset/extension rules, also enforced by the LocalObjectStore implementation + private static readonly Regex SupportedKeyRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); + private static readonly Regex UnsupportedKeyCharactersRegex = new(@"[^a-zA-Z0-9\\/_#\-\$= .]", RegexOptions.Compiled); + private static readonly Regex KeyExtensionRegex = new(@"^[a-zA-Z0-9]+$", RegexOptions.Compiled); + + // Python helpers for the PyObject APIs, lazily created under the GIL + private static PyObject _jsonSerializeMethod; + private static PyObject _jsonDeserializeMethod; + private static PyObject _dataFrameSerializeMethod; + + /// + /// Human readable description of the format an object store key must follow, stated in key validation errors + /// + public static string SupportedKeyRules { get; } = "keys may only contain english letters, numbers, spaces and the characters" + + " '/', '\\', '_', '#', '-', '$' and '=', plus at most one '.' followed by a letters-and-numbers-only extension," + + " e.g. 'folder/trade_log-2024.csv'"; + /// /// Gets the maximum storage limit in bytes /// @@ -159,6 +179,57 @@ public string ReadString(string path, Encoding encoding = null) return Read(path, encoding); } + /// + /// Determines whether the given key follows the object store key format, see + /// + /// The object store key to validate + /// True if the key is supported + public static bool IsSupportedKey(string key) + { + return !string.IsNullOrEmpty(key) && SupportedKeyRegex.IsMatch(key) + // just in case + && key.Count(c => c == '/') <= 100 && key.Count(c => c == '\\') <= 100; + } + + /// + /// Converts an arbitrary name into a supported object store key by replacing every unsupported character + /// with '_', keeping at most the final '.extension'. Useful for programmatically-built keys, for instance + /// from user-facing names: 'trade log: AI & Cloud.csv' becomes 'trade log_ AI _ Cloud.csv' + /// + /// The arbitrary name to sanitize + /// A key following + public static string SanitizeKey(string key) + { + if (string.IsNullOrEmpty(key)) + { + throw new ArgumentException("ObjectStore.SanitizeKey(): key cannot be null or empty", nameof(key)); + } + if (IsSupportedKey(key)) + { + return key; + } + + var sanitized = UnsupportedKeyCharactersRegex.Replace(key, "_"); + + // a single trailing '.extension' of letters and numbers is allowed: keep the last dot if it + // introduces a valid extension and replace every other dot + var lastDot = sanitized.LastIndexOf('.'); + if (lastDot > 0 && KeyExtensionRegex.IsMatch(sanitized.Substring(lastDot + 1))) + { + sanitized = sanitized.Substring(0, lastDot).Replace('.', '_') + sanitized.Substring(lastDot); + } + else if (lastDot >= 0) + { + sanitized = sanitized.Replace('.', '_'); + } + + if (!IsSupportedKey(sanitized)) + { + throw new ArgumentException($"ObjectStore.SanitizeKey(): unable to sanitize key '{key}': object store {SupportedKeyRules}"); + } + return sanitized; + } + /// /// Returns the JSON deserialized object data for the specified path /// @@ -174,6 +245,28 @@ public T ReadJson(string path, Encoding encoding = null, JsonSerializerSettin return JsonConvert.DeserializeObject(json, settings); } + /// + /// Returns the JSON deserialized object data for the specified path as Python objects, + /// or the given default value if the key is not present + /// + /// The object path + /// Value to return when the key is not present. Defaults to None + /// The deserialized Python object, or if the key is not present + public PyObject ReadJson(string path, PyObject defaultValue = null) + { + if (!ContainsKey(path)) + { + return defaultValue; + } + var json = Read(path); + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyJson = json.ToPython(); + return _jsonDeserializeMethod.Invoke(pyJson); + } + } + /// /// Returns the XML deserialized object data for the specified path /// @@ -239,6 +332,19 @@ public bool SaveString(string path, string text, Encoding encoding = null) return _store.SaveBytes(path, encoding.GetBytes(text)); } + /// + /// Saves the object data in text format for the specified path. + /// Alias of + /// + /// The object path + /// The string object to be saved + /// The string encoding used, by default + /// True if the object was saved successfully + public bool SaveText(string path, string text, Encoding encoding = null) + { + return Save(path, text, encoding); + } + /// /// Saves the object data in JSON format for the specified path /// @@ -255,6 +361,47 @@ public bool SaveJson(string path, T obj, Encoding encoding = null, JsonSerial return SaveString(path, json, encoding); } + /// + /// Saves the given Python object in JSON format for the specified path, tolerating types the standard + /// json module rejects: datetime/date/time are stored in ISO-8601 format, Decimal and numpy scalars as + /// numbers and any other unsupported type (e.g. Symbol) as its string representation. Non-string + /// dictionary keys are stringified + /// + /// The object path + /// The Python object to be saved + /// True if the object was saved successfully + public bool SaveJson(string path, PyObject obj) + { + string json; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var result = _jsonSerializeMethod.Invoke(obj); + json = result.As(); + } + return Save(path, json); + } + + /// + /// Saves the given pandas DataFrame (or Series) for the specified path, in JSON format if the + /// path has a '.json' extension and as CSV otherwise + /// + /// The object path + /// The pandas DataFrame or Series to be saved + /// True if the object was saved successfully + public bool SaveDataframe(string path, PyObject dataFrame) + { + string serialized; + using (Py.GIL()) + { + EnsurePythonHelpers(); + using var pyPath = (path ?? string.Empty).ToPython(); + using var result = _dataFrameSerializeMethod.Invoke(dataFrame, pyPath); + serialized = result.As(); + } + return Save(path, serialized); + } + /// /// Saves the object data in XML format for the specified path /// @@ -298,5 +445,59 @@ public void Dispose() { _store.Dispose(); } + + /// + /// Lazily creates the Python helper methods backing the PyObject APIs. Must be called under the GIL + /// + private static void EnsurePythonHelpers() + { + if (_jsonSerializeMethod == null) + { + var module = PyModule.FromString("object_store_helpers", @"from json import dumps, loads +from datetime import datetime, date, time +from decimal import Decimal +try: + import numpy +except ImportError: + numpy = None + +def _default(value): + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return float(value) + if numpy is not None and isinstance(value, numpy.generic): + return value.item() + # Symbol and any other type json does not handle + return str(value) + +def _normalize(value): + if isinstance(value, dict): + # json requires str/int/float/bool/None keys: stringify anything else (Symbol, datetime, ...) + return { (k if isinstance(k, (str, int, float, bool)) or k is None else str(_default(k))): _normalize(v) + for k, v in value.items() } + if isinstance(value, (list, tuple, set)): + return [_normalize(v) for v in value] + return value + +def serialize(value): + return dumps(_normalize(value), default=_default) + +def deserialize(json_string): + return loads(json_string) + +def serialize_dataframe(value, key): + if not hasattr(value, 'to_csv'): + raise TypeError(f'save_dataframe() expects a pandas DataFrame or Series but received {type(value).__name__}') + if key.lower().endswith('.json'): + return value.to_json() + return value.to_csv() +"); + _jsonDeserializeMethod = module.GetAttr("deserialize"); + _dataFrameSerializeMethod = module.GetAttr("serialize_dataframe"); + // last so partial initialization is never observed + _jsonSerializeMethod = module.GetAttr("serialize"); + } + } } } \ No newline at end of file diff --git a/Engine/Storage/LocalObjectStore.cs b/Engine/Storage/LocalObjectStore.cs index 80690b8a7531..600667fcd10a 100644 --- a/Engine/Storage/LocalObjectStore.cs +++ b/Engine/Storage/LocalObjectStore.cs @@ -19,7 +19,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using QuantConnect.Configuration; using QuantConnect.Interfaces; @@ -112,7 +111,6 @@ private bool IsDirty } private Timer _persistenceTimer; - private Regex _pathRegex = new(@"^\.?[a-zA-Z0-9\\/_#\-\$= ]+\.?[a-zA-Z0-9]*$", RegexOptions.Compiled); private readonly ConcurrentDictionary _storage = new(); private readonly object _persistLock = new object(); @@ -292,7 +290,14 @@ public byte[] ReadBytes(string path) // Ensure we have the key, also takes care of null or improper access if (!ContainsKey(path)) { - throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. " + + // list the available keys so a near-miss (typo, wrong extension) is visible in the error itself + var keys = Keys; + var sample = string.Join(", ", keys.Take(10).Select(key => $"'{key}'")); + if (keys.Count > 10) + { + sample += ", ..."; + } + throw new KeyNotFoundException($"Object with path '{path}' was not found in the current project. Keys: [{sample}]. " + "Please use ObjectStore.ContainsKey(key) to check if an object exists before attempting to read." ); } @@ -326,14 +331,12 @@ public bool SaveBytes(string path, byte[] contents) { throw new InvalidOperationException($"LocalObjectStore.SaveBytes(): {NoWritePermissionsError}"); } - else if (!_pathRegex.IsMatch(path)) - { - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); - } - else if (path.Count(c => c == '/') > 100 || path.Count(c => c == '\\') > 100) + else if (!ObjectStore.IsSupportedKey(path)) { - // just in case - throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'"); + // state the rules in the error: keys are often built from user-facing names and only fail here, + // frequently in OnEndOfAlgorithm after an otherwise green run + throw new ArgumentException($"LocalObjectStore: path is not supported: '{path}'. Object store {ObjectStore.SupportedKeyRules}." + + $" Use ObjectStore.{nameof(ObjectStore.SanitizeKey)}(name) to convert an arbitrary name into a supported path."); } // after we check the regex diff --git a/Tests/Common/Storage/LocalObjectStoreTests.cs b/Tests/Common/Storage/LocalObjectStoreTests.cs index 6451c116bda3..624ce2a8dad2 100644 --- a/Tests/Common/Storage/LocalObjectStoreTests.cs +++ b/Tests/Common/Storage/LocalObjectStoreTests.cs @@ -541,6 +541,104 @@ public void ThrowsKeyNotFoundException_WhenObjectStoreDoesNotContainKey() Assert.IsTrue(error.Message.Contains("Please use ObjectStore.ContainsKey(key)")); } + [Test] + public void KeyNotFoundErrorListsTheAvailableKeys() + { + // dedicated store instance: its in-memory keys are listed first in the error + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + Assert.IsTrue(store.SaveBytes("available_key_a.txt", new byte[] { 1 })); + Assert.IsTrue(store.SaveBytes("available_key_b.txt", new byte[] { 1 })); + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains("Keys: ["), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_a.txt'"), error.Message); + Assert.IsTrue(error.Message.Contains("'available_key_b.txt'"), error.Message); + + store.Delete("available_key_a.txt"); + store.Delete("available_key_b.txt"); + } + + [Test] + public void KeyNotFoundErrorTruncatesTheAvailableKeysListAtTen() + { + using var store = new ObjectStore(new TestLocalObjectStore()); + store.Initialize(0, 0, "", new Controls() { PersistenceIntervalSeconds = -1 }, AlgorithmMode.Backtesting); + for (var i = 0; i < 11; i++) + { + Assert.IsTrue(store.SaveBytes($"truncation_key_{i}.txt", new byte[] { 1 })); + } + + var error = Assert.Throws(() => store.ReadBytes("missing.missing")); + + Assert.IsTrue(error.Message.Contains(", ...]"), error.Message); + + for (var i = 0; i < 11; i++) + { + store.Delete($"truncation_key_{i}.txt"); + } + } + + [TestCase("my_key", true)] + [TestCase("./my_key/nested\\file.csv", true)] + [TestCase("file with spaces #1-$=.txt", true)] + [TestCase("trade_log_ai_hardware_&_cloud.csv", false)] + [TestCase("file.tar.gz", false)] + [TestCase("file.name_v2", false)] + [TestCase("**abc**", false)] + [TestCase("", false)] + [TestCase(null, false)] + public void ValidatesKeys(string key, bool isSupported) + { + Assert.AreEqual(isSupported, ObjectStore.IsSupportedKey(key)); + } + + [Test] + public void UnsupportedKeyErrorStatesTheKeyRules() + { + var error = Assert.Throws( + () => _store.SaveBytes("trade_log_ai_hardware_&_cloud.csv", new byte[] { 1 })); + + Assert.IsTrue(error.Message.Contains("path is not supported: 'trade_log_ai_hardware_&_cloud.csv'"), error.Message); + Assert.IsTrue(error.Message.Contains(ObjectStore.SupportedKeyRules), error.Message); + Assert.IsTrue(error.Message.Contains("SanitizeKey"), error.Message); + } + + [TestCase("my_key/file.csv", "my_key/file.csv")] + [TestCase("trade_log_ai_hardware_&_cloud.csv", "trade_log_ai_hardware___cloud.csv")] + [TestCase("trade log: AI & Cloud.csv", "trade log_ AI _ Cloud.csv")] + [TestCase("file.tar.gz", "file_tar.gz")] + [TestCase("résumé.pdf", "r_sum_.pdf")] + [TestCase("a..", "a__")] + [TestCase("100%.json", "100_.json")] + public void SanitizeKeyReturnsAStorableKey(string key, string expectedSanitizedKey) + { + var sanitized = ObjectStore.SanitizeKey(key); + + Assert.AreEqual(expectedSanitizedKey, sanitized); + Assert.IsTrue(ObjectStore.IsSupportedKey(sanitized)); + Assert.IsTrue(_store.SaveBytes(sanitized, new byte[] { 1 })); + + _store.Delete(sanitized); + } + + [TestCase(null)] + [TestCase("")] + public void SanitizeKeyThrowsOnNullOrEmptyKey(string key) + { + Assert.Throws(() => ObjectStore.SanitizeKey(key)); + } + + [Test] + public void SaveTextIsAnAliasOfSave() + { + Assert.IsTrue(_store.SaveText("save_text_alias.txt", "abc")); + Assert.AreEqual("abc", _store.Read("save_text_alias.txt")); + + _store.Delete("save_text_alias.txt"); + } + [TestCase("my_key", "./LocalObjectStoreTests/my_key")] [TestCase("test/123", "./LocalObjectStoreTests/test/123")] [TestCase("**abc**", null)] @@ -1016,6 +1114,64 @@ def add_data(object_store): } } + [Test] + public void PythonSaveJsonReadJsonSaveDataframeAndSanitizeKey() + { + using (Py.GIL()) + { + var testModule = PyModule.FromString("TestObjectStoreErgonomics", + @" +from AlgorithmImports import * +from datetime import datetime +from decimal import Decimal + +def test(object_store): + symbol = Symbol.create('SPY', SecurityType.EQUITY, Market.USA) + + # save_json tolerates datetime/date/Decimal/Symbol values and non-string dictionary keys + object_store.save_json('ergonomics/log.json', { + 'time': datetime(2024, 1, 2, 3, 4, 5), + 'date': datetime(2024, 1, 1).date(), + 'qty': Decimal('1.5'), + 'symbol': symbol, + 'fills': [{symbol: 10}] + }) + data = object_store.read_json('ergonomics/log.json') + # symbols are stored as their string representation, which depends on the symbol cache state + assert data == { + 'time': '2024-01-02T03:04:05', + 'date': '2024-01-01', + 'qty': 1.5, + 'symbol': str(symbol), + 'fills': [{str(symbol): 10}] + }, data + + # read_json returns the given default when the key is missing + assert object_store.read_json('ergonomics/missing.json') is None + assert object_store.read_json('ergonomics/missing.json', {'a': 1}) == {'a': 1} + + # save_text is an alias of save + object_store.save_text('ergonomics/report.txt', 'hello') + assert object_store.read('ergonomics/report.txt') == 'hello' + + # save_dataframe stores a DataFrame as CSV + import pandas as pd + frame = pd.DataFrame({'a': [1, 2], 'b': [3.5, 4.5]}) + object_store.save_dataframe('ergonomics/frame.csv', frame) + assert 'a,b' in object_store.read('ergonomics/frame.csv') + + # sanitize_key is reachable from the instance + assert object_store.sanitize_key('trade_log_ai_hardware_&_cloud.csv') == 'trade_log_ai_hardware___cloud.csv' + + for key in ['ergonomics/log.json', 'ergonomics/report.txt', 'ergonomics/frame.csv']: + object_store.delete(key) +"); + + dynamic test = testModule.GetAttr("test"); + Assert.DoesNotThrow(() => test(_store)); + } + } + private static void DummyMachineLearning(string outputFile, string content) { try