diff --git a/CHANGELOG.md b/CHANGELOG.md index 3737b80e..38f8028f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## [Unreleased] + +### Fixed + +- Fix mutations that make a table render a `[header]` while a bare or dotted key that is not inside it still follows at the same level, so the header swallows that key when the output is parsed again. This includes assigning a table over a child of an out-of-order dotted-key table (`doc["a"]["b"] = {...}` over `a.b`/`a.c`/`a.d`), doing so inside a regular table or array-of-tables element, promoting children in multiple super tables, and adding a table to a super table (`a.b = 1` then `doc["a"]["new"] = {...}`). Rendering now keeps relative inline keys before absolute child headers at every scope, preserves enclosing table and array-of-tables prefixes, and moves leading comment and whitespace trivia with the key it documents. ([#556](https://github.com/python-poetry/tomlkit/issues/556)) + ## [0.15.1] - 2026-07-17 ### Changed diff --git a/tests/test_toml_document.py b/tests/test_toml_document.py index 7ba61f5e..5807c2af 100644 --- a/tests/test_toml_document.py +++ b/tests/test_toml_document.py @@ -1210,6 +1210,240 @@ def test_replace_value_with_table_keeps_following_dotted_sibling() -> None: assert parse(doc.as_string()) == {"c": {"d": 2}, "x": {}} +def test_replace_out_of_order_dotted_child_with_table_keeps_following_siblings() -> ( + None +): + # https://github.com/python-poetry/tomlkit/issues/556 + # ``a.b``/``a.c``/``a.d`` parse as out-of-order fragments of super table + # ``a`` (so ``doc["a"]`` is an OutOfOrderTableProxy). Replacing a child with + # a real table promotes that fragment to a ``[a.b]`` header, which must move + # past the following inline siblings instead of swallowing them. + content = """a.b = 1 +a.c = 2 +a.d = 3 +""" + doc = parse(content) + doc["a"]["b"] = {"x": 9} + assert ( + doc.as_string() + == """a.c = 2 +a.d = 3 + +[a.b] +x = 9 +""" + ) + assert parse(doc.as_string()) == {"a": {"c": 2, "d": 3, "b": {"x": 9}}} + assert parse(doc.as_string()).unwrap() == doc.unwrap() + + +def test_replace_out_of_order_dotted_child_keeps_following_other_group() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + # The promoted header must also clear a following dotted key of a different + # top-level group, not only same-group siblings. + content = """a.b = 1 +a.c = 2 +q.e = 3 +""" + doc = parse(content) + doc["a"]["b"] = {"x": 9} + assert ( + doc.as_string() + == """a.c = 2 +q.e = 3 + +[a.b] +x = 9 +""" + ) + assert parse(doc.as_string()) == {"a": {"c": 2, "b": {"x": 9}}, "q": {"e": 3}} + assert parse(doc.as_string()).unwrap() == doc.unwrap() + + +def test_replace_out_of_order_dotted_child_with_empty_table_keeps_sibling() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + content = """a.b = 1 +a.c = 2 +""" + doc = parse(content) + doc["a"]["b"] = {} + assert ( + doc.as_string() + == """a.c = 2 + +[a.b] +""" + ) + assert parse(doc.as_string()) == {"a": {"c": 2, "b": {}}} + + +def test_replace_out_of_order_dotted_child_with_aot_keeps_sibling() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + content = """a.b = 1 +a.c = 2 +""" + doc = parse(content) + arr = tomlkit.aot() + tbl = tomlkit.table() + tbl["x"] = 9 + arr.append(tbl) + doc["a"]["b"] = arr + assert ( + doc.as_string() + == """a.c = 2 + +[[a.b]] +x = 9 +""" + ) + assert parse(doc.as_string()) == {"a": {"c": 2, "b": [{"x": 9}]}} + + +def test_replace_last_out_of_order_dotted_child_does_not_move() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + # With nothing rendering inline after it, the promoted header stays in place. + content = """a.b = 1 +a.c = 2 +""" + doc = parse(content) + doc["a"]["c"] = {"x": 9} + assert ( + doc.as_string() + == """a.b = 1 + +[a.c] +x = 9 +""" + ) + assert parse(doc.as_string()) == {"a": {"b": 1, "c": {"x": 9}}} + + +def test_replace_nested_out_of_order_dotted_child_keeps_siblings() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + # The promotion can happen a level down: a.b.c / a.b.d / a.e parse as + # out-of-order fragments and doc["a"]["b"]["c"] = {...} promotes c to + # [a.b.c]. The captured siblings live in other top-level a fragments, so the + # fix has to normalise the document body, not just the container mutated. + content = """a.b.c = 1 +a.b.d = 2 +a.e = 3 +""" + doc = parse(content) + doc["a"]["b"]["c"] = {"x": 9} + assert ( + doc.as_string() + == """a.b.d = 2 +a.e = 3 + +[a.b.c] +x = 9 +""" + ) + assert parse(doc.as_string()) == {"a": {"b": {"c": {"x": 9}, "d": 2}, "e": 3}} + assert parse(doc.as_string()).unwrap() == doc.unwrap() + + +def test_replace_deeply_nested_out_of_order_dotted_child_keeps_siblings() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + # Same at one more level of dotting, to confirm the fix is depth-agnostic. + content = """a.b.c.d = 1 +a.b.c.e = 2 +a.f = 3 +""" + doc = parse(content) + doc["a"]["b"]["c"]["d"] = {"x": 9} + assert ( + doc.as_string() + == """a.b.c.e = 2 +a.f = 3 + +[a.b.c.d] +x = 9 +""" + ) + assert parse(doc.as_string()) == { + "a": {"b": {"c": {"d": {"x": 9}, "e": 2}}, "f": 3} + } + + +def test_add_table_to_super_table_keeps_following_top_level_value() -> None: + # https://github.com/python-poetry/tomlkit/issues/556 + # Adding a table to a super table makes it render a trailing [a.new] header + # even though it is not out of order; a following top-level value must move + # before that header instead of being swallowed by it. + content = """a.b = 1 +z = 2 +""" + doc = parse(content) + doc["a"]["new"] = {"x": 1} + assert ( + doc.as_string() + == """z = 2 +a.b = 1 + +[a.new] +x = 1 +""" + ) + assert parse(doc.as_string()) == {"z": 2, "a": {"b": 1, "new": {"x": 1}}} + assert parse(doc.as_string()).unwrap() == doc.unwrap() + + +def test_add_table_to_super_table_without_trailing_newline_keeps_separator() -> None: + doc = parse("a.b=1\nz=2") + doc["a"]["new"] = {"x": 1} + + output = doc.as_string() + + assert output == "z=2\na.b=1\n\n[a.new]\nx = 1\n" + assert parse(output).unwrap() == doc.unwrap() + + +def test_promoted_children_of_multiple_super_tables_preserve_roundtrip() -> None: + doc = parse("a.x=1\nq.x=2\nr=3\n") + doc["a"]["new"] = {"v": 1} + doc["q"]["new"] = {"v": 2} + + output = doc.as_string() + + assert output.index("r=3") < output.index("a.x=1") + assert output.index("a.x=1") < output.index("q.x=2") + assert output.index("q.x=2") < output.index("[a.new]") + assert output.index("[a.new]") < output.index("[q.new]") + assert parse(output).unwrap() == doc.unwrap() + + +def test_promoted_child_inside_table_keeps_parent_prefix() -> None: + doc = parse("[outer]\np.b = 1\np.c = 2\n") + doc["outer"]["p"]["b"] = {"x": 1} + + output = doc.as_string() + + assert "[outer.p.b]" in output + assert parse(output).unwrap() == doc.unwrap() + + +def test_promoted_child_inside_aot_keeps_parent_prefix_and_siblings() -> None: + doc = parse("[[p]]\na.b = 1\na.c = 2\nz = 3\n") + doc["p"][0]["a"]["b"] = {"x": 1} + + output = doc.as_string() + + assert "[p.a.b]" in output + assert parse(output).unwrap() == doc.unwrap() + + +def test_promoted_child_moves_leading_comment_with_inline_sibling() -> None: + doc = parse("a.b = 1\n# Documentation for a.c\na.c = 2\n[z]\nq = 3\n") + doc["a"]["b"] = {"x": 9} + + output = doc.as_string() + + assert output.index("# Documentation for a.c") < output.index("a.c = 2") + assert output.index("a.c = 2") < output.index("[a.b]") + assert parse(output).unwrap() == doc.unwrap() + + def test_replace_with_comment() -> None: content = 'a = "1"' doc = parse(content) diff --git a/tomlkit/container.py b/tomlkit/container.py index 8ff30d98..6b2105ca 100644 --- a/tomlkit/container.py +++ b/tomlkit/container.py @@ -188,6 +188,180 @@ def _renders_table_header(self, table: Table) -> bool: return True return False + @staticmethod + def _append_header(rendered: str, header: str) -> str: + if not header: + return rendered + + # Match the separator rule previously repeated in each rendering loop. + # A header's own trivia can already contain the required newline. + before_header = header.partition("[")[0] + if ( + rendered.strip(" ") + and not rendered.strip(" ").endswith("\n") + and "\n" not in before_header + ): + rendered += "\n" + return rendered + header + + @staticmethod + def _append_inline(rendered: str, inline: str) -> str: + if not inline: + return rendered + + if ( + rendered.strip(" ") + and not rendered.rstrip(" ").endswith(("\n", "\r")) + and not inline.lstrip(" ").startswith(("\n", "\r")) + ): + rendered += "\n" + return rendered + inline + + @staticmethod + def _join_key(prefix: str | None, key: Key) -> str: + rendered = key.as_string() + return f"{prefix}.{rendered}" if prefix is not None else rendered + + def _table_emits_own_header(self, key: Key, table: Table) -> bool: + return ( + not table.is_super_table() + or ( + any( + not isinstance(v, (Table, AoT, Whitespace, Null)) + for _, v in table.value.body + ) + and not key.is_dotted() + ) + or ( + any( + k is not None and k.is_dotted() + for k, v in table.value.body + if isinstance(v, Table) + ) + and not key.is_dotted() + ) + ) + + def _render_body_parts( + self, + *, + header_prefix: str | None = None, + inline_prefix: str | None = None, + ) -> tuple[str, str]: + """Render a container as stable inline and header partitions. + + TOML keys at one scope must precede child table headers. A mutated super + table can contribute to both partitions, so ordering whole body entries + is insufficient when two such tables are interleaved. Leading comments + and whitespace travel with the keyed entry they describe. + """ + + pure_inline = "" + mixed_inline = "" + headers = "" + pending_trivia = "" + + for key, item in self._body: + if key is None: + pending_trivia += self._render_simple_item(key, item) + continue + + item_inline, item_headers = self._render_item_parts( + key, + item, + header_prefix=header_prefix, + inline_prefix=inline_prefix, + ) + if item_inline and item_headers: + mixed_inline += pending_trivia + item_inline + pending_trivia = "" + elif item_inline: + pure_inline += pending_trivia + item_inline + pending_trivia = "" + elif item_headers: + item_headers = pending_trivia + item_headers + pending_trivia = "" + + if item_headers: + headers = self._append_header(headers, item_headers) + + if pending_trivia: + if headers: + headers += pending_trivia + else: + pure_inline += pending_trivia + + return self._append_inline(pure_inline, mixed_inline), headers + + def _render_item_parts( + self, + key: Key, + item: Item, + *, + header_prefix: str | None, + inline_prefix: str | None, + ) -> tuple[str, str]: + if isinstance(item, Table): + return self._render_table_parts( + key, + item, + header_prefix=header_prefix, + inline_prefix=inline_prefix, + ) + if isinstance(item, AoT): + return "", self._render_aot(key, item, prefix=header_prefix) + return self._render_simple_item(key, item, prefix=inline_prefix), "" + + def _render_table_parts( + self, + key: Key, + table: Table, + *, + header_prefix: str | None, + inline_prefix: str | None, + ) -> tuple[str, str]: + if table.display_name is not None: + full_key = table.display_name + else: + full_key = self._join_key(header_prefix, key) + + if not self._table_emits_own_header(key, table): + relative_key = self._join_key(inline_prefix, key) + inline, headers = table.value._render_body_parts( + header_prefix=full_key, + inline_prefix=relative_key, + ) + if table.trivia.indent == "\n": + if inline: + inline = table.trivia.indent + inline + else: + headers = table.trivia.indent + headers + return inline, headers + + open_, close = "[", "]" + if table.is_aot_element(): + open_, close = "[[", "]]" + + newline_in_table_trivia = ( + "\n" if "\n" not in table.trivia.trail and len(table.value) > 0 else "" + ) + header = ( + f"{table.trivia.indent}" + f"{open_}" + f"{decode(full_key)}" + f"{close}" + f"{table.trivia.comment_ws}" + f"{decode(table.trivia.comment)}" + f"{table.trivia.trail}" + f"{newline_in_table_trivia}" + ) + body_inline, body_headers = table.value._render_body_parts( + header_prefix=full_key, + inline_prefix=None, + ) + body = self._append_header(body_inline, body_headers) + return "", header + body + def _validate_out_of_order_table(self, key: Key | None = None) -> None: if key is None: for k in list(self._out_of_order_keys): @@ -632,113 +806,17 @@ def last_item(self) -> Item | None: def as_string(self) -> str: """Render as TOML string.""" - s = "" - for k, v in self._body: - if k is not None: - if isinstance(v, Table): - if ( - s.strip(" ") - and not s.strip(" ").endswith("\n") - and "\n" not in v.trivia.indent - ): - s += "\n" - s += self._render_table(k, v) - elif isinstance(v, AoT): - if ( - s.strip(" ") - and not s.strip(" ").endswith("\n") - and "\n" not in v.trivia.indent - ): - s += "\n" - s += self._render_aot(k, v) - else: - s += self._render_simple_item(k, v) - else: - s += self._render_simple_item(k, v) - - return s + inline, headers = self._render_body_parts() + return self._append_header(inline, headers) def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> str: - cur = "" - - if table.display_name is not None: - _key = table.display_name - else: - _key = key.as_string() - - if prefix is not None: - _key = prefix + "." + _key - - if ( - not table.is_super_table() - or ( - any( - not isinstance(v, (Table, AoT, Whitespace, Null)) - for _, v in table.value.body - ) - and not key.is_dotted() - ) - or ( - any( - k is not None and k.is_dotted() - for k, v in table.value.body - if isinstance(v, Table) - ) - and not key.is_dotted() - ) - ): - open_, close = "[", "]" - if table.is_aot_element(): - open_, close = "[[", "]]" - - newline_in_table_trivia = ( - "\n" if "\n" not in table.trivia.trail and len(table.value) > 0 else "" - ) - cur += ( - f"{table.trivia.indent}" - f"{open_}" - f"{decode(_key)}" - f"{close}" - f"{table.trivia.comment_ws}" - f"{decode(table.trivia.comment)}" - f"{table.trivia.trail}" - f"{newline_in_table_trivia}" - ) - elif table.trivia.indent == "\n": - cur += table.trivia.indent - - for k, v in table.value.body: - if isinstance(v, Table): - if ( - cur.strip(" ") - and not cur.strip(" ").endswith("\n") - and "\n" not in v.trivia.indent - ): - cur += "\n" - assert k is not None - if v.is_super_table(): - if k.is_dotted() and not key.is_dotted(): - # Dotted key inside table - cur += self._render_table(k, v) - else: - cur += self._render_table(k, v, prefix=_key) - else: - cur += self._render_table(k, v, prefix=_key) - elif isinstance(v, AoT): - if ( - cur.strip(" ") - and not cur.strip(" ").endswith("\n") - and "\n" not in v.trivia.indent - ): - cur += "\n" - assert k is not None - cur += self._render_aot(k, v, prefix=_key) - else: - cur += self._render_simple_item( - k, v, prefix=_key if key.is_dotted() else None - ) - - return cur + inline, headers = self._render_table_parts( + key, + table, + header_prefix=prefix, + inline_prefix=None, + ) + return self._append_header(inline, headers) def _render_aot(self, key: Key, aot: AoT, prefix: str | None = None) -> str: _key = key.as_string() @@ -767,22 +845,11 @@ def _render_aot_table(self, table: Table, prefix: str | None = None) -> str: f"{table.trivia.trail}" ) - for k, v in table.value.body: - if isinstance(v, Table): - assert k is not None - if v.is_super_table(): - if k.is_dotted(): - # Dotted key inside table - cur += self._render_table(k, v) - else: - cur += self._render_table(k, v, prefix=_key) - else: - cur += self._render_table(k, v, prefix=_key) - elif isinstance(v, AoT): - assert k is not None - cur += self._render_aot(k, v, prefix=_key) - else: - cur += self._render_simple_item(k, v) + body_inline, body_headers = table.value._render_body_parts( + header_prefix=_key, + inline_prefix=None, + ) + cur += self._append_header(body_inline, body_headers) return cur