From 738b0b06f1644a824ea5c4592ff569ed6be218ea Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:09:00 -0400 Subject: [PATCH] Preserve relative paths in copied root pages --- yardang/conf.py.j2 | 12 +- yardang/markdown.py | 231 +++++++++++++++++++++++++++++++++ yardang/tests/test_markdown.py | 166 +++++++++++++++++++++++ 3 files changed, 406 insertions(+), 3 deletions(-) create mode 100644 yardang/markdown.py create mode 100644 yardang/tests/test_markdown.py diff --git a/yardang/conf.py.j2 b/yardang/conf.py.j2 index 1654791..60780cc 100644 --- a/yardang/conf.py.j2 +++ b/yardang/conf.py.j2 @@ -16,9 +16,12 @@ import os import os.path -from packaging.version import Version from pathlib import Path +from packaging.version import Version + +from yardang.markdown import copy_relative_html_assets, rebase_relative_references + ######################## # COMMON CONFIGURATION # ######################## @@ -295,11 +298,14 @@ toctree_root = f"""```{toctree_base} ###################### # INTERNAL FUNCTIONS # ###################### -def run_copyreadme(_): +def run_copyreadme(app): out = Path("{{source_dir}}") / "index.md" readme = Path(root) if (root != "" and root != "None") else Path("{{source_dir}}") / "README.md" if "index.md" not in pages: - out.write_text(toctree_root + "\n" + readme.read_text()) + content = readme.read_text() + copy_relative_html_assets(content, source=readme, destination=out, output_dir=Path(app.outdir)) + content = rebase_relative_references(content, source=readme, destination=out) + out.write_text(toctree_root + "\n" + content) def run_copycname(_): out = Path("{{source_dir}}") / "docs" / "html" / "CNAME" diff --git a/yardang/markdown.py b/yardang/markdown.py new file mode 100644 index 0000000..530d7dd --- /dev/null +++ b/yardang/markdown.py @@ -0,0 +1,231 @@ +import posixpath +import re +import shutil +from pathlib import Path +from urllib.parse import unquote, urlsplit, urlunsplit + +_MARKDOWN_LINK = re.compile( + r"(?P!?\[[^\]\n]*\]\(\s*)" + r"(?:<(?P[^>\n]+)>|(?P[^<>\s)]+))" + r"(?P(?:\s+(?:\"[^\"\n]*\"|'[^'\n]*'|\([^\)\n]*\)))?\s*\))" +) +_REFERENCE_DEFINITION = re.compile( + r"^(?P[ \t]{0,3}\[[^\]\n]+\]:[ \t]*)" + r"(?:<(?P[^>\n]+)>|(?P[^<>\s]+))" + r"(?P(?:[ \t]+(?:\"[^\"\n]*\"|'[^'\n]*'|\([^\)\n]*\)))?[ \t]*)$", + re.MULTILINE, +) +_HTML_ATTRIBUTE = re.compile( + r"(?P(?[\"'])(?P.*?)(?P=quote)|(?P[^\s\"'=<>`]+))", + re.IGNORECASE, +) +_HTML_SRCSET = re.compile( + r"(?P(?[\"'])(?P.*?)(?P=quote)|(?P[^\s\"'=<>`]+))", + re.IGNORECASE, +) +_FENCE_OPEN = re.compile(r"^ {0,3}(?P`{3,}|~{3,})") +_PROTECTED_INLINE = re.compile(r"|(?`+)(?!`).*?(? str: + parsed = urlsplit(url) + if parsed.scheme or parsed.netloc or not parsed.path or parsed.path.startswith("/"): + return url + + source_path = posixpath.normpath(posixpath.join(source_dir, parsed.path)) + rebased_path = posixpath.relpath(source_path, destination_dir) + return urlunsplit(("", "", rebased_path, parsed.query, parsed.fragment)) + + +def _replace_url_group(match: re.Match, source_dir: str, destination_dir: str) -> str: + group = "angle_url" if match.groupdict().get("angle_url") is not None else "url" + url = _rebase_url(match.group(group), source_dir, destination_dir) + value = match.group(0) + start = match.start(group) - match.start() + end = match.end(group) - match.start() + return f"{value[:start]}{url}{value[end:]}" + + +def _fenced_segments(content: str): + lines = content.splitlines(keepends=True) + active_lines = [] + protected_lines = [] + fence_character = "" + fence_length = 0 + + for line in lines: + if protected_lines: + protected_lines.append(line) + closing = line.lstrip(" ") + if len(line) - len(closing) <= 3: + closing = closing.rstrip("\r\n") + delimiter_length = len(closing) - len(closing.lstrip(fence_character)) + if delimiter_length >= fence_length and not closing[delimiter_length:].strip(): + yield "".join(protected_lines), False + protected_lines = [] + continue + + opening = _FENCE_OPEN.match(line) + if opening: + if active_lines: + yield "".join(active_lines), True + active_lines = [] + fence = opening.group("fence") + fence_character = fence[0] + fence_length = len(fence) + protected_lines = [line] + else: + active_lines.append(line) + + if protected_lines: + yield "".join(protected_lines), False + if active_lines: + yield "".join(active_lines), True + + +def _content_segments(content: str): + for segment, active in _fenced_segments(content): + if not active: + yield segment, False + continue + + position = 0 + for match in _PROTECTED_INLINE.finditer(segment): + if match.start() > position: + yield segment[position : match.start()], True + yield match.group(0), False + position = match.end() + if position < len(segment): + yield segment[position:], True + + +def _transform_active_content(content: str, transform) -> str: + return "".join(transform(segment) if active else segment for segment, active in _content_segments(content)) + + +def _rebase_markdown_links(content: str, source_dir: str, destination_dir: str) -> str: + def replace(match: re.Match) -> str: + return _replace_url_group(match, source_dir, destination_dir) + + return _transform_active_content(content, lambda segment: _MARKDOWN_LINK.sub(replace, segment)) + + +def _rebase_reference_definitions(content: str, source_dir: str, destination_dir: str) -> str: + def replace(match: re.Match) -> str: + return _replace_url_group(match, source_dir, destination_dir) + + return _transform_active_content(content, lambda segment: _REFERENCE_DEFINITION.sub(replace, segment)) + + +def _srcset_url_spans(value: str): + position = 0 + while position < len(value): + while position < len(value) and (value[position].isspace() or value[position] == ","): + position += 1 + start = position + while position < len(value) and not value[position].isspace(): + position += 1 + end = position + if start == end: + break + + while end > start and value[end - 1] == ",": + end -= 1 + has_separator = end < position + if end > start: + yield start, end + if has_separator: + continue + + parentheses = 0 + while position < len(value): + character = value[position] + if character == "(": + parentheses += 1 + elif character == ")" and parentheses: + parentheses -= 1 + elif character == "," and not parentheses: + position += 1 + break + position += 1 + + +def _rebase_srcset(value: str, source_dir: str, destination_dir: str) -> str: + result = [] + position = 0 + for start, end in _srcset_url_spans(value): + result.append(value[position:start]) + result.append(_rebase_url(value[start:end], source_dir, destination_dir)) + position = end + result.append(value[position:]) + return "".join(result) + + +def _rebase_html_attributes(content: str, source_dir: str, destination_dir: str) -> str: + def replace_url(match: re.Match) -> str: + group = "quoted_url" if match.group("quoted_url") is not None else "unquoted_url" + url = _rebase_url(match.group(group), source_dir, destination_dir) + value = match.group(0) + start = match.start(group) - match.start() + end = match.end(group) - match.start() + return f"{value[:start]}{url}{value[end:]}" + + def replace_srcset(match: re.Match) -> str: + group = "quoted_value" if match.group("quoted_value") is not None else "unquoted_value" + srcset = _rebase_srcset(match.group(group), source_dir, destination_dir) + value = match.group(0) + start = match.start(group) - match.start() + end = match.end(group) - match.start() + return f"{value[:start]}{srcset}{value[end:]}" + + def transform(segment: str) -> str: + segment = _HTML_ATTRIBUTE.sub(replace_url, segment) + return _HTML_SRCSET.sub(replace_srcset, segment) + + return _transform_active_content(content, transform) + + +def _relative_html_urls(content: str): + for segment, active in _content_segments(content): + if not active: + continue + for match in _HTML_ATTRIBUTE.finditer(segment): + group = "quoted_url" if match.group("quoted_url") is not None else "unquoted_url" + yield match.group(group) + for match in _HTML_SRCSET.finditer(segment): + group = "quoted_value" if match.group("quoted_value") is not None else "unquoted_value" + value = match.group(group) + for start, end in _srcset_url_spans(value): + yield value[start:end] + + +def copy_relative_html_assets(content: str, source: Path, destination: Path, output_dir: Path) -> None: + """Copy local files referenced by raw HTML into the generated site.""" + source_dir = source.parent.as_posix() + destination_dir = destination.parent.as_posix() + output_root = output_dir.resolve() + + for url in _relative_html_urls(content): + parsed = urlsplit(url) + if parsed.scheme or parsed.netloc or not parsed.path or parsed.path.startswith("/"): + continue + + source_path = (source.parent / unquote(parsed.path)).resolve() + rebased = urlsplit(_rebase_url(url, source_dir, destination_dir)) + output_path = (output_root / unquote(rebased.path)).resolve() + if not source_path.is_file() or not output_path.is_relative_to(output_root): + continue + + output_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source_path, output_path) + + +def rebase_relative_references(content: str, source: Path, destination: Path) -> str: + """Rebase relative Markdown and HTML references after moving content.""" + source_dir = source.parent.as_posix() + destination_dir = destination.parent.as_posix() + content = _rebase_markdown_links(content, source_dir, destination_dir) + content = _rebase_reference_definitions(content, source_dir, destination_dir) + return _rebase_html_attributes(content, source_dir, destination_dir) diff --git a/yardang/tests/test_markdown.py b/yardang/tests/test_markdown.py new file mode 100644 index 0000000..4823104 --- /dev/null +++ b/yardang/tests/test_markdown.py @@ -0,0 +1,166 @@ +import importlib.util +import os +from pathlib import Path +from types import SimpleNamespace + +from yardang.build import generate_docs_configuration +from yardang.markdown import rebase_relative_references + + +def test_rebases_markdown_and_html_references(): + content = """[Guide](./guide.md#start) +![Diagram](../img/diagram.svg?raw=1) +[reference]: ../files/example.csv "Example" +Changes + + +""" + + result = rebase_relative_references( + content, + source=Path("docs/src/home.md"), + destination=Path("index.md"), + ) + + assert "[Guide](docs/src/guide.md#start)" in result + assert "![Diagram](docs/img/diagram.svg?raw=1)" in result + assert '[reference]: docs/files/example.csv "Example"' in result + assert 'Changes' in result + assert "" in result + assert 'srcset="docs/img/small.svg 1x, docs/img/large.svg 2x"' in result + + +def test_leaves_non_relative_references_unchanged(): + content = """[Web](https://example.com/docs) +[Protocol relative](//cdn.example.com/image.svg) +[Root relative](/assets/image.svg) +[Anchor](#section) +[Email](mailto:docs@example.com) + +""" + + assert ( + rebase_relative_references( + content, + source=Path("docs/src/home.md"), + destination=Path("index.md"), + ) + == content + ) + + +def test_rebases_mixed_data_and_local_srcset_candidates(): + content = """ + +""" + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert 'srcset="data:image/svg+xml,%3Csvg%3E 1x, docs/img/large.svg 2x"' in result + assert 'srcset="docs/img/small.svg 1x, data:image/svg+xml,%3Csvg%3E 2x"' in result + + +def test_rebases_descriptorless_mixed_srcset_candidates(): + content = """ + +""" + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert 'srcset="data:image/svg+xml,%3Csvg%3E, docs/img/large.svg 2x"' in result + assert 'srcset="docs/img/small.svg, data:image/svg+xml,%3Csvg%3E 2x"' in result + + +def test_leaves_code_examples_and_comments_unchanged(): + content = """`` +```html + +``` + + +""" + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert '``' in result + assert '' in result + assert '' in result + assert '' in result + + +def test_rebases_content_after_longer_closing_fences(): + content = ( + '```html\n\n````' + ' \n\n~~~html\n' + '\n~~~~\n\n' + ) + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert content.splitlines()[1] in result + assert content.splitlines()[5] in result + assert '' in result + assert '' in result + + +def test_rebases_angle_delimited_markdown_destinations(): + content = """[Guide]() +[guide]: "Guide" +""" + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert "[Guide]()" in result + assert '[guide]: "Guide"' in result + + +def test_rebases_exact_unquoted_html_attributes_only(): + content = """ +Guide + + +""" + + result = rebase_relative_references(content, Path("docs/src/home.md"), Path("index.md")) + + assert "" in result + assert "Guide" in result + assert '' in result + assert '' in result + + +def test_generated_configuration_rebases_root_references(tmp_path): + (tmp_path / "docs" / "src").mkdir(parents=True) + (tmp_path / "docs" / "img").mkdir() + (tmp_path / "docs" / "img" / "diagram.svg").write_text("") + (tmp_path / "docs" / "img" / "diagram-dark.svg").write_text("dark") + (tmp_path / "docs" / "img" / "literal.svg").write_text("literal") + root_content = ( + '[Guide](./guide.md)\n\n' + '\n``\n' + ) + root_path = tmp_path / "docs" / "src" / "home.md" + root_path.write_text(root_content) + (tmp_path / "pyproject.toml").write_text('[project]\nname = "test-project"\nversion = "1.0.0"\n\n[tool.yardang]\nroot = "docs/src/home.md"\n') + + original_cwd = Path.cwd() + try: + os.chdir(tmp_path) + with generate_docs_configuration() as conf_dir: + spec = importlib.util.spec_from_file_location("yardang_test_conf", Path(conf_dir) / "conf.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.run_copyreadme(SimpleNamespace(outdir=tmp_path / "docs" / "html")) + finally: + os.chdir(original_cwd) + + generated = (tmp_path / "index.md").read_text() + assert "[Guide](docs/src/guide.md)" in generated + assert '' in generated + assert '' in generated + assert (tmp_path / "docs" / "html" / "docs" / "img" / "diagram.svg").read_text() == "" + assert (tmp_path / "docs" / "html" / "docs" / "img" / "diagram-dark.svg").read_text() == "dark" + assert not (tmp_path / "docs" / "html" / "docs" / "img" / "literal.svg").exists() + assert root_path.read_text() == root_content