cpdocs API documentation
Project documentation Docs

File: cpdocs/build.py

1# SPDX-FileCopyrightText: 2026 cpdocs developers2# SPDX-License-Identifier: Apache-2.03"""Build operations: describe, resolve variants, extract, merge and render.4 5Output layout below ``<output>/cpdocs``::6 7    current/            API site of the working tree (``cpdocs build``)8    versions/           versioned API site (``cpdocs versions``), one directory per ref9    work/<ref>/         requests, manifests and build directories handed to the build system10"""11 12from __future__ import annotations13 14import dataclasses15import fnmatch16import json17import os18import shutil19import subprocess20import sys21import tempfile22from importlib import resources23from pathlib import Path, PurePosixPath24 25from .clang_backend import extract_clang_graph26from .config import CpdocsConfig, load_config27from .contract import (28    ContractError,29    Description,30    Request,31    UnitManifest,32    VariantManifest,33    check_variant,34    read_description,35    read_variant_manifest,36)37from .html_renderer import render_html_site38from .model import ApiGraph, merge_graphs39from .python_backend import build_python_graph40from .rust_backend import build_rust_graph41from .versioning import SelectedRef, latest_tag, select_refs42 43_PLACEHOLDERS = ("source", "build", "request", "manifest", "python", "cpdocs.cmake")44_CONTAINER_KINDS = {"namespace", "package", "module"}45 46 47@dataclasses.dataclass(frozen=True)48class Layout:49    """Directories below ``<output>/cpdocs``."""50 51    root: Path52 53    @classmethod54    def for_output(cls, output: Path) -> Layout:55        return cls(root=(output / "cpdocs").resolve())56 57    @property58    def current(self) -> Path:59        return self.root / "current"60 61    @property62    def versions(self) -> Path:63        return self.root / "versions"64 65    def work(self, ref: str) -> Path:66        return self.root / "work" / ref.replace("/", "__")67 68 69def cmake_module() -> Path:70    """Path of the CMake adapter shipped with cpdocs."""71 72    return Path(str(resources.files("cpdocs") / "share" / "cmake" / "cpdocs.cmake"))73 74 75# --------------------------------------------------------------------------------------------------76# Talking to the build system77# --------------------------------------------------------------------------------------------------78 79 80def _expand(argument: str, request: Request, request_file: Path) -> str:81    values = {82        "source": str(request.source),83        "build": str(request.build),84        "request": str(request_file),85        "manifest": str(request.manifest),86        "python": sys.executable,87        "cpdocs.cmake": str(cmake_module()),88    }89    for name in _PLACEHOLDERS:90        argument = argument.replace("{" + name + "}", values[name])91    return argument92 93 94def _run_commands(95    commands: tuple[tuple[str, ...], ...], *, request: Request, request_file: Path, log: Path96) -> None:97    environment = dict(os.environ)98    environment["CPDOCS_REQUEST"] = str(request_file)99    environment["CPDOCS_MANIFEST"] = str(request.manifest)100    with log.open("w", encoding="utf-8") as stream:101        for command in commands:102            argv = [_expand(argument, request, request_file) for argument in command]103            stream.write("$ " + " ".join(argv) + "\n")104            stream.flush()105            try:106                result = subprocess.run(107                    argv,108                    cwd=request.source,109                    env=environment,110                    check=False,111                    text=True,112                    stdout=subprocess.PIPE,113                    stderr=subprocess.STDOUT,114                )115            except FileNotFoundError as error:116                raise RuntimeError(117                    f"cannot run build-system command {argv[0]!r}: {error}"118                ) from error119            stream.write(result.stdout)120            if result.returncode != 0:121                raise RuntimeError(122                    f"build-system command failed ({request.phase}"123                    + (f", variant {request.variant}" if request.variant else "")124                    + "):\n  "125                    + " ".join(argv)126                    + "\n\n"127                    + result.stdout.rstrip()128                )129 130 131def describe(132    commands: tuple[tuple[str, ...], ...], *, source: Path, work: Path, ref: str, ref_kind: str133) -> Description:134    """Run the describe phase and read the describe manifest."""135 136    directory = work / "describe"137    shutil.rmtree(directory, ignore_errors=True)138    directory.mkdir(parents=True)139    request = Request(140        phase="describe",141        ref=ref,142        ref_kind=ref_kind,143        source=source.resolve(),144        build=directory / "build",145        manifest=directory / "manifest.json",146        variant=None,147    )148    request_file = directory / "request.json"149    request.write(request_file)150    _run_commands(commands, request=request, request_file=request_file, log=directory / "log.txt")151    return read_description(request.manifest)152 153 154def resolve_variant(155    description: Description, name: str, *, source: Path, work: Path, ref: str, ref_kind: str156) -> VariantManifest:157    """Run one variant's commands and read its variant manifest."""158 159    directory = work / "variants" / name160    shutil.rmtree(directory, ignore_errors=True)161    directory.mkdir(parents=True)162    request = Request(163        phase="variant",164        ref=ref,165        ref_kind=ref_kind,166        source=source.resolve(),167        build=directory / "build",168        manifest=directory / "manifest.json",169        variant=name,170    )171    request_file = directory / "request.json"172    request.write(request_file)173    _run_commands(174        description.variant(name).commands,175        request=request,176        request_file=request_file,177        log=directory / "log.txt",178    )179    manifest = read_variant_manifest(request.manifest, source=request.source)180    check_variant(manifest, description=description, requested=name, path=request.manifest)181    return manifest182 183 184# --------------------------------------------------------------------------------------------------185# Extraction186# --------------------------------------------------------------------------------------------------187 188 189def _clang() -> str:190    for candidate in (os.environ.get("CPDOCS_CLANG"), "clang++", "clang"):191        if candidate and shutil.which(candidate):192            return candidate193    raise RuntimeError(194        "cpdocs needs clang to document C and C++ units; put clang++ on PATH "195        "(or name it in CPDOCS_CLANG)"196    )197 198 199def _generated_sources(unit: UnitManifest, graph: ApiGraph) -> list[str]:200    values: list[str] = []201    for root in unit.api:202        if not root.generated:203            continue204        for path in root.path.rglob("*"):205            if path.is_file():206                logical = path.relative_to(root.path).as_posix()207                if logical in graph.sources:208                    values.append(logical)209    return values210 211 212def extract_unit(213    unit: UnitManifest,214    *,215    project: str,216    version: str,217    project_root: Path,218    variant: str,219    work: Path,220    config: CpdocsConfig,221) -> ApiGraph:222    if unit.language in {"c", "cpp"}:223        assert unit.parse is not None224        graph = extract_clang_graph(225            project=project,226            version=version,227            project_root=project_root,228            language=unit.language,229            api_roots=[root.path for root in unit.api],230            parse=unit.parse,231            variant=variant,232            clang_executable=_clang(),233            work_directory=work / "clang" / unit.id,234        )235        generated = _generated_sources(unit, graph)236        if generated:237            graph.metadata["generated_sources"] = generated238        return graph239    if unit.language == "python":240        assert unit.package is not None241        return build_python_graph(242            project_root=project_root,243            package_root=unit.api[0].path,244            package=unit.package,245            version=version,246            include_private=not config.hide_private_python,247        )248    if unit.language == "rust":249        assert unit.rustdoc_json is not None250        return build_rust_graph(251            rustdoc_json=unit.rustdoc_json,252            project=project,253            version=version,254            project_root=project_root,255        )256    raise ContractError(f"unsupported unit language {unit.language!r}")257 258 259def _merge_metadata(target: dict[str, object], source: dict[str, object]) -> None:260    for key, value in source.items():261        if isinstance(value, list):262            existing = target.setdefault(key, [])263            assert isinstance(existing, list)264            existing.extend(item for item in value if item not in existing)265        else:266            target.setdefault(key, value)267 268 269def _combine(graphs: list[ApiGraph], *, project: str, version: str, language: str) -> ApiGraph:270    """Combine the graphs of several units extracted for the same variant."""271 272    combined = ApiGraph(project=project, version=version, language=language)273    for graph in graphs:274        for entity in graph.entities.values():275            combined.add(entity)276        for path, content in graph.sources.items():277            combined.sources.setdefault(path, content)278        _merge_metadata(combined.metadata, graph.metadata)279    combined.rebuild_children()280    return combined281 282 283def _apply_visibility(graph: ApiGraph, config: CpdocsConfig) -> None:284    patterns = config.hide_cpp_namespaces285    if not patterns:286        return287    hidden = {288        entity.id289        for entity in graph.entities.values()290        if entity.language in {"c", "cpp"}291        and entity.kind == "namespace"292        and any(fnmatch.fnmatchcase(entity.qualified_name, pattern) for pattern in patterns)293    }294    changed = True295    while changed:296        changed = False297        for entity in graph.entities.values():298            if entity.id not in hidden and entity.parent in hidden:299                hidden.add(entity.id)300                changed = True301    for identifier in hidden:302        del graph.entities[identifier]303    graph.rebuild_children()304 305 306def _check_undocumented(graph: ApiGraph, level: str) -> None:307    if level == "ignore":308        return309    missing = sorted(310        entity.qualified_name311        for entity in graph.entities.values()312        if entity.kind not in _CONTAINER_KINDS and not entity.documentation.strip()313    )314    if not missing:315        return316    message = f"{len(missing)} public API entities have no documentation:\n" + "\n".join(317        f"  - {name}" for name in missing318    )319    if level == "fail":320        raise RuntimeError(message)321    print(f"cpdocs: warning: {message}", file=sys.stderr)322 323 324def build_graph(325    config: CpdocsConfig,326    *,327    source: Path,328    work: Path,329    ref: str,330    ref_kind: str,331    version: str,332    variants: str,333) -> ApiGraph:334    """Describe, resolve the selected variants, and merge them into one API graph."""335 336    description = describe(config.describe, source=source, work=work, ref=ref, ref_kind=ref_kind)337    selected = (338        [variant.name for variant in description.variants]339        if variants == "all"340        else [description.primary.name]341    )342    languages = sorted({unit.language for unit in description.units})343    language = languages[0] if len(languages) == 1 else "mixed"344 345    per_variant: list[tuple[str, ApiGraph]] = []346    metadata: dict[str, object] = {}347    for name in selected:348        manifest = resolve_variant(349            description, name, source=source, work=work, ref=ref, ref_kind=ref_kind350        )351        unit_graphs = [352            extract_unit(353                unit,354                project=description.project,355                version=version,356                project_root=source,357                variant=name,358                work=work / "variants" / name,359                config=config,360            )361            for unit in manifest.units362        ]363        graph = _combine(364            unit_graphs, project=description.project, version=version, language=language365        )366        graph.variants[name] = {367            "primary": description.variant(name).primary,368            "features": {axis: list(values) for axis, values in manifest.features.items()},369        }370        _merge_metadata(metadata, graph.metadata)371        per_variant.append((name, graph))372 373    merged = merge_graphs(374        per_variant, project=description.project, version=version, language=language375    )376    metadata["features"] = {axis.name: list(axis.values) for axis in description.features}377    metadata["units"] = [{"id": unit.id, "language": unit.language} for unit in description.units]378    metadata["generator"] = description.generator379    metadata["project_version"] = description.version380    merged.metadata = metadata381 382    _apply_visibility(merged, config)383    _check_undocumented(merged, config.check_undocumented)384    return merged385 386 387# --------------------------------------------------------------------------------------------------388# Rendering389# --------------------------------------------------------------------------------------------------390 391 392def _graph_json(graph: ApiGraph) -> str:393    return json.dumps(dataclasses.asdict(graph), indent=2, sort_keys=True, default=str) + "\n"394 395 396def _project_docs_url(graph: ApiGraph, configured: str, *, root_depth: int) -> str:397    if configured != "auto":398        return configured399    version_depth = max(1, len(PurePosixPath(graph.version).parts))400    return "../" * (root_depth + version_depth)401 402 403def _render(404    graph: ApiGraph, config: CpdocsConfig, *, output: Path, work: Path, root_depth: int405) -> None:406    work.mkdir(parents=True, exist_ok=True)407    render_html_site(408        graph,409        output_directory=output,410        project_docs_url=_project_docs_url(graph, config.project_docs, root_depth=root_depth),411        copyright_text=config.copyright or f"Copyright © {graph.project} developers",412        introduction_file=config.index,413    )414    (work / "api-graph.json").write_text(_graph_json(graph), encoding="utf-8")415 416 417def build_current(418    config: CpdocsConfig,419    *,420    output: Path,421    version: str = "main",422    variants: str = "all",423    project_docs_root_depth: int = 2,424) -> ApiGraph:425    """Document the working tree into ``<output>/cpdocs/current``."""426 427    layout = Layout.for_output(output)428    work = layout.work("current")429    graph = build_graph(430        config,431        source=config.project_root,432        work=work,433        ref=version,434        ref_kind="worktree",435        version=version,436        variants=variants,437    )438    _render(439        graph,440        config,441        output=layout.current,442        work=work / "render",443        root_depth=project_docs_root_depth,444    )445    return graph446 447 448def _html_pages(root: Path) -> list[str]:449    if not root.is_dir():450        return []451    return sorted(path.relative_to(root).as_posix() for path in root.rglob("*.html"))452 453 454def _redirect_page(target: str) -> str:455    return (456        '<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'457        f'<meta http-equiv="refresh" content="0; url={target}">'458        f'<link rel="canonical" href="{target}"></head>'459        f'<body><a href="{target}">{target}</a></body></html>\n'460    )461 462 463def _write_versions(464    output: Path, built: list[SelectedRef], *, default: str, aliases: dict[str, str]465) -> None:466    names = [ref.name for ref in built]467    newest = latest_tag([ref.name for ref in built if ref.kind == "tag"])468 469    def target(value: str) -> str | None:470        if value == "latest-tag":471            return newest472        return value if value in names else None473 474    default_name = target(default) or (names[0] if names else "main")475    resolved_aliases: dict[str, str] = {}476    for alias, value in aliases.items():477        resolved = target(value)478        if resolved is None or alias in names:479            continue480        resolved_aliases[alias] = resolved481        directory = output / alias482        directory.mkdir(parents=True, exist_ok=True)483        depth = len(PurePosixPath(alias).parts)484        (directory / "index.html").write_text(485            _redirect_page("../" * depth + resolved + "/"), encoding="utf-8"486        )487 488    metadata = {489        "default": default_name,490        "aliases": resolved_aliases,491        "versions": [492            {493                "name": ref.name,494                "kind": ref.kind,495                "url": f"{ref.name}/",496                "pages": _html_pages(output / ref.name),497            }498            for ref in built499            if (output / ref.name / "index.html").is_file()500        ],501    }502    output.mkdir(parents=True, exist_ok=True)503    (output / "versions.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")504    (output / "index.html").write_text(_redirect_page(default_name + "/"), encoding="utf-8")505 506 507def _git(command: list[str], *, cwd: Path) -> None:508    result = subprocess.run(509        ["git", *command],510        cwd=cwd,511        check=False,512        text=True,513        stdout=subprocess.PIPE,514        stderr=subprocess.STDOUT,515    )516    if result.returncode != 0:517        raise RuntimeError("Git command failed:\n" + result.stdout.rstrip())518 519 520def _config_for_checkout(config: CpdocsConfig, checkout: Path) -> CpdocsConfig | None:521    """Use the checkout's own cpdocs.yml; fall back according to ``versions.legacy``."""522 523    relative = config.config_file.relative_to(config.project_root)524    candidate = checkout / relative525    if candidate.is_file():526        return load_config(candidate)527    if config.legacy == "skip":528        return None529    index = config.index530    if index is not None:531        moved = checkout / index.relative_to(config.project_root)532        index = moved if moved.is_file() else index533    return dataclasses.replace(534        config, config_file=candidate, project_root=checkout.resolve(), index=index535    )536 537 538def _repository_root(project_root: Path) -> Path:539    result = subprocess.run(540        ["git", "rev-parse", "--show-toplevel"],541        cwd=project_root,542        check=False,543        text=True,544        stdout=subprocess.PIPE,545        stderr=subprocess.PIPE,546    )547    if result.returncode != 0:548        raise RuntimeError(f"{project_root} is not inside a Git repository")549    return Path(result.stdout.strip()).resolve()550 551 552def build_versions(553    config: CpdocsConfig,554    *,555    output: Path,556    tags: str | None = None,557    project_docs_root_depth: int = 2,558) -> list[str]:559    """Document every selected ref into ``<output>/cpdocs/versions``.560 561    The project may live in a subdirectory of its Git repository. Each ref is then checked out as a562    whole and the same subdirectory is documented; refs in which it does not exist are skipped.563    """564 565    layout = Layout.for_output(output)566    project_root = config.project_root.resolve()567    subdirectory = project_root.relative_to(_repository_root(project_root))568    selected = select_refs(project_root, branches=config.branches, tags=tags or config.tags)569    shutil.rmtree(layout.versions, ignore_errors=True)570    layout.versions.mkdir(parents=True, exist_ok=True)571 572    built: list[SelectedRef] = []573    for ref in selected:574        temporary: tempfile.TemporaryDirectory[str] | None = None575        worktree: Path | None = None576        checkout = project_root577        if ref.checkout is not None:578            temporary = tempfile.TemporaryDirectory(prefix="cpdocs-ref-")579            worktree = Path(temporary.name) / "checkout"580            _git(["worktree", "add", "--detach", str(worktree), ref.checkout], cwd=project_root)581            checkout = worktree / subdirectory582        try:583            if not checkout.is_dir():584                print(585                    f"cpdocs: skipping {ref.name}: {subdirectory} does not exist", file=sys.stderr586                )587                continue588            ref_config = _config_for_checkout(config, checkout) if ref.checkout else config589            if ref_config is None:590                print(f"cpdocs: skipping {ref.name}: it has no cpdocs.yml", file=sys.stderr)591                continue592            work = layout.work(ref.name)593            policy = ref_config.branch_variants if ref.kind == "branch" else ref_config.tag_variants594            try:595                graph = build_graph(596                    ref_config,597                    source=checkout,598                    work=work,599                    ref=ref.name,600                    ref_kind=ref.kind,601                    version=ref.name,602                    variants=policy,603                )604            except RuntimeError as error:605                if config.failure == "fail":606                    raise607                print(f"cpdocs: skipping {ref.name}: {error}", file=sys.stderr)608                continue609            _render(610                graph,611                ref_config,612                output=layout.versions / ref.name,613                work=work / "render",614                root_depth=project_docs_root_depth,615            )616            built.append(ref)617        finally:618            if temporary is not None and worktree is not None:619                _git(["worktree", "remove", "--force", str(worktree)], cwd=project_root)620                temporary.cleanup()621 622    _write_versions(layout.versions, built, default=config.default_version, aliases=config.aliases)623    return [ref.name for ref in built]624 625 626def recorded_default_version(output: Path) -> str:627    """The default version recorded by the last :func:`build_versions` run."""628 629    metadata = Layout.for_output(output).versions / "versions.json"630    try:631        return str(json.loads(metadata.read_text(encoding="utf-8"))["default"])632    except (OSError, KeyError, json.JSONDecodeError):633        return "main"634 635 636__all__ = [637    "Layout",638    "build_current",639    "build_graph",640    "build_versions",641    "cmake_module",642    "describe",643    "recorded_default_version",644    "resolve_variant",645]