File: cpdocs/build.py
1# SPDX-FileCopyrightText: 2026 cpdocs developers2# SPDX-License-Identifier: Apache-2.03"""Build operations: describe, resolve variants, extract, merge and render.4 5Output layout below ``<output>/cpdocs``::6 7 current/ API site of the working tree (``cpdocs build``)8 versions/ versioned API site (``cpdocs versions``), one directory per ref9 work/<ref>/ requests, manifests and build directories handed to the build system10"""11 12from __future__ import annotations13 14import dataclasses15import fnmatch16import json17import os18import shutil19import subprocess20import sys21import tempfile22from importlib import resources23from pathlib import Path, PurePosixPath24 25from .clang_backend import extract_clang_graph26from .config import CpdocsConfig, load_config27from .contract import (28 ContractError,29 Description,30 Request,31 UnitManifest,32 VariantManifest,33 check_variant,34 read_description,35 read_variant_manifest,36)37from .html_renderer import render_html_site38from .model import ApiGraph, merge_graphs39from .python_backend import build_python_graph40from .rust_backend import build_rust_graph41from .versioning import SelectedRef, latest_tag, select_refs42 43_PLACEHOLDERS = ("source", "build", "request", "manifest", "python", "cpdocs.cmake")44_CONTAINER_KINDS = {"namespace", "package", "module"}45 46 47@dataclasses.dataclass(frozen=True)48class Layout:49 """Directories below ``<output>/cpdocs``."""50 51 root: Path52 53 @classmethod54 def for_output(cls, output: Path) -> Layout:55 return cls(root=(output / "cpdocs").resolve())56 57 @property58 def current(self) -> Path:59 return self.root / "current"60 61 @property62 def versions(self) -> Path:63 return self.root / "versions"64 65 def work(self, ref: str) -> Path:66 return self.root / "work" / ref.replace("/", "__")67 68 69def cmake_module() -> Path:70 """Path of the CMake adapter shipped with cpdocs."""71 72 return Path(str(resources.files("cpdocs") / "share" / "cmake" / "cpdocs.cmake"))73 74 75# --------------------------------------------------------------------------------------------------76# Talking to the build system77# --------------------------------------------------------------------------------------------------78 79 80def _expand(argument: str, request: Request, request_file: Path) -> str:81 values = {82 "source": str(request.source),83 "build": str(request.build),84 "request": str(request_file),85 "manifest": str(request.manifest),86 "python": sys.executable,87 "cpdocs.cmake": str(cmake_module()),88 }89 for name in _PLACEHOLDERS:90 argument = argument.replace("{" + name + "}", values[name])91 return argument92 93 94def _run_commands(95 commands: tuple[tuple[str, ...], ...], *, request: Request, request_file: Path, log: Path96) -> None:97 environment = dict(os.environ)98 environment["CPDOCS_REQUEST"] = str(request_file)99 environment["CPDOCS_MANIFEST"] = str(request.manifest)100 with log.open("w", encoding="utf-8") as stream:101 for command in commands:102 argv = [_expand(argument, request, request_file) for argument in command]103 stream.write("$ " + " ".join(argv) + "\n")104 stream.flush()105 try:106 result = subprocess.run(107 argv,108 cwd=request.source,109 env=environment,110 check=False,111 text=True,112 stdout=subprocess.PIPE,113 stderr=subprocess.STDOUT,114 )115 except FileNotFoundError as error:116 raise RuntimeError(117 f"cannot run build-system command {argv[0]!r}: {error}"118 ) from error119 stream.write(result.stdout)120 if result.returncode != 0:121 raise RuntimeError(122 f"build-system command failed ({request.phase}"123 + (f", variant {request.variant}" if request.variant else "")124 + "):\n "125 + " ".join(argv)126 + "\n\n"127 + result.stdout.rstrip()128 )129 130 131def describe(132 commands: tuple[tuple[str, ...], ...], *, source: Path, work: Path, ref: str, ref_kind: str133) -> Description:134 """Run the describe phase and read the describe manifest."""135 136 directory = work / "describe"137 shutil.rmtree(directory, ignore_errors=True)138 directory.mkdir(parents=True)139 request = Request(140 phase="describe",141 ref=ref,142 ref_kind=ref_kind,143 source=source.resolve(),144 build=directory / "build",145 manifest=directory / "manifest.json",146 variant=None,147 )148 request_file = directory / "request.json"149 request.write(request_file)150 _run_commands(commands, request=request, request_file=request_file, log=directory / "log.txt")151 return read_description(request.manifest)152 153 154def resolve_variant(155 description: Description, name: str, *, source: Path, work: Path, ref: str, ref_kind: str156) -> VariantManifest:157 """Run one variant's commands and read its variant manifest."""158 159 directory = work / "variants" / name160 shutil.rmtree(directory, ignore_errors=True)161 directory.mkdir(parents=True)162 request = Request(163 phase="variant",164 ref=ref,165 ref_kind=ref_kind,166 source=source.resolve(),167 build=directory / "build",168 manifest=directory / "manifest.json",169 variant=name,170 )171 request_file = directory / "request.json"172 request.write(request_file)173 _run_commands(174 description.variant(name).commands,175 request=request,176 request_file=request_file,177 log=directory / "log.txt",178 )179 manifest = read_variant_manifest(request.manifest, source=request.source)180 check_variant(manifest, description=description, requested=name, path=request.manifest)181 return manifest182 183 184# --------------------------------------------------------------------------------------------------185# Extraction186# --------------------------------------------------------------------------------------------------187 188 189def _clang() -> str:190 for candidate in (os.environ.get("CPDOCS_CLANG"), "clang++", "clang"):191 if candidate and shutil.which(candidate):192 return candidate193 raise RuntimeError(194 "cpdocs needs clang to document C and C++ units; put clang++ on PATH "195 "(or name it in CPDOCS_CLANG)"196 )197 198 199def _generated_sources(unit: UnitManifest, graph: ApiGraph) -> list[str]:200 values: list[str] = []201 for root in unit.api:202 if not root.generated:203 continue204 for path in root.path.rglob("*"):205 if path.is_file():206 logical = path.relative_to(root.path).as_posix()207 if logical in graph.sources:208 values.append(logical)209 return values210 211 212def extract_unit(213 unit: UnitManifest,214 *,215 project: str,216 version: str,217 project_root: Path,218 variant: str,219 work: Path,220 config: CpdocsConfig,221) -> ApiGraph:222 if unit.language in {"c", "cpp"}:223 assert unit.parse is not None224 graph = extract_clang_graph(225 project=project,226 version=version,227 project_root=project_root,228 language=unit.language,229 api_roots=[root.path for root in unit.api],230 parse=unit.parse,231 variant=variant,232 clang_executable=_clang(),233 work_directory=work / "clang" / unit.id,234 )235 generated = _generated_sources(unit, graph)236 if generated:237 graph.metadata["generated_sources"] = generated238 return graph239 if unit.language == "python":240 assert unit.package is not None241 return build_python_graph(242 project_root=project_root,243 package_root=unit.api[0].path,244 package=unit.package,245 version=version,246 include_private=not config.hide_private_python,247 )248 if unit.language == "rust":249 assert unit.rustdoc_json is not None250 return build_rust_graph(251 rustdoc_json=unit.rustdoc_json,252 project=project,253 version=version,254 project_root=project_root,255 )256 raise ContractError(f"unsupported unit language {unit.language!r}")257 258 259def _merge_metadata(target: dict[str, object], source: dict[str, object]) -> None:260 for key, value in source.items():261 if isinstance(value, list):262 existing = target.setdefault(key, [])263 assert isinstance(existing, list)264 existing.extend(item for item in value if item not in existing)265 else:266 target.setdefault(key, value)267 268 269def _combine(graphs: list[ApiGraph], *, project: str, version: str, language: str) -> ApiGraph:270 """Combine the graphs of several units extracted for the same variant."""271 272 combined = ApiGraph(project=project, version=version, language=language)273 for graph in graphs:274 for entity in graph.entities.values():275 combined.add(entity)276 for path, content in graph.sources.items():277 combined.sources.setdefault(path, content)278 _merge_metadata(combined.metadata, graph.metadata)279 combined.rebuild_children()280 return combined281 282 283def _apply_visibility(graph: ApiGraph, config: CpdocsConfig) -> None:284 patterns = config.hide_cpp_namespaces285 if not patterns:286 return287 hidden = {288 entity.id289 for entity in graph.entities.values()290 if entity.language in {"c", "cpp"}291 and entity.kind == "namespace"292 and any(fnmatch.fnmatchcase(entity.qualified_name, pattern) for pattern in patterns)293 }294 changed = True295 while changed:296 changed = False297 for entity in graph.entities.values():298 if entity.id not in hidden and entity.parent in hidden:299 hidden.add(entity.id)300 changed = True301 for identifier in hidden:302 del graph.entities[identifier]303 graph.rebuild_children()304 305 306def _check_undocumented(graph: ApiGraph, level: str) -> None:307 if level == "ignore":308 return309 missing = sorted(310 entity.qualified_name311 for entity in graph.entities.values()312 if entity.kind not in _CONTAINER_KINDS and not entity.documentation.strip()313 )314 if not missing:315 return316 message = f"{len(missing)} public API entities have no documentation:\n" + "\n".join(317 f" - {name}" for name in missing318 )319 if level == "fail":320 raise RuntimeError(message)321 print(f"cpdocs: warning: {message}", file=sys.stderr)322 323 324def build_graph(325 config: CpdocsConfig,326 *,327 source: Path,328 work: Path,329 ref: str,330 ref_kind: str,331 version: str,332 variants: str,333) -> ApiGraph:334 """Describe, resolve the selected variants, and merge them into one API graph."""335 336 description = describe(config.describe, source=source, work=work, ref=ref, ref_kind=ref_kind)337 selected = (338 [variant.name for variant in description.variants]339 if variants == "all"340 else [description.primary.name]341 )342 languages = sorted({unit.language for unit in description.units})343 language = languages[0] if len(languages) == 1 else "mixed"344 345 per_variant: list[tuple[str, ApiGraph]] = []346 metadata: dict[str, object] = {}347 for name in selected:348 manifest = resolve_variant(349 description, name, source=source, work=work, ref=ref, ref_kind=ref_kind350 )351 unit_graphs = [352 extract_unit(353 unit,354 project=description.project,355 version=version,356 project_root=source,357 variant=name,358 work=work / "variants" / name,359 config=config,360 )361 for unit in manifest.units362 ]363 graph = _combine(364 unit_graphs, project=description.project, version=version, language=language365 )366 graph.variants[name] = {367 "primary": description.variant(name).primary,368 "features": {axis: list(values) for axis, values in manifest.features.items()},369 }370 _merge_metadata(metadata, graph.metadata)371 per_variant.append((name, graph))372 373 merged = merge_graphs(374 per_variant, project=description.project, version=version, language=language375 )376 metadata["features"] = {axis.name: list(axis.values) for axis in description.features}377 metadata["units"] = [{"id": unit.id, "language": unit.language} for unit in description.units]378 metadata["generator"] = description.generator379 metadata["project_version"] = description.version380 merged.metadata = metadata381 382 _apply_visibility(merged, config)383 _check_undocumented(merged, config.check_undocumented)384 return merged385 386 387# --------------------------------------------------------------------------------------------------388# Rendering389# --------------------------------------------------------------------------------------------------390 391 392def _graph_json(graph: ApiGraph) -> str:393 return json.dumps(dataclasses.asdict(graph), indent=2, sort_keys=True, default=str) + "\n"394 395 396def _project_docs_url(graph: ApiGraph, configured: str, *, root_depth: int) -> str:397 if configured != "auto":398 return configured399 version_depth = max(1, len(PurePosixPath(graph.version).parts))400 return "../" * (root_depth + version_depth)401 402 403def _render(404 graph: ApiGraph, config: CpdocsConfig, *, output: Path, work: Path, root_depth: int405) -> None:406 work.mkdir(parents=True, exist_ok=True)407 render_html_site(408 graph,409 output_directory=output,410 project_docs_url=_project_docs_url(graph, config.project_docs, root_depth=root_depth),411 copyright_text=config.copyright or f"Copyright © {graph.project} developers",412 introduction_file=config.index,413 )414 (work / "api-graph.json").write_text(_graph_json(graph), encoding="utf-8")415 416 417def build_current(418 config: CpdocsConfig,419 *,420 output: Path,421 version: str = "main",422 variants: str = "all",423 project_docs_root_depth: int = 2,424) -> ApiGraph:425 """Document the working tree into ``<output>/cpdocs/current``."""426 427 layout = Layout.for_output(output)428 work = layout.work("current")429 graph = build_graph(430 config,431 source=config.project_root,432 work=work,433 ref=version,434 ref_kind="worktree",435 version=version,436 variants=variants,437 )438 _render(439 graph,440 config,441 output=layout.current,442 work=work / "render",443 root_depth=project_docs_root_depth,444 )445 return graph446 447 448def _html_pages(root: Path) -> list[str]:449 if not root.is_dir():450 return []451 return sorted(path.relative_to(root).as_posix() for path in root.rglob("*.html"))452 453 454def _redirect_page(target: str) -> str:455 return (456 '<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'457 f'<meta http-equiv="refresh" content="0; url={target}">'458 f'<link rel="canonical" href="{target}"></head>'459 f'<body><a href="{target}">{target}</a></body></html>\n'460 )461 462 463def _write_versions(464 output: Path, built: list[SelectedRef], *, default: str, aliases: dict[str, str]465) -> None:466 names = [ref.name for ref in built]467 newest = latest_tag([ref.name for ref in built if ref.kind == "tag"])468 469 def target(value: str) -> str | None:470 if value == "latest-tag":471 return newest472 return value if value in names else None473 474 default_name = target(default) or (names[0] if names else "main")475 resolved_aliases: dict[str, str] = {}476 for alias, value in aliases.items():477 resolved = target(value)478 if resolved is None or alias in names:479 continue480 resolved_aliases[alias] = resolved481 directory = output / alias482 directory.mkdir(parents=True, exist_ok=True)483 depth = len(PurePosixPath(alias).parts)484 (directory / "index.html").write_text(485 _redirect_page("../" * depth + resolved + "/"), encoding="utf-8"486 )487 488 metadata = {489 "default": default_name,490 "aliases": resolved_aliases,491 "versions": [492 {493 "name": ref.name,494 "kind": ref.kind,495 "url": f"{ref.name}/",496 "pages": _html_pages(output / ref.name),497 }498 for ref in built499 if (output / ref.name / "index.html").is_file()500 ],501 }502 output.mkdir(parents=True, exist_ok=True)503 (output / "versions.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")504 (output / "index.html").write_text(_redirect_page(default_name + "/"), encoding="utf-8")505 506 507def _git(command: list[str], *, cwd: Path) -> None:508 result = subprocess.run(509 ["git", *command],510 cwd=cwd,511 check=False,512 text=True,513 stdout=subprocess.PIPE,514 stderr=subprocess.STDOUT,515 )516 if result.returncode != 0:517 raise RuntimeError("Git command failed:\n" + result.stdout.rstrip())518 519 520def _config_for_checkout(config: CpdocsConfig, checkout: Path) -> CpdocsConfig | None:521 """Use the checkout's own cpdocs.yml; fall back according to ``versions.legacy``."""522 523 relative = config.config_file.relative_to(config.project_root)524 candidate = checkout / relative525 if candidate.is_file():526 return load_config(candidate)527 if config.legacy == "skip":528 return None529 index = config.index530 if index is not None:531 moved = checkout / index.relative_to(config.project_root)532 index = moved if moved.is_file() else index533 return dataclasses.replace(534 config, config_file=candidate, project_root=checkout.resolve(), index=index535 )536 537 538def _repository_root(project_root: Path) -> Path:539 result = subprocess.run(540 ["git", "rev-parse", "--show-toplevel"],541 cwd=project_root,542 check=False,543 text=True,544 stdout=subprocess.PIPE,545 stderr=subprocess.PIPE,546 )547 if result.returncode != 0:548 raise RuntimeError(f"{project_root} is not inside a Git repository")549 return Path(result.stdout.strip()).resolve()550 551 552def build_versions(553 config: CpdocsConfig,554 *,555 output: Path,556 tags: str | None = None,557 project_docs_root_depth: int = 2,558) -> list[str]:559 """Document every selected ref into ``<output>/cpdocs/versions``.560 561 The project may live in a subdirectory of its Git repository. Each ref is then checked out as a562 whole and the same subdirectory is documented; refs in which it does not exist are skipped.563 """564 565 layout = Layout.for_output(output)566 project_root = config.project_root.resolve()567 subdirectory = project_root.relative_to(_repository_root(project_root))568 selected = select_refs(project_root, branches=config.branches, tags=tags or config.tags)569 shutil.rmtree(layout.versions, ignore_errors=True)570 layout.versions.mkdir(parents=True, exist_ok=True)571 572 built: list[SelectedRef] = []573 for ref in selected:574 temporary: tempfile.TemporaryDirectory[str] | None = None575 worktree: Path | None = None576 checkout = project_root577 if ref.checkout is not None:578 temporary = tempfile.TemporaryDirectory(prefix="cpdocs-ref-")579 worktree = Path(temporary.name) / "checkout"580 _git(["worktree", "add", "--detach", str(worktree), ref.checkout], cwd=project_root)581 checkout = worktree / subdirectory582 try:583 if not checkout.is_dir():584 print(585 f"cpdocs: skipping {ref.name}: {subdirectory} does not exist", file=sys.stderr586 )587 continue588 ref_config = _config_for_checkout(config, checkout) if ref.checkout else config589 if ref_config is None:590 print(f"cpdocs: skipping {ref.name}: it has no cpdocs.yml", file=sys.stderr)591 continue592 work = layout.work(ref.name)593 policy = ref_config.branch_variants if ref.kind == "branch" else ref_config.tag_variants594 try:595 graph = build_graph(596 ref_config,597 source=checkout,598 work=work,599 ref=ref.name,600 ref_kind=ref.kind,601 version=ref.name,602 variants=policy,603 )604 except RuntimeError as error:605 if config.failure == "fail":606 raise607 print(f"cpdocs: skipping {ref.name}: {error}", file=sys.stderr)608 continue609 _render(610 graph,611 ref_config,612 output=layout.versions / ref.name,613 work=work / "render",614 root_depth=project_docs_root_depth,615 )616 built.append(ref)617 finally:618 if temporary is not None and worktree is not None:619 _git(["worktree", "remove", "--force", str(worktree)], cwd=project_root)620 temporary.cleanup()621 622 _write_versions(layout.versions, built, default=config.default_version, aliases=config.aliases)623 return [ref.name for ref in built]624 625 626def recorded_default_version(output: Path) -> str:627 """The default version recorded by the last :func:`build_versions` run."""628 629 metadata = Layout.for_output(output).versions / "versions.json"630 try:631 return str(json.loads(metadata.read_text(encoding="utf-8"))["default"])632 except (OSError, KeyError, json.JSONDecodeError):633 return "main"634 635 636__all__ = [637 "Layout",638 "build_current",639 "build_graph",640 "build_versions",641 "cmake_module",642 "describe",643 "recorded_default_version",644 "resolve_variant",645]