From 0815b547a28640f15b73e714b6259189bfc57f06 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Fri, 2 Oct 2026 13:37:40 +0700 Subject: [PATCH] docs: mirror the module tree in the docs folder by default Pages now live in folders that follow the module tree: a module's page sits next to the folder holding its sub-modules (auth.md, auth/login.md), and overview.md stays at the root. Module names stay unique across the wiki, so the module-tree key is still the page's filename stem. - doc_layout: one place that maps modules to page paths, finds pages in either layout, and after each run moves misplaced pages and rewrites links between pages to correct relative paths - prompts give each agent its page path and every module's page path; the sub-module tool reports links relative to the parent page - --flat keeps the old single-folder layout for small models - the layout is recorded in metadata.json and --update keeps it; docs without a recorded layout are treated as flat - the updater, GitHub Pages viewer, web app (now with a path traversal guard), MCP tools (doc_path in processing_order.json) and the wiki-generator skill follow the layout - '&' is no longer allowed in module names: agents shell-escape it and write pages into a stray folder Closes #125 (hierarchical output; OKF export not included) --- CHANGELOG.md | 12 + README.md | 8 +- codewiki/cli/adapters/doc_generator.py | 18 +- codewiki/cli/commands/generate.py | 35 +- codewiki/cli/html_generator.py | 9 + codewiki/mcp/server.py | 21 +- codewiki/mcp/tools/module_tree.py | 22 +- codewiki/mcp/tools/prompt_server.py | 49 ++- .../generate_sub_module_documentations.py | 63 ++- .../src/be/agent_tools/str_replace_editor.py | 13 + codewiki/src/be/caw_backend.py | 15 +- codewiki/src/be/caw_toolkit.py | 55 +-- codewiki/src/be/doc_layout.py | 333 ++++++++++++++++ codewiki/src/be/documentation_generator.py | 44 ++- codewiki/src/be/module_naming.py | 100 +++-- codewiki/src/be/prompt_template.py | 126 +++++- codewiki/src/be/pydantic_ai_backend.py | 15 +- codewiki/src/be/updater/leaf_agent.py | 26 +- codewiki/src/be/updater/orchestrator.py | 18 +- codewiki/src/be/updater/pages.py | 40 +- codewiki/src/be/updater/prompts.py | 45 ++- codewiki/src/be/updater/reference_index.py | 18 +- codewiki/src/be/updater/stale_scan.py | 2 +- codewiki/src/be/updater/tree.py | 2 +- codewiki/src/be/updater/verdicts.py | 4 +- codewiki/src/config.py | 12 + codewiki/src/fe/routes.py | 13 +- codewiki/src/fe/templates.py | 3 +- codewiki/src/fe/visualise_docs.py | 3 + codewiki/src/language.py | 2 +- .../github_pages/viewer_template.html | 40 +- guides/cli-reference.md | 28 ++ skills/codewiki-wiki-generator/SKILL.md | 11 +- tests/test_doc_layout.py | 360 ++++++++++++++++++ tests/test_overview_structure.py | 10 +- tests/test_processing_order_update.py | 5 +- 36 files changed, 1337 insertions(+), 243 deletions(-) create mode 100644 codewiki/src/be/doc_layout.py create mode 100644 tests/test_doc_layout.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f80c151..b43b07e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,18 @@ uses [Semantic Versioning](https://semver.org/). struct-literal, and call edges. `#[cfg(test)]` modules are skipped, and `build.rs` is classified as a build artifact. +### Changed + +- **Docs mirror the module tree** ([#125](https://github.com/FSoft-AI4Code/CodeWiki/issues/125)). + A module's page now sits next to the folder that holds its sub-modules + (`auth.md`, `auth/login.md`) instead of every page sharing one flat folder. + Links between pages are relative. After each run, pages saved in the wrong + folder are moved and wrong links are repaired. `--flat` keeps the old + layout for small models. The layout is recorded in `metadata.json`, and + `--update` keeps it, so existing flat docs stay flat. The GitHub Pages + viewer, the web app and the MCP tools (`doc_path` in + `processing_order.json`) follow the layout. + ## [2.0.0] - 2026-09-18 The first release since 1.0.1. Everything that landed on `main` in between is diff --git a/README.md b/README.md index 1f0a004c..54ae947b 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,8 @@ configuration in CodeWiki. ``` ./docs/ ├── overview.md # start here -├── .md ... # one page per module, leaves and parents +├── .md ... # one page per top-level module +├── /.md ... # sub-module pages, in folders mirroring the module tree ├── module_tree.json # the module hierarchy ├── first_module_tree.json # clustering result before super-grouping ├── metadata.json # model, version, commit, statistics @@ -174,6 +175,11 @@ configuration in CodeWiki. └── index.html # viewer (with --github-pages) ``` +Pages mirror the module tree: a module's page sits next to the folder holding +its sub-modules (`auth.md`, `auth/login.md`). `--flat` puts every page in +`./docs` instead, which helps small models that get relative links wrong. +`--update` keeps the layout the docs were generated with. + This repository's own output is checked in under [`./docs/`](./docs/). ## Benchmark results diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 94d3f591..472cba4e 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -19,10 +19,19 @@ # Import backend modules from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.be.doc_layout import list_doc_files +from codewiki.src.config import DEFAULT_LAYOUT from codewiki.src.config import Config as BackendConfig from codewiki.src.config import set_cli_context +def _generated_files(working_dir: str) -> list[str]: + """Docs-relative paths of the pages and the JSON files in the docs root.""" + pages = sorted(list_doc_files(working_dir).values()) + jsons = sorted(f for f in os.listdir(working_dir) if f.endswith(".json")) + return pages + jsons + + class CLIDocumentationGenerator: """ CLI adapter for documentation generation with progress reporting. @@ -154,6 +163,7 @@ def generate(self) -> DocumentationJob: artifacts_enabled=self.config.get("artifacts_enabled", True), artifact_token_budget=self.config.get("artifact_token_budget", 200_000), with_prose=self.config.get("with_prose", False), + layout=self.config.get("layout", DEFAULT_LAYOUT), ) # Run backend documentation generation @@ -372,9 +382,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig): self._merge_update_summary(working_dir) # Collect generated files - for file_path in os.listdir(working_dir): - if file_path.endswith((".md", ".json")): - self.job.files_generated.append(file_path) + self.job.files_generated.extend(_generated_files(working_dir)) except Exception as e: # noqa: BLE001 — surfaced to the user as an APIError raise APIError(f"Documentation generation failed: {e}") @@ -448,9 +456,7 @@ async def _run_incremental_update( prior_history = self._read_update_history(working_dir) doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) self._merge_update_summary(working_dir, prior_history) - for file_path in os.listdir(working_dir): - if file_path.endswith((".md", ".json")): - self.job.files_generated.append(file_path) + self.job.files_generated.extend(_generated_files(working_dir)) tree_path = os.path.join(working_dir, "module_tree.json") if os.path.exists(tree_path): with open(tree_path, encoding="utf-8") as f: diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index 24765d8c..daa1a1f8 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -3,6 +3,7 @@ """ import logging +import os import sys import time import traceback @@ -24,6 +25,8 @@ ) from codewiki.cli.utils.instructions import display_post_generation_instructions from codewiki.cli.utils.logging import create_logger +from codewiki.src.be.doc_layout import find_doc, list_doc_files, read_layout, remove_doc +from codewiki.src.config import DEFAULT_LAYOUT, LAYOUT_FLAT from codewiki.src.language import resolve_update_language from codewiki.cli.utils.repo_validator import ( check_writable_output, @@ -204,13 +207,13 @@ def _find_affected(tree, parent_names=None): if modules_to_invalidate: modules_to_invalidate.add("overview") - # Delete affected module docs + # Delete affected module docs (wherever the docs layout put them) for mod_name in modules_to_invalidate: - doc_path = output_dir / f"{mod_name}.md" - if doc_path.exists(): - doc_path.unlink() + doc_path = find_doc(str(output_dir), mod_name, module_tree) + if doc_path is not None: + remove_doc(str(output_dir), doc_path) if verbose: - logger.debug(f"Invalidated: {doc_path.name}") + logger.debug(f"Invalidated: {os.path.relpath(doc_path, output_dir)}") if verbose: logger.debug(f"Invalidated {len(modules_to_invalidate)} modules for regeneration.") @@ -353,6 +356,13 @@ def _find_affected(tree, parent_names=None): is_flag=True, help="Also read the root README and docs/ as a `prose` artifact class (off by default)", ) +@click.option( + "--flat", + is_flag=True, + help="Save every page in the output directory root instead of folders mirroring the " + "module tree (for small models that struggle with relative links). --update keeps the " + "layout the docs were generated with", +) @click.option( "--artifact-exclude", type=str, @@ -447,6 +457,7 @@ def generate_command( artifacts: bool = True, artifact_token_budget: int = 200_000, with_prose: bool = False, + flat: bool = False, artifact_exclude: str | None = None, update: bool = False, compare_to: str | None = None, @@ -585,6 +596,16 @@ def generate_command( ) except ValueError as e: raise ConfigurationError(str(e)) from None + layout = LAYOUT_FLAT if flat else DEFAULT_LAYOUT + if updating_existing_docs: + # Keep updated pages where the existing docs put them + stored_layout = read_layout(str(output_dir)) + if flat and stored_layout != LAYOUT_FLAT: + logger.warning( + f"These docs use the {stored_layout} layout; --flat is ignored by --update. " + "Rerun without --update to regenerate them flat." + ) + layout = stored_layout if update and output_dir.exists(): changed_files = _detect_changed_files( repo_path, output_dir, logger, verbose, compare_to=compare_to @@ -616,7 +637,7 @@ def generate_command( if ( not update and output_dir.exists() - and list(output_dir.glob("*.md")) + and list_doc_files(str(output_dir)) and not click.confirm( f"\n{output_dir} already contains documentation. Overwrite?", default=True ) @@ -777,6 +798,8 @@ def generate_command( "artifacts_enabled": artifacts, "artifact_token_budget": artifact_token_budget, "with_prose": with_prose, + # Docs layout (runtime-only; --update keeps the stored one) + "layout": layout, # Incremental updater (runtime-only) "update": update, "update_options": { diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py index 6fbf7586..96af6ef4 100644 --- a/codewiki/cli/html_generator.py +++ b/codewiki/cli/html_generator.py @@ -156,12 +156,21 @@ def generate( page_titles = extract_page_titles(str(docs_dir), module_tree) page_titles_json = json.dumps(page_titles, ensure_ascii=False).replace(" list[Tool]: }, "filename": { "type": "string", - "description": "Filename for the doc (e.g., 'auth_module.md')", + "description": ( + "Path of the doc relative to the output dir: the module's " + "doc_path from processing_order.json (e.g., 'auth.md' or " + "'auth/login.md'; folders are created as needed)" + ), }, "content": { "type": "string", @@ -531,6 +535,8 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]: set_cli_context(True) + from codewiki.src.be.doc_layout import docs_layout, list_doc_files + backend_config = BackendConfig.from_cli( repo_path=str(repo_path), output_dir=str(output_dir), @@ -544,6 +550,7 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]: max_tokens=config.max_tokens, agent_instructions=agent_instructions or None, use_gitignore=arguments.get("use_gitignore", True), + layout=docs_layout(str(output_dir)), ) from codewiki.cli.utils.repo_validator import get_git_commit_hash @@ -554,9 +561,9 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]: ) await doc_gen.run() - generated_files = [] + generated_files = sorted(list_doc_files(str(output_dir)).values()) for f in output_dir.iterdir(): - if f.suffix in (".md", ".json", ".html"): + if f.suffix in (".json", ".html"): generated_files.append(f.name) result = { @@ -654,10 +661,18 @@ def _write_generation_metadata(session: SessionState) -> None: except (json.JSONDecodeError, OSError): pass + from codewiki.src.be.doc_layout import docs_layout, organize_docs + + # Keep the layout of existing docs; new docs get the default. Move + # misplaced pages and fix links between pages before stamping it. + layout = docs_layout(str(output_dir)) + organize_docs(str(output_dir), layout) + existing["generation_info"] = { **existing.get("generation_info", {}), "commit_id": commit_id, "timestamp": datetime.now().isoformat(), + "layout": layout, } metadata_path.write_text( json.dumps(existing, indent=2, ensure_ascii=False), diff --git a/codewiki/mcp/tools/module_tree.py b/codewiki/mcp/tools/module_tree.py index 8e448e14..107c3dd3 100644 --- a/codewiki/mcp/tools/module_tree.py +++ b/codewiki/mcp/tools/module_tree.py @@ -13,7 +13,8 @@ from typing import Any, Dict, List, Tuple from codewiki.mcp.session import SessionStore -from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME +from codewiki.src.be.doc_layout import docs_layout, module_doc_relpath +from codewiki.src.config import DEFAULT_LAYOUT, FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME logger = logging.getLogger(__name__) @@ -30,12 +31,15 @@ def _cap(ids: List[str]) -> Tuple[List[str], bool]: def _get_processing_order( - module_tree: Dict[str, Any], parent_path: List[str] | None = None + module_tree: Dict[str, Any], + parent_path: List[str] | None = None, + layout: str = DEFAULT_LAYOUT, ) -> List[Dict[str, Any]]: """Compute leaf-first processing order from a module tree. - Returns a list of dicts with module path, name, leaf status, and - component/children info. + Returns a list of dicts with module path, name, leaf status, + component/children info and ``doc_path`` (the page to write, relative to + the output dir, for the docs ``layout``). """ if parent_path is None: parent_path = [] @@ -54,6 +58,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None: "module": module_name, "path": current_path, "is_leaf": False, + "doc_path": module_doc_relpath(current_path, layout), "children": list(children.keys()), "components": module_info.get("components", []), } @@ -64,6 +69,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None: "module": module_name, "path": current_path, "is_leaf": True, + "doc_path": module_doc_relpath(current_path, layout), "components": module_info.get("components", []), } ) @@ -189,7 +195,7 @@ def handle_save_module_tree( logger.info("save_module_tree for session %s: %s", session_id, note) # Compute processing order and write to workspace file - order = _get_processing_order(module_tree) + order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir)) order_file = None if session.workspace is not None: order_path = session.workspace.write_json("processing_order.json", order) @@ -206,7 +212,9 @@ def handle_save_module_tree( "Read the processing_order.json file for the leaf-first generation order. " "Process leaf modules first (is_leaf=true), then parent modules. " "For each leaf module: get_prompt('system_leaf') + read_code_components + write_doc_file. " - "For each parent module: get_prompt('overview_module') + write_doc_file." + "For each parent module: get_prompt('overview_module') + write_doc_file. " + "Save each module's page at its doc_path (sub-folders are created automatically) " + "and link pages with paths relative to the linking page." ), } if warning: @@ -237,7 +245,7 @@ def handle_get_processing_order( else: return json.dumps({"error": "Module tree not found. Call save_module_tree first."}) - order = _get_processing_order(module_tree) + order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir)) # Write to workspace file order_file = None diff --git a/codewiki/mcp/tools/prompt_server.py b/codewiki/mcp/tools/prompt_server.py index 8e0c3702..8c3e7914 100644 --- a/codewiki/mcp/tools/prompt_server.py +++ b/codewiki/mcp/tools/prompt_server.py @@ -19,7 +19,9 @@ format_system_prompt, format_leaf_system_prompt, format_cluster_prompt, + format_links_note, ) +from codewiki.src.config import DEFAULT_LAYOUT # Prompt catalog: maps prompt_type to (raw_template, usage_hint, variables_doc) @@ -36,16 +38,18 @@ "description": "System prompt for documenting a complex (multi-file, parent) module. Includes sub-module delegation instructions.", "usage_hint": ( "Use as the system prompt when generating docs for a parent module. " - "The agent should create {module_name}.md with architecture overview " - "and cross-references to sub-module docs." + "The agent should create the module's page at its doc_path (from " + "processing_order.json) with architecture overview and cross-references to " + "sub-module docs. Pass variables module_name and module_path." ), }, "system_leaf": { "description": "System prompt for documenting a leaf (single-file or simple) module.", "usage_hint": ( "Use as the system prompt when generating docs for a leaf module. " - "The agent should create {module_name}.md with detailed documentation " - "including Mermaid diagrams." + "The agent should create the module's page at its doc_path (from " + "processing_order.json) with detailed documentation including Mermaid diagrams. " + "Pass variables module_name and module_path." ), }, "user": { @@ -105,6 +109,14 @@ def handle_get_prompt( return json.dumps(result, indent=2, ensure_ascii=False) +def _page_location(variables: Dict[str, Any], module_name: str) -> tuple[list[str], str]: + """``module_path`` and docs ``layout`` from the variables (top-level module by default).""" + module_path = variables.get("module_path") or [module_name] + if isinstance(module_path, str): + module_path = [p for p in module_path.split("/") if p] + return list(module_path), variables.get("layout") or DEFAULT_LAYOUT + + def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: """Resolve a prompt template with optional variable substitution.""" @@ -123,27 +135,34 @@ def _resolve_prompt(prompt_type: str, variables: Dict[str, Any]) -> str: elif prompt_type == "system_complex": module_name = variables.get("module_name", "MODULE_NAME") custom_instructions = variables.get("custom_instructions", None) - return format_system_prompt(module_name, custom_instructions) + module_path, layout = _page_location(variables, module_name) + return format_system_prompt(module_name, custom_instructions, module_path, layout) elif prompt_type == "system_leaf": module_name = variables.get("module_name", "MODULE_NAME") custom_instructions = variables.get("custom_instructions", None) - return format_leaf_system_prompt(module_name, custom_instructions) + module_path, layout = _page_location(variables, module_name) + return format_leaf_system_prompt(module_name, custom_instructions, module_path, layout) elif prompt_type == "user": module_name = variables.get("module_name", "MODULE_NAME") module_tree = variables.get("module_tree", {}) + module_path, layout = _page_location(variables, module_name) # Return the template with placeholders filled as possible - return USER_PROMPT.format( - module_name=module_name, - module_tree=json.dumps(module_tree, indent=2) - if module_tree - else "", - formatted_core_component_codes=variables.get( - "formatted_core_component_codes", - "", - ), + return ( + USER_PROMPT.format( + module_name=module_name, + module_tree=json.dumps(module_tree, indent=2) + if module_tree + else "", + formatted_core_component_codes=variables.get( + "formatted_core_component_codes", + "", + ), + ) + + "\n\n" + + format_links_note(module_name, module_path, layout) ) elif prompt_type == "overview_module": diff --git a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py index 9583bd61..fabc3131 100644 --- a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -1,10 +1,9 @@ -import os - from pydantic_ai import RunContext, Tool, Agent from pydantic_ai.usage import UsageLimits from codewiki.src.be.agent_tools.deps import CodeWikiDeps -from codewiki.src.be.module_naming import plan_sub_module_specs +from codewiki.src.be.doc_layout import config_layout, module_doc_file +from codewiki.src.be.module_naming import skipped_report, plan_sub_module_specs, sub_module_report from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool from codewiki.src.be.llm_services import create_fallback_models @@ -43,7 +42,7 @@ async def generate_sub_module_documentation( fallback_models = create_fallback_models(deps.config) # Resolve name collisions against the module tree and files already on disk - # before touching the tree (issue #76): docs live in one flat directory. + # before touching the tree (issue #76): names are unique across the wiki. # A request that is already documented (plain or parent-prefixed name) is # skipped rather than renamed x_2, x_3, ... (issue #113). plan = plan_sub_module_specs( @@ -53,8 +52,11 @@ async def generate_sub_module_documentation( deps.absolute_docs_path, ) name_map = plan.name_map + layout = config_layout(deps.config) + parent_path = list(deps.path_to_current_module) + parent_doc = module_doc_file(previous_module_name, parent_path, layout) if not name_map: - return _skipped_report(plan.skipped, deps.current_module_name) + return skipped_report(plan.skipped, parent_doc) final_specs = { name_map[requested_name]: core_component_ids for requested_name, core_component_ids in sub_module_specs.items() @@ -88,7 +90,12 @@ async def generate_sub_module_documentation( model=fallback_models, name=sub_module_name, deps_type=CodeWikiDeps, - system_prompt=format_system_prompt(sub_module_name, ctx.deps.custom_instructions), + system_prompt=format_system_prompt( + sub_module_name, + ctx.deps.custom_instructions, + parent_path + [sub_module_name], + layout, + ), retries=ctx.deps.config.agent_retries, tools=[ read_code_components_tool, @@ -102,7 +109,10 @@ async def generate_sub_module_documentation( name=sub_module_name, deps_type=CodeWikiDeps, system_prompt=format_leaf_system_prompt( - sub_module_name, ctx.deps.custom_instructions + sub_module_name, + ctx.deps.custom_instructions, + parent_path + [sub_module_name], + layout, ), retries=ctx.deps.config.agent_retries, tools=[read_code_components_tool, str_replace_editor_tool], @@ -120,6 +130,8 @@ async def generate_sub_module_documentation( core_component_ids=core_component_ids, components=ctx.deps.components, module_tree=ctx.deps.module_tree, + module_path=list(deps.path_to_current_module), + layout=layout, ), deps=ctx.deps, usage_limits=UsageLimits(request_limit=ctx.deps.config.request_limit), @@ -132,35 +144,14 @@ async def generate_sub_module_documentation( # restore the previous module name deps.current_module_name = previous_module_name - # Report what actually landed on disk so the parent agent links real filenames. - saved = [] - missing = [] - for requested_name, final_name in name_map.items(): - entry = f"{final_name}.md" - if final_name != requested_name: - entry += f" (requested '{requested_name}', renamed to avoid a collision)" - if os.path.exists(os.path.join(deps.absolute_docs_path, f"{final_name}.md")): - saved.append(entry) - else: - missing.append(entry) - - report = f"Saved documentations: {', '.join(saved) if saved else 'none'}." - if missing: - report += f" MISSING (generation did not produce these files): {', '.join(missing)}." - logger.warning("Sub-module documentation missing after generation: %s", ", ".join(missing)) - if plan.skipped: - report += " " + _skipped_report(plan.skipped, deps.current_module_name) - return report - - -def _skipped_report(skipped: dict[str, str], current_module_name: str) -> str: - """Tell the parent agent, unambiguously, not to retry skipped sub-modules.""" - if not skipped: - return "No sub-modules were generated." - items = ", ".join(f"'{name}' ({reason})" for name, reason in skipped.items()) - return ( - f"Skipped sub-modules: {items}. Do NOT call generate_sub_module_documentation again " - f"for these; link the existing pages from `{current_module_name}.md` instead." + return sub_module_report( + name_map, + plan.skipped, + deps.absolute_docs_path, + deps.module_tree, + previous_module_name, + parent_path, + layout, ) diff --git a/codewiki/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py index 257630d3..a892e755 100644 --- a/codewiki/src/be/agent_tools/str_replace_editor.py +++ b/codewiki/src/be/agent_tools/str_replace_editor.py @@ -516,6 +516,10 @@ def validate_path(self, command: str, path: Path): return True def create_file(self, path: Path, file_text: str): + # Nested doc pages (hierarchical layout) live in folders that may not + # exist yet; create them, but only inside the docs directory. + if not path.parent.exists() and self._inside_docs(path): + path.parent.mkdir(parents=True, exist_ok=True) if not path.parent.exists(): self.logs.append( f"The parent directory {self._get_display_path(path.parent)} does not exist. Please create it first." @@ -525,6 +529,15 @@ def create_file(self, path: Path, file_text: str): self._file_history[path].append(file_text) self.logs.append(f"File created successfully at: {self._get_display_path(path)}") + def _inside_docs(self, path: Path) -> bool: + if self.absolute_docs_path is None: + return False + try: + path.resolve().relative_to(self.absolute_docs_path.resolve()) + except ValueError: + return False + return True + def view(self, path: Path, view_range: list[int] | None = None): """Implement the view command""" if path.is_dir(): diff --git a/codewiki/src/be/caw_backend.py b/codewiki/src/be/caw_backend.py index f10c46a7..60e76403 100644 --- a/codewiki/src/be/caw_backend.py +++ b/codewiki/src/be/caw_backend.py @@ -39,6 +39,7 @@ format_user_prompt, ) from codewiki.src.be.utils import count_tokens, is_complex_module, set_main_loop +from codewiki.src.be.doc_layout import config_layout, find_doc from codewiki.src.config import MODULE_TREE_FILENAME, OVERVIEW_FILENAME, Config from codewiki.src.utils import file_manager @@ -377,7 +378,9 @@ def _run_module_agent_sync( if os.path.exists(overview_docs_path): logger.info("✓ Overview docs already exists at %s", overview_docs_path) return module_tree - docs_path = os.path.join(working_dir, f"{module_name}.md") + docs_path = find_doc(working_dir, module_name, module_tree) if module_path else None + if docs_path is None: + docs_path = os.path.join(working_dir, f"{module_name}.md") if os.path.exists(docs_path): logger.info("✓ Module docs already exists at %s", docs_path) return module_tree @@ -404,9 +407,13 @@ def _run_module_agent_sync( ) if can_delegate: - system_prompt = format_system_prompt(module_name, custom_instructions) + system_prompt = format_system_prompt( + module_name, custom_instructions, list(module_path), config_layout(config) + ) else: - system_prompt = format_leaf_system_prompt(module_name, custom_instructions) + system_prompt = format_leaf_system_prompt( + module_name, custom_instructions, list(module_path), config_layout(config) + ) deps = CodeWikiDeps( absolute_docs_path=working_dir, @@ -437,6 +444,8 @@ def _run_module_agent_sync( core_component_ids=core_component_ids, components=components, module_tree=deps.module_tree, + module_path=list(module_path), + layout=config_layout(config), ) # caw forks claude / codex via subprocess.Popen without a cwd, so the diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py index 76f887c1..8ff30f22 100644 --- a/codewiki/src/be/caw_toolkit.py +++ b/codewiki/src/be/caw_toolkit.py @@ -28,7 +28,8 @@ from mcp.server.fastmcp import Context from codewiki.src.be.agent_tools.deps import CodeWikiDeps -from codewiki.src.be.module_naming import plan_sub_module_specs +from codewiki.src.be.doc_layout import config_layout, module_doc_file +from codewiki.src.be.module_naming import skipped_report, plan_sub_module_specs, sub_module_report if TYPE_CHECKING: from codewiki.src.be.caw_backend import CawBackend @@ -257,7 +258,8 @@ async def generate_sub_module_documentation( "(leaf module: single-file or below the token threshold, or max recursion " "depth reached). DO NOT call this tool again for this module. " "Instead, write the documentation directly with `str_replace_editor` " - f"(create command) as a single `{self._deps.current_module_name}.md` " + "(create command) as a single " + f"`{module_doc_file(self._deps.current_module_name, self._deps.path_to_current_module, config_layout(self._deps.config))}` " "file covering the provided core components inline (architecture, " "components, diagrams, etc.) — no sub-module fan-out." ) @@ -284,7 +286,7 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: previous_module_name = deps.current_module_name # Resolve name collisions against the module tree and files already on - # disk before touching the tree (issue #76): docs live in one flat directory. + # disk before touching the tree (issue #76): names are unique across the wiki. # A request that is already documented (plain or parent-prefixed name) is # skipped rather than renamed x_2, x_3, ... (issue #113). plan = plan_sub_module_specs( @@ -294,8 +296,12 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: deps.absolute_docs_path, ) name_map = plan.name_map + layout = config_layout(deps.config) + parent_path = list(deps.path_to_current_module) if not name_map: - return _skipped_report(plan.skipped, deps.current_module_name) + return skipped_report( + plan.skipped, module_doc_file(previous_module_name, parent_path, layout) + ) final_specs = { name_map[requested_name]: core_ids for requested_name, core_ids in sub_module_specs.items() @@ -341,35 +347,12 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str: finally: deps.current_module_name = previous_module_name - # Report what actually landed on disk so the parent agent links real filenames. - saved = [] - missing = [] - for requested_name, final_name in name_map.items(): - entry = f"{final_name}.md" - if final_name != requested_name: - entry += f" (requested '{requested_name}', renamed to avoid a collision)" - if os.path.exists(os.path.join(deps.absolute_docs_path, f"{final_name}.md")): - saved.append(entry) - else: - missing.append(entry) - - report = f"Saved documentations: {', '.join(saved) if saved else 'none'}." - if missing: - report += f" MISSING (generation did not produce these files): {', '.join(missing)}." - logger.warning( - "Sub-module documentation missing after generation: %s", ", ".join(missing) - ) - if plan.skipped: - report += " " + _skipped_report(plan.skipped, deps.current_module_name) - return report - - -def _skipped_report(skipped: dict[str, str], current_module_name: str) -> str: - """Tell the parent agent, unambiguously, not to retry skipped sub-modules.""" - if not skipped: - return "No sub-modules were generated." - items = ", ".join(f"'{name}' ({reason})" for name, reason in skipped.items()) - return ( - f"Skipped sub-modules: {items}. Do NOT call generate_sub_module_documentation again " - f"for these; link the existing pages from `{current_module_name}.md` instead." - ) + return sub_module_report( + name_map, + plan.skipped, + deps.absolute_docs_path, + deps.module_tree, + previous_module_name, + parent_path, + layout, + ) diff --git a/codewiki/src/be/doc_layout.py b/codewiki/src/be/doc_layout.py new file mode 100644 index 00000000..a6b6d56f --- /dev/null +++ b/codewiki/src/be/doc_layout.py @@ -0,0 +1,333 @@ +"""Where each module's page lives in the docs directory. + +Two layouts are supported: + +* ``hierarchical`` (default): pages mirror the module tree. A module's page + sits next to the folder holding its children, so a page never moves when + sub-modules are added to it mid-run:: + + overview.md + auth.md + auth/login.md + auth/session.md + auth/session/store.md + +* ``flat``: every page is ``{module_name}.md`` in the docs root (for small + models that keep getting relative links wrong). + +Module names stay unique across the whole tree in both layouts, so the +module-tree key is still the page's filename stem; only the folder differs. +Lookups are layout-agnostic (they try the nested path, then the root), so +docs written in either layout, or by an agent that saved a page in the wrong +place, still resolve. :func:`organize_docs` moves misplaced pages to where the +layout expects them and rewrites links between pages to correct relative +paths. +""" + +from __future__ import annotations + +import json +import logging +import os +import posixpath +import re +from typing import Any +from urllib.parse import unquote + +from codewiki.src.config import ( + DEFAULT_LAYOUT, + LAYOUT_FLAT, + LAYOUT_HIERARCHICAL, + MODULE_TREE_FILENAME, + OVERVIEW_FILENAME, +) + +logger = logging.getLogger(__name__) + +OVERVIEW_STEM = OVERVIEW_FILENAME[: -len(".md")] +METADATA_FILENAME = "metadata.json" +# Working files (dependency graphs, reference index) live in ``docs/temp`` +TEMP_DIR = "temp" +LAYOUTS = (LAYOUT_HIERARCHICAL, LAYOUT_FLAT) + + +def normalize_layout(layout: str | None) -> str: + return LAYOUT_FLAT if layout == LAYOUT_FLAT else LAYOUT_HIERARCHICAL + + +def config_layout(config: Any) -> str: + """Layout of ``config`` (configs built without one use the default).""" + return normalize_layout(getattr(config, "layout", None)) + + +def module_doc_relpath(module_path: list[str], layout: str | None) -> str: + """Docs-relative POSIX path of the page for the module at ``module_path``. + + ``module_path`` includes the module itself; ``[]`` is the repository + overview. + """ + if not module_path: + return OVERVIEW_FILENAME + if normalize_layout(layout) == LAYOUT_FLAT: + return f"{module_path[-1]}.md" + return posixpath.join(*module_path[:-1], f"{module_path[-1]}.md") + + +def module_doc_file(module_name: str, module_path: list[str] | None, layout: str | None) -> str: + """Docs-relative file the agent documenting ``module_name`` must write. + + Like :func:`module_doc_relpath`, except that the whole-repository agent + (``module_path == []``) writes ``{module_name}.md`` in the docs root, + which is renamed to ``overview.md`` afterwards. + """ + if not module_path: + return f"{module_name}.md" + return module_doc_relpath(module_path, layout) + + +def iter_tree_paths(module_tree: dict[str, Any] | None): + """Yield the path (list of names) of every module in the tree.""" + stack: list[tuple[list[str], Any]] = [([], module_tree or {})] + while stack: + prefix, level = stack.pop() + if not isinstance(level, dict): + continue + for name, info in level.items(): + path = prefix + [name] + yield path + if isinstance(info, dict) and isinstance(info.get("children"), dict): + stack.append((path, info["children"])) + + +def doc_relpaths(module_tree: dict[str, Any] | None, layout: str | None) -> dict[str, str]: + """Map every module name (and ``overview``) to its page's relative path.""" + paths = {OVERVIEW_STEM: OVERVIEW_FILENAME} + for path in iter_tree_paths(module_tree): + paths.setdefault(path[-1], module_doc_relpath(path, layout)) + return paths + + +def _load_json(path: str) -> Any: + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return None + + +def load_module_tree(docs_dir: str) -> dict[str, Any]: + tree = _load_json(os.path.join(docs_dir, MODULE_TREE_FILENAME)) + return tree if isinstance(tree, dict) else {} + + +def read_layout(docs_dir: str) -> str: + """Layout recorded in ``metadata.json``. + + Docs without a recorded layout predate hierarchical output, so they are + flat. + """ + metadata = _load_json(os.path.join(docs_dir, METADATA_FILENAME)) + info = metadata.get("generation_info") if isinstance(metadata, dict) else None + layout = info.get("layout") if isinstance(info, dict) else None + return layout if layout in LAYOUTS else LAYOUT_FLAT + + +def docs_layout(docs_dir: str) -> str: + """Layout of existing docs (see :func:`read_layout`), or the default for new ones.""" + if os.path.exists(os.path.join(docs_dir, METADATA_FILENAME)): + return read_layout(docs_dir) + return DEFAULT_LAYOUT + + +def _candidate_relpaths(name: str, module_tree: dict[str, Any] | None) -> list[str]: + """Places a page named ``name`` may be: its nested path, then the root.""" + if name == OVERVIEW_STEM: + return [OVERVIEW_FILENAME] + candidates = [] + for path in iter_tree_paths(module_tree): + if path[-1] == name: + candidates.append(module_doc_relpath(path, LAYOUT_HIERARCHICAL)) + break + flat = f"{name}.md" + if flat not in candidates: + candidates.append(flat) + return candidates + + +def find_doc( + docs_dir: str, + name: str, + module_tree: dict[str, Any] | None = None, + search: bool = False, +) -> str | None: + """Absolute path of the existing page for ``name``, in either layout. + + ``search`` also walks the docs folders for ``{name}.md``, for a page whose + module was already dropped from the tree (the updater removes those). + """ + if module_tree is None: + module_tree = load_module_tree(docs_dir) + for rel in _candidate_relpaths(name, module_tree): + path = os.path.join(docs_dir, rel) + if os.path.isfile(path): + return path + if search and name != OVERVIEW_STEM: + target = f"{name}.md" + for root, dirs, files in os.walk(docs_dir): + dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d != TEMP_DIR) + if target in files: + return os.path.join(root, target) + return None + + +def list_doc_files(docs_dir: str, module_tree: dict[str, Any] | None = None) -> dict[str, str]: + """Map page stem -> relative path for every page in the docs directory. + + Covers every ``.md`` in the docs root plus the nested page of each module + in the tree. Other nested Markdown (e.g. a user's own ``docs/guides``) + is deliberately not treated as a page. + """ + pages: dict[str, str] = {} + try: + for entry in os.listdir(docs_dir): + if entry.endswith(".md") and not entry.startswith("."): + if os.path.isfile(os.path.join(docs_dir, entry)): + pages[entry[: -len(".md")]] = entry + except OSError: + return {} + if module_tree is None: + module_tree = load_module_tree(docs_dir) + for path in iter_tree_paths(module_tree): + rel = module_doc_relpath(path, LAYOUT_HIERARCHICAL) + if "/" in rel and os.path.isfile(os.path.join(docs_dir, rel)): + pages[path[-1]] = rel + return pages + + +def doc_path_map(docs_dir: str, module_tree: dict[str, Any] | None = None) -> dict[str, str]: + """Page stem -> relative path for viewers: where each page is, else where it belongs.""" + if module_tree is None: + module_tree = load_module_tree(docs_dir) + paths = doc_relpaths(module_tree, read_layout(docs_dir)) + paths.update(list_doc_files(docs_dir, module_tree)) + return paths + + +def target_doc_path( + docs_dir: str, name: str, layout: str | None, module_tree: dict[str, Any] | None = None +) -> str: + """Absolute path where the page for ``name`` should be written.""" + if name == OVERVIEW_STEM: + return os.path.join(docs_dir, OVERVIEW_FILENAME) + if module_tree is None: + module_tree = load_module_tree(docs_dir) + for path in iter_tree_paths(module_tree): + if path[-1] == name: + return os.path.join(docs_dir, module_doc_relpath(path, layout)) + return os.path.join(docs_dir, f"{name}.md") + + +def relative_link(from_rel: str, to_rel: str) -> str: + """Relative link from page ``from_rel`` to page ``to_rel`` (both docs-relative).""" + return posixpath.relpath(to_rel, posixpath.dirname(from_rel) or ".") + + +# [text](target.md#anchor "title") — target without scheme, spaces or '#' +_LINK_RE = re.compile( + r"(\[[^\]\n]*\]\(\s*#]+\.md)((?:#[^)\s>]*)?>?(?:\s+\"[^\"]*\")?\s*\))" +) +_FENCE_RE = re.compile(r"^\s*(```|~~~)") + + +def rewrite_page_links(text: str, page_rel: str, relpaths: dict[str, str]) -> str: + """Point every link to a known page at its correct path relative to ``page_rel``.""" + lower = {k.lower(): v for k, v in relpaths.items()} + + def fix(m: re.Match) -> str: + target = m.group(2) + if "://" in target or target.startswith("/"): + return m.group(0) + stem = posixpath.basename(unquote(target))[: -len(".md")] + dest = relpaths.get(stem) or lower.get(stem.lower()) + if dest is None: + return m.group(0) + return f"{m.group(1)}{relative_link(page_rel, dest)}{m.group(3)}" + + out = [] + in_fence = False + for line in text.splitlines(keepends=True): + if _FENCE_RE.match(line): + in_fence = not in_fence + out.append(line) + elif in_fence or ".md" not in line: + out.append(line) + else: + out.append(_LINK_RE.sub(fix, line)) + return "".join(out) + + +def relocate_docs(docs_dir: str, module_tree: dict[str, Any], layout: str | None) -> list[str]: + """Move pages found in the other layout's place to where ``layout`` expects them.""" + moved = [] + for path in iter_tree_paths(module_tree): + target_rel = module_doc_relpath(path, layout) + target = os.path.join(docs_dir, target_rel) + if os.path.isfile(target): + continue + for rel in ( + module_doc_relpath(path, LAYOUT_HIERARCHICAL), + module_doc_relpath(path, LAYOUT_FLAT), + ): + source = os.path.join(docs_dir, rel) + if rel != target_rel and os.path.isfile(source): + os.makedirs(os.path.dirname(target), exist_ok=True) + os.replace(source, target) + _remove_empty_parents(docs_dir, source) + moved.append(target_rel) + logger.info("Moved %s to %s", rel, target_rel) + break + return moved + + +def organize_docs( + docs_dir: str, layout: str | None, module_tree: dict[str, Any] | None = None +) -> None: + """Put every page where ``layout`` expects it and fix links between pages. + + Agents occasionally save a page at the docs root instead of its nested + path, or compute a relative link wrongly; this repairs both. + """ + if module_tree is None: + module_tree = load_module_tree(docs_dir) + relocate_docs(docs_dir, module_tree, layout) + relpaths = doc_relpaths(module_tree, layout) + for stem, rel in list_doc_files(docs_dir, module_tree).items(): + relpaths.setdefault(stem, rel) + for rel in sorted(set(list_doc_files(docs_dir, module_tree).values())): + path = os.path.join(docs_dir, rel) + try: + with open(path, encoding="utf-8") as f: + text = f.read() + except (OSError, UnicodeDecodeError): + continue + fixed = rewrite_page_links(text, rel, relpaths) + if fixed != text: + with open(path, "w", encoding="utf-8") as f: + f.write(fixed) + + +def remove_doc(docs_dir: str, path: str) -> None: + """Delete a page and the folders its removal leaves empty (up to the docs root).""" + os.remove(path) + _remove_empty_parents(docs_dir, path) + + +def _remove_empty_parents(docs_dir: str, path: str) -> None: + root = os.path.abspath(docs_dir) + parent = os.path.dirname(os.path.abspath(path)) + while parent != root and parent.startswith(root + os.sep): + try: + os.rmdir(parent) + except OSError: + break + parent = os.path.dirname(parent) diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index add1392d..eddaaecd 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -15,6 +15,13 @@ ) from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index +from codewiki.src.be.doc_layout import ( + config_layout, + list_doc_files, + module_doc_relpath, + organize_docs, + relative_link, +) from codewiki.src.be.module_naming import ( dedupe_module_tree_names, find_missing_module_docs, @@ -75,6 +82,8 @@ def create_documentation_metadata( "commit_id": self.commit_id, # Read back by `--update` so updated pages keep this language "language": getattr(self.config, "language", None), + # Read back by `--update` and the viewers to locate pages + "layout": config_layout(self.config), }, "statistics": { "total_components": len(components), @@ -84,10 +93,10 @@ def create_documentation_metadata( "files_generated": ["overview.md", "module_tree.json", "first_module_tree.json"], } - # Add generated markdown files to the metadata + # Add generated markdown files (docs-relative paths) to the metadata try: - for file_path in os.listdir(working_dir): - if file_path.endswith(".md") and file_path not in metadata["files_generated"]: + for file_path in sorted(list_doc_files(working_dir).values()): + if file_path not in metadata["files_generated"]: metadata["files_generated"].append(file_path) except Exception as e: # noqa: BLE001 — metadata listing is best-effort logger.warning(f"Could not list generated files: {e}") @@ -154,14 +163,16 @@ def build_overview_structure( if "children" in module_info: module_info = module_info["children"] + layout = config_layout(self.config) + page_rel = module_doc_relpath(module_path, layout) for child_name, child_info in module_info.items(): - child_docs_path = self._resolve_child_docs_path(working_dir, child_name) + child_rel = module_doc_relpath(module_path + [child_name], layout) + child_docs_path = self._resolve_child_docs_path(working_dir, child_name, module_tree) if child_docs_path is not None: child_info["docs_path"] = child_docs_path + child_info["link"] = relative_link(page_rel, child_rel) else: - logger.warning( - f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}" - ) + logger.warning(f"Module docs not found at {os.path.join(working_dir, child_rel)}") child_info["docs_path"] = None return processed_module_tree @@ -179,7 +190,9 @@ def _strip_components(cls, tree: dict[str, Any]) -> None: cls._strip_components(children) @staticmethod - def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None: + def _resolve_child_docs_path( + working_dir: str, child_name: str, module_tree: dict[str, Any] | None = None + ) -> str | None: """Resolve the on-disk path for a child module's .md doc. Sub-agents sometimes save files under a sanitized variant of the @@ -188,7 +201,7 @@ def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None: before giving up so the overview prompt still gets the children's content as context. """ - return resolve_module_doc_path(working_dir, child_name) + return resolve_module_doc_path(working_dir, child_name, module_tree) def validate_generated_docs(self, working_dir: str) -> list[str]: """Check the final module tree against the docs on disk. @@ -299,6 +312,9 @@ async def generate_module_documentation( if os.path.exists(repo_overview_path): os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) + # Move pages an agent saved in the wrong folder and fix links between pages + organize_docs(working_dir, config_layout(self.config)) + return working_dir async def generate_parent_module_docs( @@ -328,9 +344,14 @@ async def generate_parent_module_docs( # path below resolves to overview.md, so a blanket "overview exists → # skip" check is not needed here and would mask missing parent docs # on resume) + existing_docs_path = ( + resolve_module_doc_path(working_dir, module_name, module_tree) if module_path else None + ) + if existing_docs_path is not None: + logger.info(f"✓ Parent docs already exists at {existing_docs_path}") + return module_tree parent_docs_path = os.path.join( - working_dir, - f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md", + working_dir, module_doc_relpath(module_path, config_layout(self.config)) ) if os.path.exists(parent_docs_path): logger.info(f"✓ Parent docs already exists at {parent_docs_path}") @@ -379,6 +400,7 @@ async def generate_parent_module_docs( f"using raw response as markdown." ) parent_content = parent_docs.strip() + os.makedirs(os.path.dirname(parent_docs_path), exist_ok=True) file_manager.save_text(parent_content, parent_docs_path) logger.debug(f"Successfully generated parent documentation for: {module_name}") diff --git a/codewiki/src/be/module_naming.py b/codewiki/src/be/module_naming.py index 23a0a1a2..f64a8f5f 100644 --- a/codewiki/src/be/module_naming.py +++ b/codewiki/src/be/module_naming.py @@ -1,10 +1,12 @@ """Utilities for keeping LLM-chosen module names unique and file-safe. -All module docs live in one flat directory as ``{module_name}.md``, and the -module-tree key must stay equal to the filename stem (the HTML viewer and -``--update`` invalidation rely on it). Names are chosen freely by the LLM at -every hierarchy level, so collisions must be resolved before a name is -inserted into the tree (issue #76). +Every module doc is saved as ``{module_name}.md`` (in a folder mirroring the +module tree, or in the docs root with ``--flat``; see ``doc_layout``), and the +module-tree key must stay equal to the filename stem (the HTML viewer, link +repair and ``--update`` invalidation rely on it). Names are kept unique across +the whole tree in both layouts. Names are chosen freely by the LLM at every +hierarchy level, so collisions must be resolved before a name is inserted +into the tree (issue #76). """ import os @@ -13,12 +15,24 @@ import logging +from codewiki.src.be.doc_layout import ( + OVERVIEW_STEM, + find_doc, + list_doc_files, + module_doc_file, + module_doc_relpath, + relative_link, +) + logger = logging.getLogger(__name__) # Filename stems used by CodeWiki itself; never assign them to a module. RESERVED_STEMS = {"overview", "module_tree", "first_module_tree", "metadata", "index"} -_UNSAFE_FILENAME_CHARS = set('/\\:*?"<>|\0') +# ``&`` is legal in filenames, but agents running shell tools escape it +# (``A_\\&_B/``) and write pages into a stray folder; nested layouts make +# that common, so module names never contain it. +_UNSAFE_FILENAME_CHARS = set('/\\:*?"<>|\0&') def sanitize_module_name(name: str) -> str: @@ -60,12 +74,7 @@ def resolve_unique_name(name: str, parent_name: Optional[str], taken: Set[str]) def _existing_doc_stems(working_dir: str) -> Set[str]: - try: - return { - os.path.splitext(entry)[0] for entry in os.listdir(working_dir) if entry.endswith(".md") - } - except OSError: - return set() + return set(list_doc_files(working_dir)) def normalize_sub_module_specs( @@ -77,7 +86,7 @@ def normalize_sub_module_specs( """Map requested sub-module names to unique, file-safe final names. A name is taken if it already appears anywhere in the module tree, if a - ``.md`` with that stem exists in the flat docs dir, if it is reserved, or + page with that stem exists in the docs dir, if it is reserved, or if it was assigned earlier in this batch. """ taken = collect_module_tree_names(module_tree) @@ -158,6 +167,52 @@ def plan_sub_module_specs( return plan +def sub_module_report( + name_map: dict[str, str], + skipped: dict[str, str], + docs_dir: str, + module_tree: dict, + parent_name: str, + parent_path: list[str], + layout: str, +) -> str: + """Report what actually landed on disk so the parent agent links real files. + + Each saved page is given as the link to use from the parent's page. + """ + parent_doc = module_doc_file(parent_name, parent_path, layout) + saved = [] + missing = [] + for requested_name, final_name in name_map.items(): + entry = relative_link(parent_doc, module_doc_relpath(parent_path + [final_name], layout)) + if final_name != requested_name: + entry += f" (requested '{requested_name}', renamed to avoid a collision)" + if find_doc(docs_dir, final_name, module_tree) is not None: + saved.append(entry) + else: + missing.append(entry) + + report = f"Saved documentations (link them from `{parent_doc}` as): " + report += f"{', '.join(saved) if saved else 'none'}." + if missing: + report += f" MISSING (generation did not produce these files): {', '.join(missing)}." + logger.warning("Sub-module documentation missing after generation: %s", ", ".join(missing)) + if skipped: + report += " " + skipped_report(skipped, parent_doc) + return report + + +def skipped_report(skipped: dict[str, str], parent_doc: str) -> str: + """Tell the parent agent, unambiguously, not to retry skipped sub-modules.""" + if not skipped: + return "No sub-modules were generated." + items = ", ".join(f"'{name}' ({reason})" for name, reason in skipped.items()) + return ( + f"Skipped sub-modules: {items}. Do NOT call generate_sub_module_documentation again " + f"for these; link the existing pages from `{parent_doc}` instead." + ) + + def dedupe_module_tree_names(module_tree: Dict[str, Any]) -> Dict[str, Any]: """Sanitize and uniquify all module names in a freshly clustered tree. @@ -181,13 +236,18 @@ def dedupe_level(level: Dict[str, Any], parent_name: Optional[str]) -> Dict[str, return dedupe_level(module_tree, None) -def resolve_module_doc_path(working_dir: str, module_name: str) -> Optional[str]: - """Resolve the on-disk path for a module's .md doc. +def resolve_module_doc_path( + working_dir: str, module_name: str, module_tree: Optional[Dict[str, Any]] = None +) -> Optional[str]: + """Resolve the on-disk path for a module's .md doc, in either layout. Sub-agents sometimes save files under a sanitized variant of the module name (spaces → underscores, lowercased, etc.) rather than the exact key in the module tree. Try a small set of common variants before giving up. """ + found = find_doc(working_dir, module_name, module_tree) + if found is not None: + return found candidates = [] seen = set() base_variants = [ @@ -217,7 +277,7 @@ def find_missing_module_docs( """Return module names from the tree whose docs are missing on disk.""" missing = [] for name in sorted(collect_module_tree_names(module_tree)): - if resolve_module_doc_path(working_dir, name) is None: + if resolve_module_doc_path(working_dir, name, module_tree) is None: missing.append(name) if overview_required and not os.path.exists(os.path.join(working_dir, "overview.md")): missing.append("overview") @@ -247,12 +307,8 @@ def extract_page_titles(working_dir: str, module_tree: Dict[str, Any]) -> Dict[s Pages that are missing or have no H1 are left out. """ titles = {} - for name in ["overview", *sorted(collect_module_tree_names(module_tree))]: - path = ( - os.path.join(working_dir, "overview.md") - if name == "overview" - else resolve_module_doc_path(working_dir, name) - ) + for name in [OVERVIEW_STEM, *sorted(collect_module_tree_names(module_tree))]: + path = resolve_module_doc_path(working_dir, name, module_tree) title = _first_h1(path) if path and os.path.exists(path) else None if title: titles[name] = title diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index dfeb6b34..90263374 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -2,6 +2,8 @@ from collections import defaultdict from typing import Any +from codewiki.src.be.doc_layout import doc_relpaths, module_doc_file +from codewiki.src.config import LAYOUT_FLAT from codewiki.src.utils import file_manager SYSTEM_PROMPT = """ @@ -19,14 +21,14 @@ Generate documentation following this structure: -1. **Main Documentation File** (`{module_name}.md`): +1. **Main Documentation File** (`{doc_file}`): - Brief introduction and purpose - Architecture overview with diagrams - High-level functionality of each sub-module including references to its documentation file - Link to other module documentation instead of duplicating information 2. **Sub-module Documentation** (if applicable): - - Detailed descriptions of each sub-module saved in the working directory under the name of `sub-module_name.md` + - Detailed descriptions of each sub-module saved {sub_module_location} - Core components and their responsibilities 3. **Visual Documentation**: @@ -37,10 +39,10 @@ 1. Analyze the provided code components and module structure, explore the not given dependencies between the components if needed -2. Create the main `{module_name}.md` file with overview and architecture in working directory -3. Use `generate_sub_module_documentation` to generate detailed sub-modules documentation for COMPLEX modules which at least have more than 1 code file and are able to clearly split into sub-modules. Sub-module names must be unique across the whole wiki (all docs share one flat directory) — prefer names prefixed with the current module name, e.g. `{module_name}_search` +2. Create the main `{doc_file}` file with overview and architecture in working directory +3. Use `generate_sub_module_documentation` to generate detailed sub-modules documentation for COMPLEX modules which at least have more than 1 code file and are able to clearly split into sub-modules. Sub-module names must be unique across the whole wiki ({uniqueness_reason}) — prefer names prefixed with the current module name, e.g. `{module_name}_search` 4. Include relevant Mermaid diagrams throughout the documentation -5. After all sub-modules are documented, adjust `{module_name}.md` with ONLY ONE STEP to ensure all generated files including sub-modules documentation are properly cross-refered, using the final file names reported by `generate_sub_module_documentation` +5. After all sub-modules are documented, adjust `{doc_file}` with ONLY ONE STEP to ensure all generated files including sub-modules documentation are properly cross-refered, using the final file names reported by `generate_sub_module_documentation` @@ -73,7 +75,7 @@ 1. Analyze provided code components and module structure 2. Explore dependencies between components if needed -3. Generate complete {module_name}.md documentation file +3. Generate complete {doc_file} documentation file @@ -89,7 +91,7 @@ {module_tree} -* NOTE: You can refer the other modules in the module tree based on the dependencies between their core components to make the documentation more structured and avoid repeating the same information. Know that all documentation files are saved in the same folder not structured as module tree. e.g. [alt text]([ref_module_name].md) +* NOTE: You can refer the other modules in the module tree based on the dependencies between their core components to make the documentation more structured and avoid repeating the same information. {formatted_core_component_codes} @@ -109,7 +111,7 @@ {repo_structure} -The core modules' documentation is NOT inlined above. Each top-level module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). +The core modules' documentation is NOT inlined above. Each top-level module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). When referencing a child's documentation, link it with the relative path in its `link` field, e.g. [alt text](). Please generate the overview of the `{repo_name}` repository in markdown format with the following structure: @@ -130,7 +132,7 @@ {repo_structure} -The child modules' documentation is NOT inlined above. Each child of the target module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). +The child modules' documentation is NOT inlined above. Each child of the target module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). When referencing a child's documentation, link it with the relative path in its `link` field, e.g. [alt text](). Please generate the overview of the `{module_name}` module in markdown format with the following structure: @@ -287,6 +289,22 @@ "file-reading tools to read the full files]" ) +# How pages link to each other, appended to the user prompt (after the module +# tree) by layout. Kept out of USER_PROMPT so the MCP prompt server, which +# formats USER_PROMPT directly, keeps working. +FLAT_LINKS_NOTE = ( + "* NOTE: Know that all documentation files are saved in the same folder not structured " + "as module tree. e.g. [alt text]([ref_module_name].md)" +) + +HIERARCHICAL_LINKS_NOTE = ( + "* NOTE: Documentation files are organized in folders mirroring the module tree; each " + "module above is listed with its page path (relative to the docs working directory). " + "Your page is `{doc_file}`{children_hint}. Link to other pages with paths relative to " + "your page, e.g. from `a/b.md` to `c.md` write [alt text](../c.md) and to `a/b/d.md` " + "write [alt text](b/d.md)." +) + # Appended to the user prompt (after USER_PROMPT) when the dependency graph # contains artifact nodes. Kept out of USER_PROMPT itself so callers that # format the template directly (MCP prompt server) keep working. @@ -360,26 +378,67 @@ } +def is_flat_layout(layout: str | None) -> bool: + return layout == LAYOUT_FLAT + + +def _layout_fields(module_name: str, module_path: list[str] | None, layout: str | None) -> dict: + doc_file = module_doc_file(module_name, module_path, layout) + if is_flat_layout(layout): + return { + "doc_file": doc_file, + "sub_module_location": "in the working directory under the name of `sub-module_name.md`", + "uniqueness_reason": "all docs share one flat directory", + } + folder = doc_file[: -len(".md")] + "/" if module_path else "" + location = ( + f"by `generate_sub_module_documentation` under `{folder}` as `sub-module_name.md`" + if folder + else "by `generate_sub_module_documentation` as `sub-module_name.md`" + ) + return { + "doc_file": doc_file, + "sub_module_location": location, + "uniqueness_reason": "the module name is the page's filename", + } + + +def format_links_note(module_name: str, module_path: list[str] | None, layout: str | None) -> str: + """How the agent documenting ``module_name`` names and links pages.""" + if is_flat_layout(layout): + return FLAT_LINKS_NOTE + doc_file = module_doc_file(module_name, module_path, layout) + children_hint = ( + f" and its sub-module pages go in `{doc_file[: -len('.md')]}/`" if module_path else "" + ) + return HIERARCHICAL_LINKS_NOTE.format(doc_file=doc_file, children_hint=children_hint) + + def _format_module_tree_str( module_tree: dict[str, Any], current_module_name: str | None = None, include_components: bool = True, + doc_paths: dict[str, str] | None = None, ) -> str: """ Render a module tree as an indented text outline. With include_components=False only module names and hierarchy are emitted, which keeps the outline small enough for huge trees that would - otherwise blow past MAX_USER_PROMPT_CHARS. + otherwise blow past MAX_USER_PROMPT_CHARS. ``doc_paths`` (module name -> + page path) adds each module's page path, for the hierarchical layout. """ lines: list[str] = [] def _walk(tree: dict[str, Any], indent: int = 0) -> None: for key, value in tree.items(): + label = key + if doc_paths and key in doc_paths: + label += f" [page: {doc_paths[key]}]" if key == current_module_name: - lines.append(f"{' ' * indent}{key} (current module)") + lines.append(f"{' ' * indent}{label} (current module)") else: - lines.append(f"{' ' * indent}{key}") + lines.append(f"{' ' * indent}{label}") if include_components: # Group components by file @@ -447,6 +506,8 @@ def format_user_prompt( core_component_ids: list[str], components: dict[str, Any], module_tree: dict[str, any], + module_path: list[str] | None = None, + layout: str | None = LAYOUT_FLAT, ) -> str: """ Format the user prompt with module name and organized core component codes. @@ -455,13 +516,19 @@ def format_user_prompt( module_name: Name of the module to document core_component_ids: List of component IDs to include components: Dictionary mapping component IDs to CodeComponent objects + module_tree: Current module tree + module_path: Path of the module in the tree (``[]`` for the whole repo) + layout: Docs layout; the hierarchical layout lists each module's page path Returns: Formatted user prompt string """ from codewiki.src.be.dependency_analyzer.analyzers.artifact import render_artifact_index - formatted_module_tree = _format_module_tree_str(module_tree, module_name) + doc_paths = None if is_flat_layout(layout) else doc_relpaths(module_tree, layout) + links_note = format_links_note(module_name, module_path, layout) + + formatted_module_tree = _format_module_tree_str(module_tree, module_name, doc_paths=doc_paths) # Group core component IDs by their file path grouped_components: dict[str, list[str]] = {} @@ -510,6 +577,7 @@ def _assemble(codes: str, tree: str) -> str: formatted_core_component_codes=codes, module_tree=tree, ) + + f"\n\n{links_note}" + artifact_section ) @@ -520,7 +588,9 @@ def _assemble(codes: str, tree: str) -> str: formatted_module_tree = ( MODULE_TREE_TRIMMED_NOTE + "\n\n" - + _format_module_tree_str(module_tree, module_name, include_components=False) + + _format_module_tree_str( + module_tree, module_name, include_components=False, doc_paths=doc_paths + ) ) prompt = _assemble(core_component_codes, formatted_module_tree) logger.warning( @@ -610,13 +680,20 @@ def format_super_group_prompt(module_tree: dict[str, Any]) -> str: return SUPER_GROUP_PROMPT.format(formatted_modules="\n".join(lines)) -def format_system_prompt(module_name: str, custom_instructions: str | None = None) -> str: +def format_system_prompt( + module_name: str, + custom_instructions: str | None = None, + module_path: list[str] | None = None, + layout: str | None = LAYOUT_FLAT, +) -> str: """ Format the system prompt with module name and optional custom instructions. Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append + module_path: Path of the module in the tree (``[]`` for the whole repo) + layout: Docs layout, which decides the page path the agent writes Returns: Formatted system prompt string @@ -625,16 +702,27 @@ def format_system_prompt(module_name: str, custom_instructions: str | None = Non if custom_instructions: custom_section = f"\n\n\n{custom_instructions}\n" - return SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() + return SYSTEM_PROMPT.format( + module_name=module_name, + custom_instructions=custom_section, + **_layout_fields(module_name, module_path, layout), + ).strip() -def format_leaf_system_prompt(module_name: str, custom_instructions: str | None = None) -> str: +def format_leaf_system_prompt( + module_name: str, + custom_instructions: str | None = None, + module_path: list[str] | None = None, + layout: str | None = LAYOUT_FLAT, +) -> str: """ Format the leaf system prompt with module name and optional custom instructions. Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append + module_path: Path of the module in the tree (``[]`` for the whole repo) + layout: Docs layout, which decides the page path the agent writes Returns: Formatted leaf system prompt string @@ -644,7 +732,9 @@ def format_leaf_system_prompt(module_name: str, custom_instructions: str | None custom_section = f"\n\n\n{custom_instructions}\n" return LEAF_SYSTEM_PROMPT.format( - module_name=module_name, custom_instructions=custom_section + module_name=module_name, + custom_instructions=custom_section, + doc_file=module_doc_file(module_name, module_path, layout), ).strip() diff --git a/codewiki/src/be/pydantic_ai_backend.py b/codewiki/src/be/pydantic_ai_backend.py index 3be7dada..9b7e6a99 100644 --- a/codewiki/src/be/pydantic_ai_backend.py +++ b/codewiki/src/be/pydantic_ai_backend.py @@ -32,6 +32,7 @@ format_user_prompt, ) from codewiki.src.be.utils import is_complex_module +from codewiki.src.be.doc_layout import config_layout, find_doc from codewiki.src.config import MODULE_TREE_FILENAME, OVERVIEW_FILENAME, Config from codewiki.src.utils import file_manager @@ -118,7 +119,9 @@ async def run_module_agent( if os.path.exists(overview_docs_path): logger.info("✓ Overview docs already exists at %s", overview_docs_path) return module_tree - docs_path = os.path.join(working_dir, f"{module_name}.md") + docs_path = find_doc(working_dir, module_name, module_tree) if module_path else None + if docs_path is None: + docs_path = os.path.join(working_dir, f"{module_name}.md") if os.path.exists(docs_path): logger.info("✓ Module docs already exists at %s", docs_path) return module_tree @@ -133,7 +136,9 @@ async def run_module_agent( str_replace_editor_tool, generate_sub_module_documentation_tool, ], - system_prompt=format_system_prompt(module_name, self._custom_instructions), + system_prompt=format_system_prompt( + module_name, self._custom_instructions, module_path, config_layout(config) + ), retries=config.agent_retries, ) else: @@ -142,7 +147,9 @@ async def run_module_agent( name=module_name, deps_type=CodeWikiDeps, tools=[read_code_components_tool, str_replace_editor_tool], - system_prompt=format_leaf_system_prompt(module_name, self._custom_instructions), + system_prompt=format_leaf_system_prompt( + module_name, self._custom_instructions, module_path, config_layout(config) + ), retries=config.agent_retries, ) @@ -167,6 +174,8 @@ async def run_module_agent( core_component_ids=core_component_ids, components=components, module_tree=deps.module_tree, + module_path=module_path, + layout=config_layout(config), ), deps=deps, usage_limits=UsageLimits(request_limit=config.request_limit), diff --git a/codewiki/src/be/updater/leaf_agent.py b/codewiki/src/be/updater/leaf_agent.py index feaa20ac..b1b684ea 100644 --- a/codewiki/src/be/updater/leaf_agent.py +++ b/codewiki/src/be/updater/leaf_agent.py @@ -8,6 +8,7 @@ from typing import Any from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be import doc_layout as L from codewiki.src.be.backend import LLMBackend from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.updater import pages as P @@ -63,10 +64,16 @@ def _deps( allowed_write_paths=allowed, ) + def _page_paths(self, roles: dict[str, list[str]]) -> dict[str, str]: + """Docs-relative path of every page in the write set and the tree.""" + paths = L.doc_relpaths(self.tree, L.read_layout(self.docs_dir)) + paths.update(L.list_doc_files(self.docs_dir)) + for stem in roles: + paths[stem] = P.page_rel(self.docs_dir, stem) + return paths + def _remove_page(self, stem: str, by_leaf: str, reason: str) -> None: - path = P.page_path(self.docs_dir, stem) - if os.path.exists(path): - os.remove(path) + if P.remove_page(self.docs_dir, stem): self.record.pages_removed.append(stem) self.record.add_verdict(PageVerdict(stem, "delete", reason, by_leaf, True)) @@ -74,10 +81,7 @@ async def _regenerate_leaf( self, leaf_name: str, module_path: list[str], component_ids: list[str], why: str ) -> None: """Delete the page (if any) and let the normal module agent write it anew.""" - path = P.page_path(self.docs_dir, leaf_name) - existed = os.path.exists(path) - if existed: - os.remove(path) + existed = P.remove_page(self.docs_dir, leaf_name) started = time.time() err = None try: @@ -100,12 +104,11 @@ async def _regenerate_leaf( err, ) ) + written = P.page_exists(self.docs_dir, leaf_name) self.record.add_verdict( - PageVerdict( - leaf_name, "rewrite" if existed else "create", why, leaf_name, os.path.exists(path) - ) + PageVerdict(leaf_name, "rewrite" if existed else "create", why, leaf_name, written) ) - if os.path.exists(path): + if written: self.record.pages_written.append(leaf_name) async def _run_editing_agent( @@ -134,6 +137,7 @@ async def _run_editing_agent( component_ids=component_ids, graph=self.graph, leaf_page_text=P.read_page(self.docs_dir, leaf_name) if leaf_name in roles else None, + page_paths=self._page_paths(roles), ) before = P.page_hashes(self.docs_dir) started = time.time() diff --git a/codewiki/src/be/updater/orchestrator.py b/codewiki/src/be/updater/orchestrator.py index f85c5563..a895107d 100644 --- a/codewiki/src/be/updater/orchestrator.py +++ b/codewiki/src/be/updater/orchestrator.py @@ -48,6 +48,7 @@ RoutingDecision, repair_tree, ) +from codewiki.src.be.doc_layout import organize_docs, read_layout from codewiki.src.config import MODULE_TREE_FILENAME, Config from codewiki.src.utils import file_manager @@ -74,6 +75,13 @@ def __init__( self.doc_generator = doc_generator self.opts = opts self.docs_dir = os.path.abspath(config.docs_dir) + # Updated pages keep the layout the docs were generated in. The config + # is shared with the doc generator and backend (their agents write the + # pages), so align it rather than keeping a second source of truth. + self.layout = read_layout(self.docs_dir) + if getattr(config, "layout", None) != self.layout: + logger.info("Docs use the %s layout; updating them in place", self.layout) + config.layout = self.layout self.repo_name = os.path.basename(os.path.normpath(config.repo_path)) self.whole_repo = False self._deleted_nodes: list[tuple[str, ...]] = [] @@ -167,12 +175,10 @@ def _recluster( done.add(parent) for p in old_units: stem = p[-1] - if P.page_exists(self.docs_dir, stem): - os.remove(P.page_path(self.docs_dir, stem)) + if P.remove_page(self.docs_dir, stem): removed_pages.add(stem) self.record.pages_removed.append(stem) - if P.page_exists(self.docs_dir, parent[-1]): - os.remove(P.page_path(self.docs_dir, parent[-1])) + if P.remove_page(self.docs_dir, parent[-1]): removed_pages.add(parent[-1]) self.record.pages_removed.append(parent[-1]) new_info = T.node_at(tree, parent) or {} @@ -444,7 +450,9 @@ async def _run( diff, old_graph, set(rec.pages_written), removed_all, replacements ) - # ---- Step 6.3: rebuild the reference index + # ---- Step 6.3: move misplaced pages, fix links, rebuild the reference index + if not self.whole_repo: + organize_docs(self.docs_dir, self.layout) new_index = build_reference_index( self.docs_dir, new_graph, None if self.whole_repo else new_tree ) diff --git a/codewiki/src/be/updater/pages.py b/codewiki/src/be/updater/pages.py index bf4f543e..f80407a5 100644 --- a/codewiki/src/be/updater/pages.py +++ b/codewiki/src/be/updater/pages.py @@ -1,17 +1,33 @@ -"""Small helpers over the flat docs directory (page stems <-> files, hashes).""" +"""Small helpers over the docs directory (page stems <-> files, hashes). + +Pages are addressed by stem (the module name). Where the file lives depends +on the docs layout recorded in ``metadata.json`` (see ``doc_layout``): an +existing page is found in either layout, a new one is placed where the +recorded layout expects it. +""" from __future__ import annotations import hashlib import os +from codewiki.src.be import doc_layout as L from codewiki.src.config import OVERVIEW_FILENAME OVERVIEW_STEM = OVERVIEW_FILENAME[: -len(".md")] def page_path(docs_dir: str, stem: str) -> str: - return os.path.join(docs_dir, f"{stem}.md") + tree = L.load_module_tree(docs_dir) + found = L.find_doc(docs_dir, stem, tree, search=True) + if found is not None: + return found + return L.target_doc_path(docs_dir, stem, L.read_layout(docs_dir), tree) + + +def page_rel(docs_dir: str, stem: str) -> str: + """Docs-relative POSIX path of the page (what agents pass to the editor).""" + return os.path.relpath(page_path(docs_dir, stem), docs_dir).replace(os.sep, "/") def page_exists(docs_dir: str, stem: str) -> bool: @@ -27,25 +43,29 @@ def read_page(docs_dir: str, stem: str) -> str | None: def list_pages(docs_dir: str) -> list[str]: - try: - return sorted( - f[:-3] for f in os.listdir(docs_dir) if f.endswith(".md") and not f.startswith(".") - ) - except OSError: - return [] + return sorted(L.list_doc_files(docs_dir)) def page_hashes(docs_dir: str) -> dict[str, str]: out = {} - for stem in list_pages(docs_dir): + for stem, rel in L.list_doc_files(docs_dir).items(): try: - with open(page_path(docs_dir, stem), "rb") as f: + with open(os.path.join(docs_dir, rel), "rb") as f: out[stem] = hashlib.sha1(f.read()).hexdigest() except OSError: continue return out +def remove_page(docs_dir: str, stem: str) -> bool: + """Delete the page (and any folder left empty); False when it did not exist.""" + path = page_path(docs_dir, stem) + if not os.path.isfile(path): + return False + L.remove_doc(docs_dir, path) + return True + + def changed_pages(before: dict[str, str], after: dict[str, str]) -> set[str]: """Pages created, removed, or whose bytes changed.""" return {s for s in set(before) | set(after) if before.get(s) != after.get(s)} diff --git a/codewiki/src/be/updater/prompts.py b/codewiki/src/be/updater/prompts.py index a2a2338e..af8f044e 100644 --- a/codewiki/src/be/updater/prompts.py +++ b/codewiki/src/be/updater/prompts.py @@ -31,7 +31,7 @@ sentence that is still true word for word. Do not reflow, restyle, or "improve" prose that the report does not touch. 3. Per page role: - - the LEAF PAGE ({leaf_name}.md): update sections, tables and diagrams that describe changed + - the LEAF PAGE (`{leaf_name}.md`, at the path given in the WRITE SET): update sections, tables and diagrams that describe changed components; add new components; remove deleted ones. An OWN entry marked "not listed by this module" is a nearby component the page never lists: fix only what its change makes wrong in the page's description, do not add a section for it. If the change is so large that the @@ -45,7 +45,9 @@ 4. Use `read_code_components` to read the fresh code of any component id when the diff alone is not enough. Use `str_replace_editor` with `working_dir="docs"` and `command="view"` to read a page before editing it. -5. Mermaid diagrams must stay valid. Links between pages are relative: `[text](page.md)`. +5. Mermaid diagrams must stay valid. Links between pages are relative to the linking page, + using the page paths shown in the WRITE SET and the MODULE TREE: `[text](page.md)`, + `[text](../page.md)`, `[text](sub/page.md)`. @@ -81,7 +83,7 @@ {leaf_components} - + {leaf_page} @@ -174,7 +176,7 @@ """.strip() STALE_FIX_USER_PROMPT = """ -Page to fix: `{page}.md` (view it with str_replace_editor, working_dir="docs"). +Page to fix: `{page_path}` (view it with str_replace_editor, working_dir="docs"). {items} @@ -255,11 +257,15 @@ def render_report(report: LeafReport, diff: GraphDiff) -> str: ) -def render_write_set(roles: dict[str, list[str]]) -> str: - """``roles``: page stem -> list of roles (leaf/ancestor/dependent/referrer).""" +def render_write_set(roles: dict[str, list[str]], page_paths: dict[str, str] | None = None) -> str: + """``roles``: page stem -> list of roles (leaf/ancestor/dependent/referrer). + + ``page_paths`` maps a stem to its docs-relative path (default ``.md``). + """ + page_paths = page_paths or {} lines = [] for page, rs in roles.items(): - lines.append(f"- {page}.md ({', '.join(rs)})") + lines.append(f"- {page_paths.get(page, f'{page}.md')} ({', '.join(rs)})") return "\n".join(lines) if lines else "(empty)" @@ -301,8 +307,12 @@ def render_leaf_components( return "\n".join(lines) -def render_tree_outline(tree: dict[str, Any], current: str | None) -> str: - return _format_module_tree_str(tree, current, include_components=False) +def render_tree_outline( + tree: dict[str, Any], current: str | None, page_paths: dict[str, str] | None = None +) -> str: + return _format_module_tree_str( + tree, current, include_components=False, doc_paths=page_paths or None + ) def format_update_system_prompt(leaf_name: str, custom_instructions: str | None) -> str: @@ -336,16 +346,23 @@ def format_update_user_prompt( component_ids: list[str], graph: dict[str, Node], leaf_page_text: str | None, + page_paths: dict[str, str] | None = None, ) -> str: + """``page_paths``: page stem -> docs-relative path, for nested docs layouts.""" changed = set(report.own) + page_paths = page_paths or {} return UPDATE_LEAF_USER_PROMPT.format( leaf_name=leaf_name, mode=mode, mode_note=MODE_NOTES.get(mode, MODE_NOTES["edit"]), - write_set=render_write_set(roles), + write_set=render_write_set(roles, page_paths), report=render_report(report, diff), - module_tree=render_tree_outline(tree, leaf_name), + # Paths only matter (and are only shown) when pages live in folders + module_tree=render_tree_outline( + tree, leaf_name, page_paths if any("/" in p for p in page_paths.values()) else None + ), leaf_components=render_leaf_components(component_ids, graph, changed), + leaf_page_path=page_paths.get(leaf_name, f"{leaf_name}.md"), leaf_page=( leaf_page_text if leaf_page_text is not None else "(page does not exist / was removed)" ), @@ -374,7 +391,9 @@ def format_routing_prompt( return template.format(module_tree=tree_outline, orphans="\n".join(blocks)) -def format_stale_prompt(page: str, items: list[dict[str, Any]]) -> str: +def format_stale_prompt( + page: str, items: list[dict[str, Any]], page_path: str | None = None +) -> str: lines = [] for it in items: kind = it.get("kind") @@ -390,4 +409,4 @@ def format_stale_prompt(page: str, items: list[dict[str, Any]]) -> str: ) else: lines.append(f"- {json.dumps(it)}") - return STALE_FIX_USER_PROMPT.format(page=page, items="\n".join(lines)) + return STALE_FIX_USER_PROMPT.format(page_path=page_path or f"{page}.md", items="\n".join(lines)) diff --git a/codewiki/src/be/updater/reference_index.py b/codewiki/src/be/updater/reference_index.py index 4b3d8543..47f34b50 100644 --- a/codewiki/src/be/updater/reference_index.py +++ b/codewiki/src/be/updater/reference_index.py @@ -12,6 +12,7 @@ from typing import Any from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.doc_layout import list_doc_files from codewiki.src.be.updater import tree as T INDEX_FILENAME = "reference_index.json" @@ -27,17 +28,6 @@ def index_path(docs_dir: str) -> str: return os.path.join(docs_dir, "temp", INDEX_FILENAME) -def _page_stems(docs_dir: str) -> list[str]: - try: - return sorted( - os.path.splitext(f)[0] - for f in os.listdir(docs_dir) - if f.endswith(".md") and not f.startswith(".") - ) - except OSError: - return [] - - def extract_references( text: str, known_ids: set[str], @@ -70,14 +60,14 @@ def build_reference_index( name = node.name if name and len(name) >= 3: known_names.setdefault(name, set()).add(cid) - pages = _page_stems(docs_dir) + pages = list_doc_files(docs_dir, tree) known_pages = set(pages) if tree is not None: known_pages |= {p[-1] for p, _ in T.iter_nodes(tree)} index: dict[str, dict[str, list[str]]] = {} - for stem in pages: + for stem, rel in sorted(pages.items()): try: - with open(os.path.join(docs_dir, f"{stem}.md"), encoding="utf-8") as f: + with open(os.path.join(docs_dir, rel), encoding="utf-8") as f: text = f.read() except OSError: continue diff --git a/codewiki/src/be/updater/stale_scan.py b/codewiki/src/be/updater/stale_scan.py index beb9cd9c..cd5a4613 100644 --- a/codewiki/src/be/updater/stale_scan.py +++ b/codewiki/src/be/updater/stale_scan.py @@ -136,7 +136,7 @@ async def _fix(self, stem: str, items: list[dict[str, Any]]) -> None: try: reply = await self.backend.run_update_agent( format_stale_fix_system_prompt(deps.custom_instructions), - format_stale_prompt(stem, items), + format_stale_prompt(stem, items, P.page_rel(self.docs_dir, stem)), deps, ) text, usage = reply.text, reply.usage diff --git a/codewiki/src/be/updater/tree.py b/codewiki/src/be/updater/tree.py index 1da90e0e..cc3b5478 100644 --- a/codewiki/src/be/updater/tree.py +++ b/codewiki/src/be/updater/tree.py @@ -248,7 +248,7 @@ def copy_tree(tree: dict[str, Any]) -> dict[str, Any]: def page_stem(path: Path) -> str: - """Module page file stem (docs are flat: ``.md``).""" + """Module page file stem (``.md``, in either docs layout).""" return path[-1] diff --git a/codewiki/src/be/updater/verdicts.py b/codewiki/src/be/updater/verdicts.py index 9f3e2695..50d99d1e 100644 --- a/codewiki/src/be/updater/verdicts.py +++ b/codewiki/src/be/updater/verdicts.py @@ -47,7 +47,7 @@ def parse_verdicts(text: str) -> tuple[dict[str, dict[str, str]], str]: verdicts: dict[str, dict[str, str]] = {} if isinstance(raw, dict): for page, v in raw.items(): - stem = str(page) + stem = str(page).replace("\\", "/").rsplit("/", 1)[-1] if stem.endswith(".md"): stem = stem[:-3] if isinstance(v, str): @@ -60,7 +60,7 @@ def parse_verdicts(text: str) -> tuple[dict[str, dict[str, str]], str]: elif isinstance(raw, list): for v in raw: if isinstance(v, dict) and "page" in v: - stem = str(v["page"]) + stem = str(v["page"]).replace("\\", "/").rsplit("/", 1)[-1] stem = stem[:-3] if stem.endswith(".md") else stem verdicts[stem] = { "verdict": str(v.get("verdict", "")).strip().lower(), diff --git a/codewiki/src/config.py b/codewiki/src/config.py index b14c8b1c..12176a45 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -16,6 +16,12 @@ FIRST_MODULE_TREE_FILENAME = "first_module_tree.json" MODULE_TREE_FILENAME = "module_tree.json" OVERVIEW_FILENAME = "overview.md" +# Docs layout: "hierarchical" mirrors the module tree in nested folders +# (``auth.md`` + ``auth/login.md``); "flat" keeps every page in the docs root +# (for small models that keep getting relative links wrong). +LAYOUT_HIERARCHICAL = "hierarchical" +LAYOUT_FLAT = "flat" +DEFAULT_LAYOUT = LAYOUT_HIERARCHICAL MAX_DEPTH = 2 # Default max token settings DEFAULT_MAX_TOKENS = 32_768 @@ -115,6 +121,8 @@ class Config: # Also read the root README and docs/ as a `prose` artifact class (off by # default: documentation without existing prose is the benchmark setting) with_prose: bool = False + # Docs layout (LAYOUT_HIERARCHICAL or LAYOUT_FLAT) + layout: str = DEFAULT_LAYOUT @property def artifact_exclude(self) -> list[str] | None: @@ -216,6 +224,7 @@ def from_args(cls, args: argparse.Namespace) -> "Config": cluster_model=CLUSTER_MODEL, fallback_model=FALLBACK_MODEL_1, use_gitignore=getattr(args, "use_gitignore", True), + layout=getattr(args, "layout", DEFAULT_LAYOUT), ) @classmethod @@ -246,6 +255,7 @@ def from_cli( artifacts_enabled: bool = True, artifact_token_budget: int = DEFAULT_ARTIFACT_TOKEN_BUDGET, with_prose: bool = False, + layout: str = DEFAULT_LAYOUT, ) -> "Config": """ Create configuration for CLI context. @@ -280,6 +290,7 @@ def from_cli( the dependency graph and document them artifact_token_budget: Total token budget for artifact file contents with_prose: Also read README and docs/ as a `prose` artifact class + layout: Docs layout, "hierarchical" (nested folders) or "flat" Returns: Config instance @@ -314,4 +325,5 @@ def from_cli( artifacts_enabled=artifacts_enabled, artifact_token_budget=artifact_token_budget, with_prose=with_prose, + layout=layout, ) diff --git a/codewiki/src/fe/routes.py b/codewiki/src/fe/routes.py index 2f92050c..11eaa861 100644 --- a/codewiki/src/fe/routes.py +++ b/codewiki/src/fe/routes.py @@ -19,6 +19,7 @@ from .templates import WEB_INTERFACE_TEMPLATE from .template_utils import render_template from .config import WebAppConfig +from codewiki.src.be.doc_layout import doc_path_map from codewiki.src.utils import file_manager @@ -235,9 +236,14 @@ async def serve_generated_docs( except Exception: pass - # Serve the requested file - file_path = docs_path / filename - if not file_path.exists(): + # Serve the requested file (pages may sit in nested folders; never + # outside the docs directory) + file_path = (docs_path / filename).resolve() + try: + file_path.relative_to(docs_path.resolve()) + except ValueError: + raise HTTPException(status_code=403, detail="Access denied") from None + if not file_path.is_file(): raise HTTPException(status_code=404, detail=f"File {filename} not found") try: @@ -255,6 +261,7 @@ async def serve_generated_docs( "title": title, "content": html_content, "navigation": module_tree, + "doc_paths": doc_path_map(str(docs_path), module_tree), "current_page": filename, "job_id": job_id, "metadata": metadata, diff --git a/codewiki/src/fe/templates.py b/codewiki/src/fe/templates.py index 6657c7e7..b4a4862d 100644 --- a/codewiki/src/fe/templates.py +++ b/codewiki/src/fe/templates.py @@ -617,8 +617,9 @@ {% set indent_class = 'nav-subsection' if depth > 0 else '' %} {% set indent_style = 'margin-left: ' + (depth * 15)|string + 'px;' if depth > 0 else '' %}
+ {% set page = doc_paths[key] if doc_paths and key in doc_paths else key + '.md' %} {% if data.components %} - + {{ key.replace('_', ' ').title() }} {% else %} diff --git a/codewiki/src/fe/visualise_docs.py b/codewiki/src/fe/visualise_docs.py index ed6d118e..1fdd61d8 100644 --- a/codewiki/src/fe/visualise_docs.py +++ b/codewiki/src/fe/visualise_docs.py @@ -23,6 +23,7 @@ from .template_utils import render_template from .templates import DOCS_VIEW_TEMPLATE +from codewiki.src.be.doc_layout import doc_path_map from codewiki.src.utils import file_manager app = FastAPI( @@ -139,6 +140,7 @@ async def index(): "title": title, "content": html_content, "navigation": MODULE_TREE, + "doc_paths": doc_path_map(str(DOCS_FOLDER), MODULE_TREE) if DOCS_FOLDER else None, "current_page": "overview.md", } @@ -187,6 +189,7 @@ async def serve_doc(filename: str): "title": title, "content": html_content, "navigation": MODULE_TREE, + "doc_paths": doc_path_map(str(DOCS_FOLDER), MODULE_TREE) if DOCS_FOLDER else None, "current_page": filename, } diff --git a/codewiki/src/language.py b/codewiki/src/language.py index f2316d7d..407ef516 100644 --- a/codewiki/src/language.py +++ b/codewiki/src/language.py @@ -73,7 +73,7 @@ def language_tag(language: str | None) -> str: LANGUAGE_DIRECTIVE = """ Write ALL documentation prose in {language}: page titles, headings, paragraphs, lists, table text, Mermaid node/edge labels and captions. Keep code, identifiers, file paths, CLI commands and API names exactly as they are. -Filenames, module names and link targets are fixed identifiers: never translate or rename them. Save the page under exactly the `.md` filename named above, even when it differs from a package or directory name; keep sub-module names as given; link to other pages as `[text](.md)`. Only the visible link text may be translated. +Filenames, module names and link targets are fixed identifiers: never translate or rename them. Save the page under exactly the `.md` path named above, even when it differs from a package or directory name; keep sub-module names as given; link to other pages by their exact `.md` file name (with its relative folder path when pages live in folders), e.g. `[text](.md)`. Only the visible link text may be translated. Start each page with a single `# ` heading that is the page title in {language}. """ diff --git a/codewiki/templates/github_pages/viewer_template.html b/codewiki/templates/github_pages/viewer_template.html index 3cb8f320..597e61d1 100644 --- a/codewiki/templates/github_pages/viewer_template.html +++ b/codewiki/templates/github_pages/viewer_template.html @@ -969,6 +969,35 @@

Generation Info

document.getElementById('nav-overview').textContent = '📄 ' + PAGE_TITLES.overview; } const DOCS_BASE_PATH = '{{DOCS_BASE_PATH}}'; + // Module name -> page path relative to the docs folder. Pages may sit in + // folders mirroring the module tree (auth.md, auth/login.md) or all in + // the docs root (--flat); a name missing here falls back to .md. + const DOC_PATHS = {{DOC_PATHS_JSON}}; + + function docFileFor(key) { + return Object.prototype.hasOwnProperty.call(DOC_PATHS, key) ? DOC_PATHS[key] : key + '.md'; + } + + // Resolve a relative link found on page `fromFile` to a docs-relative path. + function resolveDocLink(fromFile, href) { + const parts = (fromFile || '').split('/').slice(0, -1).concat(href.split('/')); + const out = []; + for (const part of parts) { + if (part === '..') { + out.pop(); + } else if (part && part !== '.') { + out.push(part); + } + } + const resolved = out.join('/'); + const known = Object.values(DOC_PATHS); + if (known.length === 0 || known.indexOf(resolved) !== -1) { + return resolved; + } + // A link that misses its page: match the page by module name. + const stem = resolved.split('/').pop().replace(/\.md$/i, ''); + return Object.prototype.hasOwnProperty.call(DOC_PATHS, stem) ? DOC_PATHS[stem] : resolved; + } // Router / rendering state let currentFile = null; @@ -1126,7 +1155,7 @@

Generation Info

} function buildNavItem(key, data, depth) { - const fileName = key + '.md'; + const fileName = docFileFor(key); const hasChildren = data.children && Object.keys(data.children).length > 0; const hasComponents = data.components && data.components.length > 0; @@ -1297,11 +1326,12 @@

Generation Info

// Relative markdown links, optionally with an anchor // (e.g. "Other_Module.md#some-section") if (/^https?:/i.test(href)) return; - const mdMatch = href.match(/([^\/]*\.md)(?:#(.*))?$/i); + if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.charAt(0) === '/') return; + const mdMatch = href.match(/^([^#?]*\.md)(?:#(.*))?$/i); if (mdMatch) { e.preventDefault(); e.stopPropagation(); - const filename = safeDecode(mdMatch[1]); + const filename = resolveDocLink(currentFile || 'overview.md', safeDecode(mdMatch[1])); const anchor = mdMatch[2] ? safeDecode(mdMatch[2]) : null; navigateTo(filename, anchor); } @@ -1647,7 +1677,7 @@

Generation Info

const hasComponents = data.components && data.components.length > 0; // Mirror buildNavItem: skip bare headers without a doc if (depth > 0 || hasChildren || hasComponents) { - order.push(key + '.md'); + order.push(docFileFor(key)); } if (hasChildren) { walk(data.children, depth + 1); @@ -1664,7 +1694,7 @@

Generation Info

const DOC_ORDER = flattenModuleTree(); function pageTitleFor(filename) { - const base = filename.replace(/\.md$/i, ''); + const base = filename.split('/').pop().replace(/\.md$/i, ''); if (base === 'overview') { return PAGE_TITLES.overview || 'Overview'; } diff --git a/guides/cli-reference.md b/guides/cli-reference.md index 67cb5fc1..d5b76c19 100644 --- a/guides/cli-reference.md +++ b/guides/cli-reference.md @@ -44,6 +44,7 @@ the repository first. | `--instructions TEXT` | none | Free-form instructions passed to the documentation agent | | `--language`, `-l LANG` | English | Language of the generated docs, as a code or name (`ja`, `Japanese`, `vi`, `zh`, ...). See [Output language](#output-language) | | `--use-gitignore` / `--no-gitignore` | enabled | Respect root and nested `.gitignore` files | +| `--flat` | off | Save every page in the output root instead of folders mirroring the module tree. See [Docs layout](#docs-layout) | Pattern rules: @@ -69,6 +70,32 @@ The language is stored in `metadata.json`. `--update` reuses it, and stops with an error if `--language` names a different one. To switch language, regenerate without `--update`. +### Docs layout + +Pages mirror the module tree. A module's page sits next to the folder that +holds its sub-modules, and `overview.md` stays at the root: + +``` +docs/overview.md +docs/auth.md # links to auth/login.md, auth/session.md +docs/auth/login.md +docs/auth/session.md +docs/auth/session/store.md +docs/billing.md +``` + +Links between pages are relative to the linking page. After each run, +CodeWiki moves any page an agent saved in the wrong folder and repairs links +that point to the wrong path. + +`--flat` keeps every page in the output root as `.md`, the layout +of 2.0 and earlier. Use it with small models that keep getting relative links wrong. +Module names are unique across the wiki in both layouts. + +The layout is stored in `metadata.json`, and `--update` keeps it. Docs with +no stored layout were generated flat, and they stay flat. To switch layout, +regenerate without `--update`. + ### Artifact-aware generation (new in 2.0) Build, CI, container, packaging, manifest, configuration, schema, and script @@ -126,6 +153,7 @@ codewiki generate --include "*.cs" --exclude "Tests,Specs,*.test.cs" codewiki generate --focus "src/core,src/api" --doc-type architecture codewiki generate --instructions "Focus on public APIs and include usage examples" codewiki generate --language ja # docs in Japanese +codewiki generate --flat # all pages in ./docs, no folders codewiki generate --max-tokens 16384 --max-depth 3 ``` diff --git a/skills/codewiki-wiki-generator/SKILL.md b/skills/codewiki-wiki-generator/SKILL.md index 85ea1951..0aae9092 100644 --- a/skills/codewiki-wiki-generator/SKILL.md +++ b/skills/codewiki-wiki-generator/SKILL.md @@ -95,11 +95,11 @@ Read `processing_order.json` to get the processing order. **Process leaf modules **For each leaf module** (is_leaf=true): -1. Get system prompt: `get_prompt` → `{"prompt_type": "system_leaf", "variables": {"module_name": ""}}` +1. Get system prompt: `get_prompt` → `{"prompt_type": "system_leaf", "variables": {"module_name": "", "module_path": }}` 2. Read source code: `read_code_components` → all component IDs in this module, then read files under `sources/` 3. For additional context, use your file reading tools directly to read relevant source files in the repository -4. Write documentation including: module introduction and core functionality, architecture diagram (at least 1 Mermaid diagram), component responsibility descriptions, cross-references `[Module Name](module_name.md)` -5. Save: `write_doc_file` → `{"session_id": "...", "filename": ".md", "content": "..."}` +4. Write documentation including: module introduction and core functionality, architecture diagram (at least 1 Mermaid diagram), component responsibility descriptions, cross-references to other modules' pages (paths relative to this page, see below) +5. Save: `write_doc_file` → `{"session_id": "...", "filename": "", "content": "..."}` If Mermaid validation fails, fix the syntax and retry with `edit_doc_file` (`command: "str_replace"`). @@ -108,7 +108,7 @@ If Mermaid validation fails, fix the syntax and retry with `edit_doc_file` (`com 1. Read all child modules' generated `.md` files directly using your file reading tools 2. Get overview prompt: `get_prompt` → `{"prompt_type": "overview_module", "variables": {"module_name": ""}}` 3. Synthesize child module documentation into a parent module overview -4. Save with `write_doc_file` +4. Save with `write_doc_file` at the module's `doc_path` ### Phase 4: Generate Repository Overview @@ -158,7 +158,8 @@ The granularity of incremental updates is **module-level** — if any component - **Language**: Write in English by default (unless the user specifies another language) - **Mermaid diagrams**: At least 1 architecture diagram per module, prefer `graph TD` or `graph LR` -- **Cross-references**: Use `[Module Name](module_name.md)` format when referencing other modules +- **Page layout**: Pages mirror the module tree. A module's page sits next to the folder holding its children (`auth.md`, `auth/login.md`, `auth/login/tokens.md`); `overview.md` stays at the root. Each entry of `processing_order.json` gives its `doc_path` +- **Cross-references**: Link with paths relative to the linking page, e.g. from `auth/login.md` to `billing.md` write `[Billing](../billing.md)`, and from `auth.md` to `auth/login.md` write `[Login](auth/login.md)`. Links are re-checked and repaired when the session closes - **Code examples**: Show signatures and brief usage for key functions/classes - **Length**: Leaf module docs 200-500 lines, parent module overviews 100-300 lines, repository overview 80-200 lines diff --git a/tests/test_doc_layout.py b/tests/test_doc_layout.py new file mode 100644 index 00000000..68358085 --- /dev/null +++ b/tests/test_doc_layout.py @@ -0,0 +1,360 @@ +"""Docs layout (issue #125): pages mirror the module tree by default, or sit flat.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +from codewiki.src.be import doc_layout as L +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.be.module_naming import find_missing_module_docs, sub_module_report +from codewiki.src.be.prompt_template import ( + format_leaf_system_prompt, + format_system_prompt, + format_user_prompt, +) +from codewiki.src.be.updater import pages as P + +TREE = { + "auth": { + "components": ["a.py::A"], + "children": { + "login": {"components": ["a.py::login"], "children": {}}, + "session": { + "components": ["a.py::S"], + "children": {"store": {"components": ["a.py::store"], "children": {}}}, + }, + }, + }, + "billing": {"components": ["b.py::B"], "children": {}}, +} + + +def _write(docs: Path, rel: str, text: str = "# page\n") -> Path: + path = docs / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _setup(docs: Path, layout: str | None = "hierarchical") -> None: + docs.mkdir(exist_ok=True) + (docs / "module_tree.json").write_text(json.dumps(TREE)) + if layout is not None: + (docs / "metadata.json").write_text(json.dumps({"generation_info": {"layout": layout}})) + + +# --------------------------------------------------------------------- paths + + +def test_module_doc_relpath_per_layout(): + assert L.module_doc_relpath([], "hierarchical") == "overview.md" + assert L.module_doc_relpath(["auth"], "hierarchical") == "auth.md" + assert L.module_doc_relpath(["auth", "session", "store"], "hierarchical") == ( + "auth/session/store.md" + ) + assert L.module_doc_relpath(["auth", "session", "store"], "flat") == "store.md" + + +def test_doc_relpaths_cover_tree_and_overview(): + paths = L.doc_relpaths(TREE, "hierarchical") + assert paths == { + "overview": "overview.md", + "auth": "auth.md", + "login": "auth/login.md", + "session": "auth/session.md", + "store": "auth/session/store.md", + "billing": "billing.md", + } + assert L.doc_relpaths(TREE, "flat")["store"] == "store.md" + + +def test_read_layout_defaults_to_flat_for_old_docs(tmp_path): + assert L.read_layout(str(tmp_path)) == "flat" + (tmp_path / "metadata.json").write_text(json.dumps({"generation_info": {"language": None}})) + assert L.read_layout(str(tmp_path)) == "flat" + assert L.docs_layout(str(tmp_path)) == "flat" + (tmp_path / "metadata.json").write_text( + json.dumps({"generation_info": {"layout": "hierarchical"}}) + ) + assert L.read_layout(str(tmp_path)) == "hierarchical" + + +def test_docs_layout_defaults_to_hierarchical_for_new_docs(tmp_path): + assert L.docs_layout(str(tmp_path)) == "hierarchical" + + +def test_find_doc_and_listing_work_in_both_layouts(tmp_path): + _setup(tmp_path) + _write(tmp_path, "overview.md") + _write(tmp_path, "auth/session/store.md") + _write(tmp_path, "billing.md") + _write(tmp_path, "temp/ignored.md") + _write(tmp_path, "guides/user_notes.md") # not a module page + + assert L.find_doc(str(tmp_path), "store") == str(tmp_path / "auth/session/store.md") + assert L.find_doc(str(tmp_path), "login") is None + # a page saved flat is still found + _write(tmp_path, "login.md") + assert L.find_doc(str(tmp_path), "login") == str(tmp_path / "login.md") + + assert L.list_doc_files(str(tmp_path)) == { + "overview": "overview.md", + "store": "auth/session/store.md", + "billing": "billing.md", + "login": "login.md", + } + + +def test_find_doc_search_finds_page_dropped_from_tree(tmp_path): + _setup(tmp_path) + _write(tmp_path, "auth/old_module.md") + assert L.find_doc(str(tmp_path), "old_module") is None + assert L.find_doc(str(tmp_path), "old_module", search=True) == str( + tmp_path / "auth/old_module.md" + ) + + +# --------------------------------------------------------------------- links + + +def test_rewrite_page_links_fixes_relative_paths(): + paths = L.doc_relpaths(TREE, "hierarchical") + text = ( + "See [billing](billing.md), [store](store.md#api), [login](./login.md), " + '[wrong](../../billing.md "Billing"), [web](https://x.io/a.md), ' + "[unknown](nope.md) and [overview](overview.md).\n" + "```\n[in code](billing.md)\n```\n" + ) + out = L.rewrite_page_links(text, "auth/session.md", paths) + assert "[billing](../billing.md)" in out + assert "[store](session/store.md#api)" in out + assert "[login](login.md)" in out + assert '[wrong](../billing.md "Billing")' in out + assert "[web](https://x.io/a.md)" in out + assert "[unknown](nope.md)" in out + assert "[overview](../overview.md)" in out + assert "[in code](billing.md)" in out # fenced code is left alone + + +def test_rewrite_page_links_is_identity_for_flat_layout(): + paths = L.doc_relpaths(TREE, "flat") + text = "[a](auth.md) [s](store.md#x) [o](overview.md)\n" + assert L.rewrite_page_links(text, "login.md", paths) == text + + +def test_organize_docs_moves_misplaced_pages_and_fixes_links(tmp_path): + _setup(tmp_path) + _write(tmp_path, "overview.md", "[auth](auth.md) [store](store.md)\n") + _write(tmp_path, "auth.md", "[login](login.md)\n") + _write(tmp_path, "login.md", "[billing](billing.md) [store](store.md)\n") # misplaced + _write(tmp_path, "auth/session.md") + _write(tmp_path, "store.md") # misplaced + _write(tmp_path, "billing.md") + + L.organize_docs(str(tmp_path), "hierarchical") + + assert not (tmp_path / "login.md").exists() + assert not (tmp_path / "store.md").exists() + assert (tmp_path / "auth/login.md").read_text() == ( + "[billing](../billing.md) [store](session/store.md)\n" + ) + assert (tmp_path / "auth/session/store.md").exists() + assert (tmp_path / "auth.md").read_text() == "[login](auth/login.md)\n" + assert (tmp_path / "overview.md").read_text() == ( + "[auth](auth.md) [store](auth/session/store.md)\n" + ) + + +def test_organize_docs_flattens_nested_pages(tmp_path): + _setup(tmp_path, layout="flat") + _write(tmp_path, "auth/session/store.md", "[billing](../../billing.md)\n") + _write(tmp_path, "billing.md") + + L.organize_docs(str(tmp_path), "flat") + + assert (tmp_path / "store.md").read_text() == "[billing](billing.md)\n" + assert not (tmp_path / "auth").exists() # emptied folders are removed + + +# ------------------------------------------------------------------- prompts + + +def test_prompts_name_the_nested_page_path(): + path = ["auth", "session"] + system = format_system_prompt("session", None, path, "hierarchical") + assert "`auth/session.md`" in system + assert "under `auth/session/`" in system + assert "auth/session.md documentation file" in format_leaf_system_prompt( + "session", None, path, "hierarchical" + ) + user = format_user_prompt("session", [], {}, TREE, path, "hierarchical") + assert "store [page: auth/session/store.md]" in user + assert "Your page is `auth/session.md`" in user + + +def test_flat_prompts_keep_flat_wording(): + system = format_system_prompt("session", None, ["auth", "session"], "flat") + assert "`session.md`" in system + assert "all docs share one flat directory" in system + user = format_user_prompt("session", [], {}, TREE, ["auth", "session"], "flat") + assert "[page:" not in user + assert "saved in the same folder" in user + + +def test_sub_module_report_gives_links_from_parent_page(tmp_path): + _setup(tmp_path) + _write(tmp_path, "auth/session/store.md") + report = sub_module_report( + {"store": "store", "cache": "cache"}, + {}, + str(tmp_path), + TREE, + "session", + ["auth", "session"], + "hierarchical", + ) + assert "link them from `auth/session.md`" in report + assert "Saved documentations (link them from `auth/session.md` as): session/store.md." in ( + report + ) + assert "MISSING (generation did not produce these files): session/cache.md" in report + + +# ---------------------------------------------------------------- generation + + +class _FakeBackend: + """Writes each leaf page where the hierarchical layout expects it.""" + + def __init__(self): + self.calls: list[str] = [] + + async def run_module_agent( + self, module_name, components, core_component_ids, module_path, working_dir + ): + tree = json.loads(Path(working_dir, "module_tree.json").read_text()) + if L.find_doc(working_dir, module_name, tree): + return tree + self.calls.append(module_name) + rel = L.module_doc_relpath(module_path, "hierarchical") + _write(Path(working_dir), rel, f"# {module_name}\n\n[billing](billing.md)\n") + return tree + + def complete(self, prompt, model=None, system_prompt=None): + return "[store](store.md)" + + +def test_generation_writes_nested_pages(tmp_path): + _setup(tmp_path, layout=None) + gen = object.__new__(DocumentationGenerator) + gen.config = SimpleNamespace( + docs_dir=str(tmp_path), + repo_path=str(tmp_path), + get_prompt_addition=lambda: "", + layout="hierarchical", + ) + gen.backend = _FakeBackend() + + asyncio.run(gen.generate_module_documentation(components={}, leaf_nodes=[])) + + for rel in ("overview.md", "auth.md", "auth/login.md", "auth/session.md", "billing.md"): + assert (tmp_path / rel).exists(), rel + assert (tmp_path / "auth/session/store.md").read_text() == ( + "# store\n\n[billing](../../billing.md)\n" + ) + assert (tmp_path / "auth.md").read_text() == "[store](auth/session/store.md)" + assert find_missing_module_docs(TREE, str(tmp_path)) == [] + + # Resume: everything exists, nothing is regenerated + calls = list(gen.backend.calls) + asyncio.run(gen.generate_module_documentation(components={}, leaf_nodes=[])) + assert gen.backend.calls == calls + + +# ------------------------------------------------------------------- updater + + +def test_updater_pages_follow_recorded_layout(tmp_path): + _setup(tmp_path, layout="hierarchical") + _write(tmp_path, "auth/login.md", "old") + assert P.page_rel(str(tmp_path), "login") == "auth/login.md" + assert P.read_page(str(tmp_path), "login") == "old" + # a new page goes where the layout wants it + assert P.page_rel(str(tmp_path), "store") == "auth/session/store.md" + assert P.list_pages(str(tmp_path)) == ["login"] + assert P.remove_page(str(tmp_path), "login") + assert not (tmp_path / "auth").exists() + + flat = tmp_path / "flat" + _setup(flat, layout=None) # old docs: no layout recorded -> flat + assert P.page_rel(str(flat), "store") == "store.md" + + +# -------------------------------------------------------------- editor/viewer + + +def _editor_ctx(docs: Path): + deps = SimpleNamespace( + registry={}, + absolute_docs_path=str(docs), + absolute_repo_path=str(docs / "repo"), + allowed_write_paths=None, + ) + return SimpleNamespace(deps=deps) + + +def test_editor_creates_nested_page_folders(tmp_path): + from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor + + (tmp_path / "repo").mkdir() + out = asyncio.run( + str_replace_editor( + _editor_ctx(tmp_path), "docs", "create", path="auth/session/store.md", file_text="# s\n" + ) + ) + assert "File created successfully" in out + assert (tmp_path / "auth/session/store.md").read_text() == "# s\n" + + out = asyncio.run( + str_replace_editor( + _editor_ctx(tmp_path), "docs", "create", path="../escape/x.md", file_text="x" + ) + ) + assert "escapes" in out + assert not (tmp_path.parent / "escape").exists() + + +def test_viewer_embeds_page_paths(tmp_path): + from codewiki.cli.html_generator import HTMLGenerator + + _setup(tmp_path) + _write(tmp_path, "overview.md") + _write(tmp_path, "auth/session/store.md") + out = tmp_path / "index.html" + HTMLGenerator().generate(output_path=out, title="repo", docs_dir=tmp_path) + html = out.read_text(encoding="utf-8") + assert '"store": "auth/session/store.md"' in html + assert "{{DOC_PATHS_JSON}}" not in html + + +def test_metadata_records_layout(tmp_path): + gen = object.__new__(DocumentationGenerator) + gen.config = SimpleNamespace(main_model="m", repo_path=".", max_depth=2, layout="flat") + gen.commit_id = None + _setup(tmp_path, layout=None) + _write(tmp_path, "auth/login.md") + gen.create_documentation_metadata(str(tmp_path), {}, 0) + metadata = json.loads((tmp_path / "metadata.json").read_text()) + assert metadata["generation_info"]["layout"] == "flat" + assert "auth/login.md" in metadata["files_generated"] + + +def test_module_names_drop_ampersand(): + from codewiki.src.be.module_naming import dedupe_module_tree_names, sanitize_module_name + + assert sanitize_module_name("Data_Model_&_Persistence") == "Data_Model___Persistence" + tree = dedupe_module_tree_names({"A & B": {"components": [], "children": {}}}) + assert list(tree) == ["A___B"] diff --git a/tests/test_overview_structure.py b/tests/test_overview_structure.py index 77f541cf..0b11d0e4 100644 --- a/tests/test_overview_structure.py +++ b/tests/test_overview_structure.py @@ -12,6 +12,7 @@ import json import os +from types import SimpleNamespace from codewiki.src.be.documentation_generator import DocumentationGenerator @@ -19,9 +20,12 @@ def _generator() -> DocumentationGenerator: - # build_overview_structure touches neither config nor backend; skip the - # heavyweight __init__ (graph builder, backend resolution). - return DocumentationGenerator.__new__(DocumentationGenerator) + # build_overview_structure only reads the docs layout from config and never + # touches the backend; skip the heavyweight __init__ (graph builder, + # backend resolution). + gen = DocumentationGenerator.__new__(DocumentationGenerator) + gen.config = SimpleNamespace(layout="flat") + return gen def _make_tree(n_modules: int, n_components: int) -> dict: diff --git a/tests/test_processing_order_update.py b/tests/test_processing_order_update.py index d3449bf1..02b98ece 100644 --- a/tests/test_processing_order_update.py +++ b/tests/test_processing_order_update.py @@ -87,7 +87,10 @@ def _generator(docs_dir: Path) -> tuple[DocumentationGenerator, FakeBackend]: # Bypass __init__: it wires a real LLM backend and dependency analyzer. gen = object.__new__(DocumentationGenerator) gen.config = SimpleNamespace( - docs_dir=str(docs_dir), repo_path=str(docs_dir), get_prompt_addition=lambda: "" + docs_dir=str(docs_dir), + repo_path=str(docs_dir), + get_prompt_addition=lambda: "", + layout="flat", ) gen.backend = FakeBackend(docs_dir) return gen, gen.backend