Mirror the module tree in the docs folder by default

CHANGELOG.md

@@ -13,6 +13,18 @@ uses [Semantic Versioning](https://semver.org/).
   struct-literal, and call edges. `#[cfg(test)]` modules are skipped, and
   `build.rs` is classified as a build artifact.
 
+### Changed
+
+- **Docs mirror the module tree** ([#125](https://github.com/FSoft-AI4Code/CodeWiki/issues/125)).
+  A module's page now sits next to the folder that holds its sub-modules
+  (`auth.md`, `auth/login.md`) instead of every page sharing one flat folder.
+  Links between pages are relative. After each run, pages saved in the wrong
+  folder are moved and wrong links are repaired. `--flat` keeps the old
+  layout for small models. The layout is recorded in `metadata.json`, and
+  `--update` keeps it, so existing flat docs stay flat. The GitHub Pages
+  viewer, the web app and the MCP tools (`doc_path` in
+  `processing_order.json`) follow the layout.
+
 ## [2.0.0] - 2026-09-18
 
 The first release since 1.0.1. Everything that landed on `main` in between is

README.md

@@ -164,7 +164,8 @@ configuration in CodeWiki.
 ```
 ./docs/
 ├── overview.md                  # start here
-├── <module>.md ...              # one page per module, leaves and parents
+├── <module>.md ...              # one page per top-level module
+├── <module>/<sub-module>.md ... # sub-module pages, in folders mirroring the module tree
 ├── module_tree.json             # the module hierarchy
 ├── first_module_tree.json       # clustering result before super-grouping
 ├── metadata.json                # model, version, commit, statistics
@@ -174,6 +175,11 @@ configuration in CodeWiki.
 └── index.html                   # viewer (with --github-pages)
 ```
 
+Pages mirror the module tree: a module's page sits next to the folder holding
+its sub-modules (`auth.md`, `auth/login.md`). `--flat` puts every page in
+`./docs` instead, which helps small models that get relative links wrong.
+`--update` keeps the layout the docs were generated with.
+
 This repository's own output is checked in under [`./docs/`](./docs/).
 
 ## Benchmark results

codewiki/cli/adapters/doc_generator.py

@@ -19,10 +19,19 @@
 
 # Import backend modules
 from codewiki.src.be.documentation_generator import DocumentationGenerator
+from codewiki.src.be.doc_layout import list_doc_files
+from codewiki.src.config import DEFAULT_LAYOUT
 from codewiki.src.config import Config as BackendConfig
 from codewiki.src.config import set_cli_context
 
 
+def _generated_files(working_dir: str) -> list[str]:
+    """Docs-relative paths of the pages and the JSON files in the docs root."""
+    pages = sorted(list_doc_files(working_dir).values())
+    jsons = sorted(f for f in os.listdir(working_dir) if f.endswith(".json"))
+    return pages + jsons
+
+
 class CLIDocumentationGenerator:
     """
     CLI adapter for documentation generation with progress reporting.
@@ -154,6 +163,7 @@ def generate(self) -> DocumentationJob:
                 artifacts_enabled=self.config.get("artifacts_enabled", True),
                 artifact_token_budget=self.config.get("artifact_token_budget", 200_000),
                 with_prose=self.config.get("with_prose", False),
+                layout=self.config.get("layout", DEFAULT_LAYOUT),
             )
 
             # Run backend documentation generation
@@ -372,9 +382,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
             self._merge_update_summary(working_dir)
 
             # Collect generated files
-            for file_path in os.listdir(working_dir):
-                if file_path.endswith((".md", ".json")):
-                    self.job.files_generated.append(file_path)
+            self.job.files_generated.extend(_generated_files(working_dir))
 
         except Exception as e:  # noqa: BLE001 — surfaced to the user as an APIError
             raise APIError(f"Documentation generation failed: {e}")
@@ -448,9 +456,7 @@ async def _run_incremental_update(
             prior_history = self._read_update_history(working_dir)
             doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
             self._merge_update_summary(working_dir, prior_history)
-            for file_path in os.listdir(working_dir):
-                if file_path.endswith((".md", ".json")):
-                    self.job.files_generated.append(file_path)
+            self.job.files_generated.extend(_generated_files(working_dir))
             tree_path = os.path.join(working_dir, "module_tree.json")
             if os.path.exists(tree_path):
                 with open(tree_path, encoding="utf-8") as f:

codewiki/cli/commands/generate.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+import os
 import sys
 import time
 import traceback
@@ -24,6 +25,8 @@
 )
 from codewiki.cli.utils.instructions import display_post_generation_instructions
 from codewiki.cli.utils.logging import create_logger
+from codewiki.src.be.doc_layout import find_doc, list_doc_files, read_layout, remove_doc
+from codewiki.src.config import DEFAULT_LAYOUT, LAYOUT_FLAT
 from codewiki.src.language import resolve_update_language
 from codewiki.cli.utils.repo_validator import (
     check_writable_output,
@@ -204,13 +207,13 @@ def _find_affected(tree, parent_names=None):
     if modules_to_invalidate:
         modules_to_invalidate.add("overview")
 
-    # Delete affected module docs
+    # Delete affected module docs (wherever the docs layout put them)
     for mod_name in modules_to_invalidate:
-        doc_path = output_dir / f"{mod_name}.md"
-        if doc_path.exists():
-            doc_path.unlink()
+        doc_path = find_doc(str(output_dir), mod_name, module_tree)
+        if doc_path is not None:
+            remove_doc(str(output_dir), doc_path)
             if verbose:
-                logger.debug(f"Invalidated: {doc_path.name}")
+                logger.debug(f"Invalidated: {os.path.relpath(doc_path, output_dir)}")
 
     if verbose:
         logger.debug(f"Invalidated {len(modules_to_invalidate)} modules for regeneration.")
@@ -353,6 +356,13 @@ def _find_affected(tree, parent_names=None):
     is_flag=True,
     help="Also read the root README and docs/ as a `prose` artifact class (off by default)",
 )
+@click.option(
+    "--flat",
+    is_flag=True,
+    help="Save every page in the output directory root instead of folders mirroring the "
+    "module tree (for small models that struggle with relative links). --update keeps the "
+    "layout the docs were generated with",
+)
 @click.option(
     "--artifact-exclude",
     type=str,
@@ -447,6 +457,7 @@ def generate_command(
     artifacts: bool = True,
     artifact_token_budget: int = 200_000,
     with_prose: bool = False,
+    flat: bool = False,
     artifact_exclude: str | None = None,
     update: bool = False,
     compare_to: str | None = None,
@@ -585,6 +596,16 @@ def generate_command(
                 )
             except ValueError as e:
                 raise ConfigurationError(str(e)) from None
+        layout = LAYOUT_FLAT if flat else DEFAULT_LAYOUT
+        if updating_existing_docs:
+            # Keep updated pages where the existing docs put them
+            stored_layout = read_layout(str(output_dir))
+            if flat and stored_layout != LAYOUT_FLAT:
+                logger.warning(
+                    f"These docs use the {stored_layout} layout; --flat is ignored by --update. "
+                    "Rerun without --update to regenerate them flat."
+                )
+            layout = stored_layout
         if update and output_dir.exists():
             changed_files = _detect_changed_files(
                 repo_path, output_dir, logger, verbose, compare_to=compare_to
@@ -616,7 +637,7 @@ def generate_command(
         if (
             not update
             and output_dir.exists()
-            and list(output_dir.glob("*.md"))
+            and list_doc_files(str(output_dir))
             and not click.confirm(
                 f"\n{output_dir} already contains documentation. Overwrite?", default=True
             )
@@ -777,6 +798,8 @@ def generate_command(
                 "artifacts_enabled": artifacts,
                 "artifact_token_budget": artifact_token_budget,
                 "with_prose": with_prose,
+                # Docs layout (runtime-only; --update keeps the stored one)
+                "layout": layout,
                 # Incremental updater (runtime-only)
                 "update": update,
                 "update_options": {

codewiki/cli/html_generator.py

@@ -156,12 +156,21 @@ def generate(
             page_titles = extract_page_titles(str(docs_dir), module_tree)
         page_titles_json = json.dumps(page_titles, ensure_ascii=False).replace("</", "<\\/")
 
+        # Where each page lives (nested folders or the docs root)
+        doc_paths = {}
+        if docs_dir:
+            from codewiki.src.be.doc_layout import doc_path_map
+
+            doc_paths = doc_path_map(str(docs_dir), module_tree)
+        doc_paths_json = json.dumps(doc_paths, ensure_ascii=False).replace("</", "<\\/")
+
         # Replace placeholders
         html_content = template_content
         replacements = {
             "{{TITLE}}": self._escape_html(title),
             "{{HTML_LANG}}": language_tag(language),
             "{{PAGE_TITLES_JSON}}": page_titles_json,
+            "{{DOC_PATHS_JSON}}": doc_paths_json,
             "{{REPO_LINK}}": repo_link,
             "{{SHOW_INFO}}": show_info,
             "{{INFO_CONTENT}}": info_content,

codewiki/mcp/server.py

@@ -150,7 +150,11 @@ def _fine_grained_tools() -> list[Tool]:
                     },
                     "filename": {
                         "type": "string",
-                        "description": "Filename for the doc (e.g., 'auth_module.md')",
+                        "description": (
+                            "Path of the doc relative to the output dir: the module's "
+                            "doc_path from processing_order.json (e.g., 'auth.md' or "
+                            "'auth/login.md'; folders are created as needed)"
+                        ),
                     },
                     "content": {
                         "type": "string",
@@ -531,6 +535,8 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
 
     set_cli_context(True)
 
+    from codewiki.src.be.doc_layout import docs_layout, list_doc_files
+
     backend_config = BackendConfig.from_cli(
         repo_path=str(repo_path),
         output_dir=str(output_dir),
@@ -544,6 +550,7 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
         max_tokens=config.max_tokens,
         agent_instructions=agent_instructions or None,
         use_gitignore=arguments.get("use_gitignore", True),
+        layout=docs_layout(str(output_dir)),
     )
 
     from codewiki.cli.utils.repo_validator import get_git_commit_hash
@@ -554,9 +561,9 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
     )
     await doc_gen.run()
 
-    generated_files = []
+    generated_files = sorted(list_doc_files(str(output_dir)).values())
     for f in output_dir.iterdir():
-        if f.suffix in (".md", ".json", ".html"):
+        if f.suffix in (".json", ".html"):
             generated_files.append(f.name)
 
     result = {
@@ -654,10 +661,18 @@ def _write_generation_metadata(session: SessionState) -> None:
             except (json.JSONDecodeError, OSError):
                 pass
 
+        from codewiki.src.be.doc_layout import docs_layout, organize_docs
+
+        # Keep the layout of existing docs; new docs get the default. Move
+        # misplaced pages and fix links between pages before stamping it.
+        layout = docs_layout(str(output_dir))
+        organize_docs(str(output_dir), layout)
+
         existing["generation_info"] = {
             **existing.get("generation_info", {}),
             "commit_id": commit_id,
             "timestamp": datetime.now().isoformat(),
+            "layout": layout,
         }
         metadata_path.write_text(
             json.dumps(existing, indent=2, ensure_ascii=False),

codewiki/mcp/tools/module_tree.py

@@ -13,7 +13,8 @@
 from typing import Any, Dict, List, Tuple
 
 from codewiki.mcp.session import SessionStore
-from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
+from codewiki.src.be.doc_layout import docs_layout, module_doc_relpath
+from codewiki.src.config import DEFAULT_LAYOUT, FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
 
 logger = logging.getLogger(__name__)
 
@@ -30,12 +31,15 @@ def _cap(ids: List[str]) -> Tuple[List[str], bool]:
 
 
 def _get_processing_order(
-    module_tree: Dict[str, Any], parent_path: List[str] | None = None
+    module_tree: Dict[str, Any],
+    parent_path: List[str] | None = None,
+    layout: str = DEFAULT_LAYOUT,
 ) -> List[Dict[str, Any]]:
     """Compute leaf-first processing order from a module tree.
 
-    Returns a list of dicts with module path, name, leaf status, and
-    component/children info.
+    Returns a list of dicts with module path, name, leaf status,
+    component/children info and ``doc_path`` (the page to write, relative to
+    the output dir, for the docs ``layout``).
     """
     if parent_path is None:
         parent_path = []
@@ -54,6 +58,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None:
                         "module": module_name,
                         "path": current_path,
                         "is_leaf": False,
+                        "doc_path": module_doc_relpath(current_path, layout),
                         "children": list(children.keys()),
                         "components": module_info.get("components", []),
                     }
@@ -64,6 +69,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None:
                         "module": module_name,
                         "path": current_path,
                         "is_leaf": True,
+                        "doc_path": module_doc_relpath(current_path, layout),
                         "components": module_info.get("components", []),
                     }
                 )
@@ -189,7 +195,7 @@ def handle_save_module_tree(
         logger.info("save_module_tree for session %s: %s", session_id, note)
 
     # Compute processing order and write to workspace file
-    order = _get_processing_order(module_tree)
+    order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir))
     order_file = None
     if session.workspace is not None:
         order_path = session.workspace.write_json("processing_order.json", order)
@@ -206,7 +212,9 @@ def handle_save_module_tree(
             "Read the processing_order.json file for the leaf-first generation order. "
             "Process leaf modules first (is_leaf=true), then parent modules. "
             "For each leaf module: get_prompt('system_leaf') + read_code_components + write_doc_file. "
-            "For each parent module: get_prompt('overview_module') + write_doc_file."
+            "For each parent module: get_prompt('overview_module') + write_doc_file. "
+            "Save each module's page at its doc_path (sub-folders are created automatically) "
+            "and link pages with paths relative to the linking page."
         ),
     }
     if warning:
@@ -237,7 +245,7 @@ def handle_get_processing_order(
         else:
             return json.dumps({"error": "Module tree not found. Call save_module_tree first."})
 
-    order = _get_processing_order(module_tree)
+    order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir))
 
     # Write to workspace file
     order_file = None