From d30b8276902a5db043e36be958f9d283e39521a9 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Sun, 27 Sep 2026 12:14:50 +0100 Subject: [PATCH] fix(board): resolve workspace files across file/directory config forms; allow-list autonerves-owned keys The override map compared a workspace file only against a library file at the same relative path. autonerves' RecursiveConfig treats a library `X.yaml` top-level section `f` and a workspace `X/f.yaml` (and the reverse) as one config node, so PyAutoCTI's single visualize.yaml was never compared with autocti_workspace's visualize/ directory and 13 keys it defines showed as orphans. resolve_in_library now walks a library the way RecursiveConfig does (a file shadows its directory, sections are directory entries, stems lowercased); prior files keep same-path matching. The version-handshake keys autonerves itself reads from a workspace's general.yaml (version.{workspace_version, minimum_library_version, workspace_version_check, python_version_check}) can never be defined by a library YAML, so NERVES_OWNED excludes them from orphans and the repo page notes them as "owned by autonerves". On the real stack this clears 20 of the 79 orphan keys (14 files -> 12, 79 keys -> 59) and changes no other override. Refs PyAutoLabs/PyAutoMind#441 Co-Authored-By: Claude Opus 5.5 --- scripts/board.py | 118 ++++++++++++++++++++++++++++++---- test_autonerves/test_board.py | 83 ++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 14 deletions(-) diff --git a/scripts/board.py b/scripts/board.py index 017b45a..cfdaecb 100644 --- a/scripts/board.py +++ b/scripts/board.py @@ -19,13 +19,17 @@ or lower/upper · width modifier · limits; * **the override map**: autonerves resolves a key workspace → last-imported library → … → PyAutoFit (``conf.Config.push(keep_first=True)``, keys - lowercased), so every workspace file is looked up, by relative path, - across that workspace's library stack in the same order. The *counterpart* - is the first library holding the file; the diff is against the merged - stack (the value autonerves would fall back to): keys whose value - differs, keys only the workspace sets (**orphans** — no library defines - them), keys only the stack sets. ``build/*.yaml`` is workspace tooling - (CI/build lists, not library settings) and is grouped apart; + lowercased), so every workspace file is looked up across that + workspace's library stack in the same order — by relative path, where a + library ``X.yaml``'s section ``f`` is the workspace's ``X/f.yaml`` and the + other way round, as ``RecursiveConfig`` merges them (prior files: same + path only). The *counterpart* is the first library holding the file; + the diff is against the merged stack (the value autonerves would fall + back to): keys whose value differs, keys only the workspace sets + (**orphans** — no library defines them), keys only the stack sets. The + version-handshake keys autonerves reads itself (``NERVES_OWNED``) are + *owned*, never orphans. ``build/*.yaml`` is workspace tooling (CI/build + lists, not library settings) and is grouped apart; * **possibly unused library keys**: every library package's ``.py`` (and ``autonerves/``) is scanned for config lookups (``scan_lookups``) and each key of a library settings file is classed ``used`` / ``section-read`` / @@ -351,10 +355,81 @@ def read_file(repo: str, rel: str, text: str, kind: str) -> dict: return rec +# Keys autonerves itself reads from a workspace's config — the version +# handshake (``autonerves/workspace.py``, ``autonerves/__init__.py``). The +# Nerves ship no YAML, so no library defines them: owned, never orphans. +NERVES_OWNED = { + "general.yaml": ("version.workspace_version", + "version.minimum_library_version", + "version.workspace_version_check", + "version.python_version_check"), +} + +_MISSING = object() + + +def _section(data, parts: list[str]): + """``data`` descended through ``parts`` (keys lowercased, as autonerves + reads them), else ``_MISSING``.""" + for p in parts: + if not isinstance(data, dict): + return _MISSING + data = next((v for k, v in data.items() if str(k).lower() == p), + _MISSING) + if data is _MISSING: + return _MISSING + return data + + +def resolve_in_library(lib_files: dict, rel: str) -> dict | None: + """What one library gives for a workspace file ``rel`` (``lib_files`` is + ``{path: record}``), resolved the way autonerves' ``RecursiveConfig`` + walks a config dir: at each level of the path a ``.yml``/``.yaml`` + file shadows a ``/`` directory, and a file's top-level sections are + the files of the directory form. So a library ``visualize.yaml``'s + ``general:`` section is the workspace's ``visualize/general.yaml``, and a + library ``visualize/`` directory is a workspace ``visualize.yaml``. + Returns a stack record (``repo``, ``path`` — the library file, with a + ``§ section`` when one — and ``_data``), or ``None``.""" + # file stems reach autonerves lowercased (DictWrapper): match them so + files = {p.lower(): r for p, r in lib_files.items()} + parts = rel.rsplit(".", 1)[0].lower().split("/") + for i in range(len(parts)): + stem = "/".join(parts[:i + 1]) + rec = next((files[stem + s] for s in YAML_SUFFIXES[::-1] + if stem + s in files), None) + if rec is not None: + if rec["error"]: + return None + data = _section(rec.get("_data"), parts[i + 1:]) + if data is _MISSING: + return None + where = rec["path"] + (f" § {'.'.join(parts[i + 1:])}" + if parts[i + 1:] else "") + return {"repo": rec["repo"], "path": where, "_data": data} + if not any(p.startswith(stem + "/") for p in files): + return None + # the whole path is a library directory: each entry is a section + children = set() + for p in files: + if p.startswith(stem + "/"): + head, _, tail = p[len(stem) + 1:].partition("/") + children.add(head if tail else head.rsplit(".", 1)[0]) + data = {} + for child in sorted(children - {"priors"}): # RecursiveConfig skips priors + sub = resolve_in_library(lib_files, f"{stem}/{child}.yaml") + if sub is not None: + data[child] = sub["_data"] + if not data: + return None + repo = next(iter(files.values()))["repo"] + return {"repo": repo, "path": stem + "/", "_data": data} + + def diff_against_stack(ws: dict, stack_recs: list[dict]) -> dict: """The override record for one workspace file against its library stack (``stack_recs`` in autonerves lookup order, only the libraries that hold - the same relative path).""" + it — by relative path, or through ``resolve_in_library``).""" # A prior file is compared per parameter: a workspace that swaps a # Gaussian for a Uniform changes one spec, it does not orphan the new # prior's lower/upper fields. @@ -370,12 +445,17 @@ def diff_against_stack(ws: dict, stack_recs: list[dict]) -> dict: lines = {k["k"].lower(): k["line"] for k in ws.get("keys") or []} differs = sorted(k for k in wflat if k in merged and merged[k] != wflat[k]) ws_only = sorted(k for k in wflat if k not in merged) + owned = set(NERVES_OWNED.get(ws["path"], ())) + nerves_owned = [k for k in ws_only if k in owned] + ws_only = [k for k in ws_only if k not in owned] lib_only = sorted(k for k in merged if k not in wflat) first = stack_recs[0] if stack_recs else None return {"repo": ws["repo"], "path": ws["path"], "counterpart": first["repo"] if first else None, + "counterpart_path": first["path"] if first else None, "stack": [r["repo"] for r in stack_recs], "differs": differs, "workspace_only": ws_only, + "nerves_owned": nerves_owned, "library_only": lib_only[:50], "library_only_count": len(lib_only), "first_orphan_line": min((lines.get(k, 0) for k in ws_only), default=0) or 1} @@ -724,9 +804,16 @@ def build_snapshot(files: list[dict], sources: list[dict], env: list[dict], for rel, rec in sorted((by_repo.get(src["repo"]) or {}).items()): if rec["tooling"] or rec["error"]: continue - stack = [by_repo[lib][rel] for lib in src.get("stack") or () - if rel in (by_repo.get(lib) or {}) - and not by_repo[lib][rel]["error"]] + if rec["prior"]: + # priors resolve by class through JSONPriorConfig, not + # RecursiveConfig: same relative path only + stack = [by_repo[lib][rel] for lib in src.get("stack") or () + if rel in (by_repo.get(lib) or {}) + and not by_repo[lib][rel]["error"]] + else: + stack = [r for r in (resolve_in_library(by_repo[lib], rel) + for lib in src.get("stack") or () + if by_repo.get(lib)) if r is not None] if not stack: continue overrides.append(diff_against_stack(rec, stack)) @@ -1364,10 +1451,13 @@ def _file_html(snap: dict, f: dict, ov: dict | None) -> str: if f.get("error"): notes.append(f"

does not parse: {_esc(f['error'])}

") if ov: + owned = (f" · {len(ov['nerves_owned'])} owned by autonerves" + if ov.get("nerves_owned") else "") notes.append(f"

overrides {_esc(ov['counterpart'])}" - f"/{_esc(f['path'])} (stack: " - f"{_esc(' → '.join(ov['stack']))}) · {len(ov['differs'])} " - f"differ · {len(ov['workspace_only'])} orphan · " + f"/{_esc(ov.get('counterpart_path') or f['path'])} " + f"(stack: {_esc(' → '.join(ov['stack']))}) · " + f"{len(ov['differs'])} differ · " + f"{len(ov['workspace_only'])} orphan{owned} · " f"{ov['library_only_count']} from the stack

") chips = "".join(f"{_esc(k)}" for k in f.get("top_keys", [])[:40]) diff --git a/test_autonerves/test_board.py b/test_autonerves/test_board.py index e0ec5b6..0f02c26 100644 --- a/test_autonerves/test_board.py +++ b/test_autonerves/test_board.py @@ -315,6 +315,89 @@ def test_override_diff_runs_across_the_whole_stack(snap): assert _file(snap, "ws_demo", "build/no_run.yaml")["tooling"] is True +def _overrides(lib_files: dict, ws_files: dict, stack=("LibHigh", "LibLow")): + """Override records from in-memory files: ``{repo: {rel: text}}`` for + the libraries, ``{rel: text}`` for ws_demo.""" + files = [board.read_file(repo, rel, text, "library") + for repo, rels in lib_files.items() for rel, text in rels.items()] + files += [board.read_file("ws_demo", rel, text, "workspace") + for rel, text in ws_files.items()] + sources = [{"repo": r, "kind": "library", "config": "config", "stack": []} + for r in lib_files] + sources.append({"repo": "ws_demo", "kind": "workspace", "config": "config", + "stack": list(stack)}) + snap = board.build_snapshot(files, sources, [], []) + return {o["path"]: o for o in snap["overrides"]}, snap + + +def test_a_library_file_section_is_the_workspace_directory_file(): + # autonerves merges LibHigh's single visualize.yaml and a visualize/ + # directory alike: section `plots` of the file IS visualize/plots.yaml + lib = {"LibHigh": {"visualize.yaml": "plots:\n fit:\n subplot: true\n" + "general:\n backend: agg\n"}, + "LibLow": {"visualize/plots.yaml": "dataset:\n image: true\n"}} + ov, snap = _overrides(lib, { + "visualize/plots.yaml": "fit:\n subplot: false\ndataset:\n" + " image: true\nstale:\n flag: 1\n"}) + o = ov["visualize/plots.yaml"] + assert o["stack"] == ["LibHigh", "LibLow"] + assert o["counterpart_path"] == "visualize.yaml § plots" + assert o["differs"] == ["fit.subplot"] + # a genuinely unknown key is still an orphan + assert o["workspace_only"] == ["stale.flag"] + assert [x["path"] for x in board.orphans(snap)] == ["visualize/plots.yaml"] + # …and the other way round: a workspace file against a library directory + ov, _ = _overrides({"LibLow": {"visualize/general.yaml": + "general:\n backend: agg\n"}}, + {"visualize.yaml": "general:\n general:\n" + " backend: tkagg\n"}, + stack=("LibLow",)) + o = ov["visualize.yaml"] + assert o["counterpart_path"] == "visualize/" + assert o["differs"] == ["general.general.backend"] + assert o["workspace_only"] == [] + + +def test_a_library_file_shadows_its_directory_and_a_missing_section_is_none(): + lib = {"visualize.yaml": board.read_file("L", "visualize.yaml", + "plots:\n a: 1\n", "library"), + "visualize/plots.yaml": board.read_file("L", "visualize/plots.yaml", + "b: 2\n", "library")} + # RecursiveConfig takes the file: the directory is never read + assert board.resolve_in_library(lib, "visualize/plots.yaml")["_data"] == \ + {"a": 1} + assert board.resolve_in_library(lib, "visualize/other.yaml") is None + assert board.resolve_in_library(lib, "general.yaml") is None + + +def test_nerves_owned_version_keys_are_not_orphans(monkeypatch): + monkeypatch.setattr(board, "theme", lambda: FAKE_THEME) + ov, snap = _overrides( + {"LibLow": {"general.yaml": "output:\n log_level: INFO\n"}}, + {"general.yaml": "version:\n minimum_library_version: 2026.1.1\n" + " workspace_version_check: false\n" + " python_version_check: false\n" + " workspace_version: 2025.1.1\n" + " made_up_check: true\n"}, + stack=("LibLow",)) + o = ov["general.yaml"] + assert o["nerves_owned"] == ["version.minimum_library_version", + "version.python_version_check", + "version.workspace_version", + "version.workspace_version_check"] + # the allow-list is exact: an unknown key under version: still flags + assert o["workspace_only"] == ["version.made_up_check"] + page = board.render(snap, "html-repo", "ws_demo") + assert "1 orphan · 4 owned by autonerves" in page + # and only general.yaml is allow-listed + ov, _ = _overrides( + {"LibLow": {"other.yaml": "a: 1\n"}}, + {"other.yaml": "version:\n workspace_version_check: false\n"}, + stack=("LibLow",)) + assert ov["other.yaml"]["workspace_only"] == \ + ["version.workspace_version_check"] + + def test_env_var_panel_from_module_text(): env = {e["name"]: e for e in board.env_vars_from({"demo.py": FAKE_MODULE})} assert set(env) == {"PYAUTO_DEMO_QUIET", "PYAUTO_DEMO_LIMIT"}