From db3e96c18fc7c2d9be48671050eaf97bda0fc487 Mon Sep 17 00:00:00 2001 From: ivanmilevtues Date: Sat, 26 Sep 2026 21:01:38 +0200 Subject: [PATCH] feat: say why a run failed when the LLM quota runs out or the key is rejected The engine exits 3 when the LLM quota is exhausted and 2 when the key is rejected, and stops without publishing an analysis. analyze_repository.py maps those codes to one line saying what to fix, raises it as the error annotation, and writes it to the step's failure_reason output, which the review failure comment now shows above the link to the logs. Any other failure is reported as before. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +++ action.yml | 2 ++ scripts/analyze_repository.py | 13 +++++++++++++ tests/test_analyze_repository.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/README.md b/README.md index 8f13b94..dc391e3 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,9 @@ Every run reports what it resolved, so the answer never has to be inferred from - a job-summary table naming the tier and provider; - on a configuration failure, an error annotation and — in review mode — a pull request comment with the fix, so the person who has to add the secret sees it where they are. +- when the provider rejects the key or its quota runs out during the analysis, the run fails + with an error annotation saying which, and in review mode the failure comment says it too. + No partial analysis is published. ## Model selection diff --git a/action.yml b/action.yml index a371995..5e32aa3 100644 --- a/action.yml +++ b/action.yml @@ -668,4 +668,6 @@ runs: message: | ### CodeBoarding review · failed + ${{ steps.review_analyze.outputs.failure_reason }} + See the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). diff --git a/scripts/analyze_repository.py b/scripts/analyze_repository.py index aba775a..446bdce 100755 --- a/scripts/analyze_repository.py +++ b/scripts/analyze_repository.py @@ -5,6 +5,7 @@ import argparse import json +import os import shutil import subprocess import sys @@ -12,6 +13,12 @@ PROG = "codeboarding" +# The engine's exit codes for an LLM refusal the user has to fix, and what to tell them. +ENGINE_REFUSALS = { + 2: "The LLM provider rejected the API key. Check the key's secret and re-run.", + 3: "The LLM provider's token or credit quota is exhausted. Add credits or raise the quota, then re-run.", +} + class AnalysisError(RuntimeError): pass @@ -84,6 +91,12 @@ def _run_command(args: list[str], output_dir: Path) -> str: return_code = process.wait() stdout = "".join(stdout_lines) if return_code != 0: + reason = ENGINE_REFUSALS.get(return_code) + if reason: + # Written straight to the step's outputs: the shell never sees this script's stdout on failure. + with open(os.environ.get("GITHUB_OUTPUT", os.devnull), "a", encoding="utf-8") as outputs: + outputs.write(f"failure_reason={reason}\n") + raise AnalysisError(reason) details = stdout.strip() or f"exit code {return_code}; see command logs above" raise AnalysisError(f"Command failed ({' '.join(args)}): {details}") diff --git a/tests/test_analyze_repository.py b/tests/test_analyze_repository.py index d394736..7c923fa 100644 --- a/tests/test_analyze_repository.py +++ b/tests/test_analyze_repository.py @@ -70,6 +70,37 @@ def test_run_command_streams_stdout_to_action_logs(self) -> None: self.assertIn('{"requiresFullAnalysis": true}', stderr.getvalue()) self.assertEqual(stdout, 'Analyzing repository...\n{"requiresFullAnalysis": true}\n') + def _fail_with(self, exit_code: int, outputs: Path) -> ar.AnalysisError: + command = [sys.executable, "-c", f"import sys; sys.exit({exit_code})"] + with patch.dict("os.environ", {"GITHUB_OUTPUT": str(outputs)}), patch("sys.stderr", io.StringIO()): + with self.assertRaises(ar.AnalysisError) as caught: + ar._run_command(command, outputs.parent / "out") + return caught.exception + + def test_run_command_names_an_exhausted_quota(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + outputs = Path(tmp) / "github_output" + + error = self._fail_with(3, outputs) + + self.assertIn("quota is exhausted", str(error)) + self.assertEqual(outputs.read_text(encoding="utf-8"), f"failure_reason={error}\n") + + def test_run_command_names_a_rejected_key(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + error = self._fail_with(2, Path(tmp) / "github_output") + + self.assertIn("rejected the API key", str(error)) + + def test_run_command_leaves_other_failures_unnamed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + outputs = Path(tmp) / "github_output" + + error = self._fail_with(1, outputs) + + self.assertIn("exit code 1", str(error)) + self.assertFalse(outputs.exists()) + def test_parse_main_incremental_success(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp)