fix: EP projection raises ProjectionException on non-finite sufficient statistics; factor_step recovers (#1653)

autofit/exc.py

@@ -62,6 +62,21 @@ class SearchException(Exception):
     pass
 
 
+class ProjectionException(ValueError):
+    """
+    Raised when an importance-weighted message projection
+    (`AbstractMessage.project`) produces non-finite sufficient statistics —
+    from non-finite samples, nan / +inf log weights, all-zero weights, or
+    overflow of the weighted moments.
+
+    A `ValueError` subclass, deliberately: EP's `factor_step` recovers from
+    it by keeping the factor's previous message for that sweep (a failed
+    sweep update, not a failed graph fit). It is not a `MessageException`,
+    which several callers turn into a silent revert or a -inf likelihood, and
+    not a `SearchException`, which signals a misconfigured search.
+    """
+
+
 class SamplesWarning(Warning):
     """
     Raises warnings associated with the `non_linear` module and `NonLinearSearch` classes.

autofit/graphical/expectation_propagation/optimiser.py

@@ -152,6 +152,7 @@ def factor_step(factor_approx, optimiser, model_approx=None):
         ArithmeticError,
         RuntimeError,
         exc.InitializerException,
+        exc.ProjectionException,
     ) as e:
         # `InitializerException` is raised when a factor's own optimiser cannot
         # find a start point — most commonly because EP has driven the factor to
@@ -160,6 +161,11 @@ def factor_step(factor_approx, optimiser, model_approx=None):
         # degrade to the factor's previous message and let the sweep continue,
         # with the failure recorded. `EPOptimiser` aborts if one factor keeps
         # failing (see `max_consecutive_failures`).
+        #
+        # `ProjectionException` (a `ValueError`, listed for intent) is raised
+        # when one factor's weighted samples project to non-finite sufficient
+        # statistics: a non-finite projection of one factor's samples is a
+        # failed sweep update, not a failed graph fit, and takes the same path.
         logger.exception(e)
         status = Status(
             success=False,

autofit/mapper/prior/abstract.py

@@ -8,6 +8,7 @@
 
 from autonerves import conf
 
+from autofit import exc
 from autofit.mapper.prior.arithmetic import ArithmeticMixin
 from autofit.mapper.prior.constant import Constant
 from autofit.mapper.prior.deferred import DeferredArgument
@@ -234,12 +235,17 @@ def project(self, samples, log_weight_list):
             the prior bounds (see PyAutoFit#1382).
         """
         result = copy(self)
-        result.message = self.message.project(
-            samples=samples,
-            log_weight_list=log_weight_list,
-            id_=self.id,
-            **self.message._support_kwargs,
-        )
+        try:
+            result.message = self.message.project(
+                samples=samples,
+                log_weight_list=log_weight_list,
+                id_=self.id,
+                **self.message._support_kwargs,
+            )
+        except exc.ProjectionException as e:
+            raise exc.ProjectionException(
+                f"Projecting prior id={self.id} ({type(self).__name__}) failed: {e}"
+            ) from e
         return result
 
     def __getattr__(self, item):

autofit/messages/abstract.py

@@ -12,6 +12,7 @@
 import numpy as np
 
 from autonerves import cached_property
+from .. import exc
 from ..mapper.variable import Variable
 
 from .interface import MessageInterface
@@ -313,11 +314,63 @@ def project(
         w /= norm
         suff_stats = (tx * w[None, ...]).mean(1)
 
-        assert np.isfinite(suff_stats).all()
+        if not np.isfinite(suff_stats).all():
+            raise exc.ProjectionException(
+                cls._nonfinite_projection_reason(samples, log_weight_list, suff_stats)
+            )
 
         cls_ = cls._projection_class or cls._Base_class or cls
         return cls_.from_sufficient_statistics(suff_stats, log_norm=log_norm, **kwargs)
 
+    @classmethod
+    def _nonfinite_projection_reason(
+        cls, samples: np.ndarray, log_weight_list: np.ndarray, suff_stats: np.ndarray
+    ) -> str:
+        """
+        Explain why `project` produced non-finite sufficient statistics.
+
+        Only called on the failure path. The inputs are checked in the order
+        a cause propagates: non-finite samples, then nan / +inf log weights,
+        then all log weights -inf (every sample has zero weight), then
+        overflow of the weighted sufficient statistics T(x)·w.
+        """
+        prefix = (
+            f"{cls.__name__}.project: non-finite sufficient statistics "
+            f"{suff_stats}; "
+        )
+        samples = np.asarray(samples, dtype=float)
+        log_weights = np.asarray(log_weight_list, dtype=float)
+
+        flat = samples.ravel()
+        bad = ~np.isfinite(flat)
+        if bad.any():
+            return prefix + (
+                f"{int(bad.sum())} non-finite samples of {flat.size} "
+                f"(nan={int(np.isnan(flat).sum())}, "
+                f"inf={int(np.isinf(flat).sum())}), "
+                f"first at index {int(np.flatnonzero(bad)[0])}"
+            )
+
+        flat_w = log_weights.ravel()
+        bad_w = np.isnan(flat_w) | (flat_w == np.inf)
+        if bad_w.any():
+            return prefix + (
+                f"{int(bad_w.sum())} log weights are nan or +inf "
+                f"(nan={int(np.isnan(flat_w).sum())}, "
+                f"+inf={int((flat_w == np.inf).sum())}), "
+                f"first at index {int(np.flatnonzero(bad_w)[0])}"
+            )
+
+        if flat_w.size and np.all(flat_w == -np.inf):
+            return prefix + (
+                "all log weights are -inf (every sample has zero weight)"
+            )
+
+        return prefix + (
+            "overflow in the weighted sufficient statistics T(x)·w "
+            f"(max |x| = {np.max(np.abs(flat)) if flat.size else float('nan')})"
+        )
+
     @classmethod
     def from_mode(
         cls, mode: np.ndarray, covariance: np.ndarray, **kwargs

autofit/non_linear/result.py

@@ -495,13 +495,18 @@ def projected_model(self) -> AbstractPriorModel:
             )
         with np.errstate(divide="ignore"):
             log_weight_list = np.log(weights)
-        arguments = {
-            prior: prior.project(
-                samples=np.array(self.samples.values_for_path(path)),
-                log_weight_list=log_weight_list,
-            )
-            for path, prior in self.samples.model.path_priors_tuples
-        }
+        arguments = {}
+        for path, prior in self.samples.model.path_priors_tuples:
+            try:
+                arguments[prior] = prior.project(
+                    samples=np.array(self.samples.values_for_path(path)),
+                    log_weight_list=log_weight_list,
+                )
+            except exc.ProjectionException as e:
+                raise exc.ProjectionException(
+                    f"Projecting the model parameter path={'.'.join(path)} "
+                    f"failed: {e}"
+                ) from e
         return self.samples.model.mapper_from_prior_arguments(arguments)
 
     @property

test_autofit/graphical/functionality/test_factor_failure_recovery.py

@@ -701,3 +701,73 @@ def test_a_group_no_member_of_which_moves_a_variable_is_named_once(tmp_path):
     assert "STALE FACTORS" in warnings[0]
     assert "variable 's' of group_stale" in warnings[0]
     assert "updates (reverted on every projection of that factor)" in warnings[0]
+
+
+class NonFiniteProjectionOptimiser(AbstractFactorOptimiser):
+    """
+    Stands in for a per-factor search whose weighted samples project to
+    non-finite sufficient statistics (RAL 342411, PyAutoFit#1653): the first
+    `n_failures` calls project a sample set containing a nan, then it defers
+    to an exact fit.
+    """
+
+    def __init__(self, n_failures=1):
+        super().__init__()
+        self.n_failures = n_failures
+        self.call_count = 0
+
+    def optimise(self, factor_approx, status=graph.Status()):
+        self.call_count += 1
+        if self.call_count <= self.n_failures:
+            with np.errstate(all="ignore"):
+                NormalMessage.project(np.array([1.0, 2.0, np.nan]), np.zeros(3))
+        return self.exact_fit(factor_approx, status)
+
+
+def test_nonfinite_projection_degrades_factor_step_to_previous_message():
+    """
+    A non-finite projection of one factor's samples is a failed sweep update,
+    not a failed graph fit: `factor_step` keeps the factor's previous message
+    and records the failure, rather than letting the error kill the run.
+    """
+    from autofit.graphical.expectation_propagation.optimiser import factor_step
+
+    model_approx, _, prior, _ = make_shared_variable_approx()
+    factor_approx = model_approx.factor_approximation(prior)
+
+    new_dist, status = factor_step(factor_approx, NonFiniteProjectionOptimiser(1))
+
+    assert status.flag is StatusFlag.EXCEPTION
+    assert status.updated is False
+    assert new_dist is factor_approx.model_dist
+    assert "non-finite" in status.messages[0]
+    assert "samples" in status.messages[0]
+
+
+def test_nonfinite_projection_does_not_abort_the_ep_run():
+    """
+    End to end: one bad projection on the first sweep leaves the EP run going,
+    it returns a finite mean field, and the failure stays loud in the
+    diagnostics rows.
+    """
+    model_approx, factor_graph, prior, likelihood = make_shared_variable_approx()
+
+    failing = NonFiniteProjectionOptimiser(n_failures=1)
+    optimiser = graph.EPOptimiser(
+        factor_graph,
+        factor_optimisers={prior: failing, likelihood: ExactFactorFit()},
+        paths=False,
+    )
+
+    result = optimiser.run(model_approx, max_steps=4)
+
+    assert failing.call_count > 1, "the failing factor was never retried"
+    (x,) = [v for v in result.mean_field if v.name == "x"]
+    assert np.isfinite(result.mean_field[x].mean)
+
+    flags = [
+        row["flag"]
+        for row in optimiser.diagnostics.factor_rows
+        if row["factor"] == prior.name
+    ]
+    assert StatusFlag.EXCEPTION.name in flags

test_autofit/graphical/test_unification.py

@@ -61,6 +61,39 @@ def test_projected_model():
     assert isinstance(projected_model.centre, af.UniformPrior)
 
 
+def test_projected_model_nonfinite_names_path_and_prior_id():
+    """
+    A non-finite sample for one parameter raises `ProjectionException` (not a
+    bare `AssertionError`) naming the parameter path and the prior id
+    (PyAutoFit#1653).
+    """
+    model = af.Model(af.ex.Gaussian, centre=af.UniformPrior())
+    samples = af.Samples(
+        model,
+        [
+            af.Sample(
+                -1.0,
+                -1.0,
+                weight=random() + 0.1,
+                kwargs={
+                    ("centre",): np.nan if i == 3 else random(),
+                    ("normalization",): random(),
+                    ("sigma",): random(),
+                },
+            )
+            for i in range(100)
+        ],
+    )
+    result = af.mock.MockResult(samples=samples)
+
+    with np.errstate(all="ignore"):
+        with pytest.raises(ValueError, match="path=centre") as info:
+            result.projected_model
+
+    assert isinstance(info.value, af.exc.ProjectionException)
+    assert f"id={model.centre.id}" in str(info.value)
+
+
 def test_projected_model_moments():
     """
     Regression test for PyAutoFit#1382: `projected_model` must convert the

test_autofit/messages/test_project_nonfinite.py

@@ -0,0 +1,96 @@
+"""
+A non-finite importance-weighted projection must raise a named, catchable
+exception that says which input was non-finite (PyAutoFit#1653).
+
+`AbstractMessage.project` used to end on a bare
+`assert np.isfinite(suff_stats).all()`. An `AssertionError` is not in the
+recovery tuple of `factor_step`, so one factor's bad projection killed a
+51-factor EP run (RAL 342411), and the message gave no hint of the cause.
+Under `python -O` the assert vanished and the non-finite statistics flowed on
+silently.
+
+The exception is a `ValueError` subclass, which is what `factor_step`
+recovers from, so these tests match on `ValueError` and then check the
+concrete type: against the old assert they fail with the `AssertionError`
+itself.
+"""
+
+import numpy as np
+import pytest
+
+import autofit as af
+from autofit import exc
+from autofit.messages.normal import NormalMessage
+
+nan = np.nan
+inf = np.inf
+
+
+@pytest.mark.parametrize(
+    "samples, log_weight_list, match",
+    [
+        ([1.0, 2.0, nan], [0.0, 0.0, 0.0], "non-finite samples"),
+        ([1.0, inf, 3.0], [0.0, 0.0, 0.0], "non-finite samples"),
+        ([1.0, 2.0, 3.0], [0.0, nan, 0.0], "log weights are nan or \\+inf"),
+        ([1.0, 2.0, 3.0], [0.0, inf, 0.0], "log weights are nan or \\+inf"),
+        ([1.0, 2.0, 3.0], [-inf, -inf, -inf], "all log weights are -inf"),
+        ([1e200, 2.0, 3.0], [0.0, 0.0, 0.0], "overflow"),
+    ],
+    ids=[
+        "nan_sample",
+        "inf_sample",
+        "nan_log_weight",
+        "pos_inf_log_weight",
+        "all_neg_inf_log_weights",
+        "overflow",
+    ],
+)
+def test_nonfinite_projection_raises_projection_exception(
+    samples, log_weight_list, match
+):
+    with np.errstate(all="ignore"):
+        with pytest.raises(ValueError, match=match) as info:
+            NormalMessage.project(np.array(samples), np.array(log_weight_list))
+
+    assert isinstance(info.value, exc.ProjectionException)
+    assert not isinstance(info.value, AssertionError)
+    assert issubclass(exc.ProjectionException, ValueError)
+    assert "NormalMessage.project: non-finite sufficient statistics" in str(
+        info.value
+    )
+
+
+def test_gaussian_prior_projection_names_the_prior_id():
+    prior = af.GaussianPrior(mean=0.0, sigma=1.0)
+
+    with np.errstate(all="ignore"):
+        with pytest.raises(ValueError, match="non-finite samples") as info:
+            prior.project(np.array([1.0, 2.0, nan]), np.zeros(3))
+
+    assert isinstance(info.value, exc.ProjectionException)
+    assert f"id={prior.id}" in str(info.value)
+    assert "GaussianPrior" in str(info.value)
+
+
+def test_uniform_prior_projection_names_the_prior_id():
+    """
+    `UniformPrior`'s message is a `TransformedMessage`, whose `project` drops
+    `id_` before reaching `AbstractMessage.project` — the id must still reach
+    the error, so it is added at `Prior.project`.
+    """
+    prior = af.UniformPrior(lower_limit=0.0, upper_limit=1.0)
+
+    with np.errstate(all="ignore"):
+        with pytest.raises(ValueError) as info:
+            prior.project(np.array([0.2, 0.5, nan]), np.zeros(3))
+
+    assert isinstance(info.value, exc.ProjectionException)
+    assert f"id={prior.id}" in str(info.value)
+    assert "UniformPrior" in str(info.value)
+
+
+def test_finite_projection_is_unchanged():
+    projected = NormalMessage.project(np.array([1.0, 2.0, 3.0]), np.zeros(3))
+
+    assert projected.mean == pytest.approx(2.0)
+    assert np.isfinite(projected.sigma)