From 499e087abef91f81fa2a918e7596770043aadfd1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 14:20:31 +0000 Subject: [PATCH] small datasets: honour the cap card, the JAX switch and the cap in mesh shapes Three findings of the phase-8 diagnosis of the user-workspace smoke timings, all of them the same shape: a harness-level fact the library did not act on. 1. Capped datasets whose shape cannot corroborate their stamp were deleted and re-simulated on every run. `_is_capped_at_the_current_cap` required `SMALLDAT = T` *and* a measured `data.fits` at exactly the cap, because `SMALLDAT` records "the env var was set at write time", not "capped at today's cap". Interferometer data is `(n_visibilities, 2)`, multi_dataset prefixes its FITS and datacubes nest theirs per channel, so none of them could ever corroborate: 5.2-6.5 s per script, ~26 s per autolens_workspace CI run and ~28 s per autogalaxy_workspace one, plus every local run. The `SMALLSHP` card (PyAutoNerves#159) states the proposition directly, so corroboration is no longer needed and those families are reachable. Two things keep the safety property intact. The card is read *with* the both-axes contradiction guard the full-resolution branch already applies -- both cards record the writing *process*, so a 180x180 image written in a shell exporting `PYAUTO_SMALL_DATASETS=1` carries them while being full resolution, and reusing it in a capped run is the shape-mismatch class the capped branch exists to prevent. And an absent or malformed card falls back to the shape heuristic rather than being coerced, so every pre-card dataset behaves exactly as it did. The path resolution (`_capped_data_paths`) is widened for the capped branch only, by exact suffix at two known levels -- `data.fits`, then `{waveband}_data.fits`, then `channel_*/data.fits`, never a `*.fits` glob. The full-resolution branch still reads `data.fits` alone: widening the capped branch can only move a dataset from delete to keep, whereas widening the destructive one is its own change with its own review. `test__small_regime__interferometer_dataset__is_always_regenerated` asserted the conservative behaviour this issue exists to change, and is rewritten to pin the half that stands: without a cap card, that family still regenerates. 2. `apply_sparse_operator(use_jax=True)` ignored `PYAUTO_DISABLE_JAX=1`, so the smoke profile paid 2.3-3.2 s of JIT for a backend it had asked to disable. The interferometer method now lets the harness win over an explicit `True`, through `autonerves.test_mode.disable_jax()`. The imaging method is deliberately left alone, and says why in its docstring: it has no `use_jax` argument to override -- it *is* the JAX implementation, and its NumPy sibling `apply_sparse_operator_cpu` returns a different operator class and requires numba. Routing to that under an env var would change the returned type based on the environment, which is a larger and (on these measurements) unmotivated change. 3. Pixelization mesh `shape=` did not honour the cap the data does, so a capped run reconstructed 1600-2500 source pixels from ~80 image pixels. The rectangular meshes and `image_mesh.Overlay` now cap per axis under `PYAUTO_SMALL_DATASETS=1`, with the same `respect_small_datasets=True` escape hatch `Grid2D.uniform` carries. Measured on the four HowTo chapter-3 scripts whose guards this subsumes: 43.6 s -> 17.3 s. `disable_jax` is imported defensively: an autonerves too old to carry the predicate is exactly the case the sibling literal in `dataset_util` exists for, and a hard import would turn a safe degradation into an `ImportError` at module load. Refs PyAutoLabs/PyAutoArray#528 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0151gQm9fk3XGLi5f18Urdba --- autoarray/dataset/imaging/dataset.py | 14 + autoarray/dataset/interferometer/dataset.py | 29 ++ .../inversion/mesh/image_mesh/overlay.py | 11 +- .../rectangular_bilinear_adapt_density.py | 5 +- .../mesh/rectangular_bilinear_adapt_image.py | 2 + .../mesh/rectangular_rtu_adapt_density.py | 11 + .../mesh/mesh/rectangular_rtu_adapt_image.py | 8 +- autoarray/util/dataset_util.py | 302 ++++++++++++++-- .../dataset/interferometer/test_dataset.py | 55 +++ .../pixelization/image_mesh/test_overlay.py | 19 ++ .../pixelization/mesh/test_rectangular.py | 53 +++ test_autoarray/util/test_dataset_util.py | 323 +++++++++++++++++- 12 files changed, 799 insertions(+), 33 deletions(-) diff --git a/autoarray/dataset/imaging/dataset.py b/autoarray/dataset/imaging/dataset.py index 91a74f8cb..47e5424d0 100644 --- a/autoarray/dataset/imaging/dataset.py +++ b/autoarray/dataset/imaging/dataset.py @@ -602,6 +602,20 @@ def apply_sparse_operator( Imaging A new `Imaging` dataset with the precomputed `ImagingSparseOperator` attached, enabling efficient pixelized source reconstruction via the sparse linear algebra formalism. + + Notes + ----- + `PYAUTO_DISABLE_JAX=1` is *not* honoured here, unlike + `Interferometer.apply_sparse_operator`, and the asymmetry is deliberate. There the + variable overrides a `use_jax` argument that already selects between two backends + computing the same operator. Here there is no such argument: this method is the JAX + implementation, and the NumPy/CPU alternative is the separately named + `apply_sparse_operator_cpu`, which returns a different operator class + (`SparseLinAlgImagingNumba`) and requires numba. Silently returning that under an + environment variable would change the type of the returned object based on the + environment, which is a larger change than honouring a switch -- and an unmeasured + one: every JIT cost the phase-8 workspace timings attribute to this variable + (2.3-3.2 s per script) was on the interferometer path. """ if self.psf is not None and self.psf.convolve_over_sample_size > 1: diff --git a/autoarray/dataset/interferometer/dataset.py b/autoarray/dataset/interferometer/dataset.py index 4835c529d..4b9759ee2 100644 --- a/autoarray/dataset/interferometer/dataset.py +++ b/autoarray/dataset/interferometer/dataset.py @@ -5,6 +5,24 @@ from autonerves.fitsable import ndarray_via_fits_from from autonerves import cached_property +try: + from autonerves.test_mode import disable_jax +except ImportError: + # `disable_jax()` arrives in the autonerves release that closes + # PyAutoNerves#159. Importing it unconditionally would make an autonerves + # older than that release an `ImportError` at module load -- and a + # `--no-deps` install, an editable checkout or a hand-built virtualenv can + # all put one on the path regardless of the floor in `pyproject.toml`, + # which constrains resolution only. This is the same trade `dataset_util` + # records against `SMALL_DATASETS_HEADER_KEY`: degrade to the predicate's + # own one-line body rather than fail hard to avoid restating it. Delete the + # fallback when the floor names a release carrying the predicate. + import os + + def disable_jax(): + return os.environ.get("PYAUTO_DISABLE_JAX") == "1" + + from autoarray.dataset.abstract.dataset import AbstractDataset from autoarray.dataset.grids import GridsDataset from autoarray.inversion.inversion.interferometer.inversion_interferometer_util import ( @@ -266,6 +284,14 @@ def apply_sparse_operator( use_jax If `True`, JAX is used to accelerate the NUFFT precision matrix computation. + `PYAUTO_DISABLE_JAX=1` overrides this to `False`. That variable is a + harness-level switch, not a preference: it is the documented way to force the + NumPy path (the workspace `start_here` guides name it beside `use_jax=False`), + and the smoke profiles set it so a fast run does not pay a JIT compile. An + explicit `use_jax=True` in a script -- which is the right thing for a script + demonstrating the production path to say -- must therefore not defeat it, or + the harness pays 2.3-3.2 s of compile for a backend it asked to disable. + Precondition ------------ Every visibility must have equal real and imaginary noise sigma @@ -289,6 +315,9 @@ def apply_sparse_operator( If any visibility has unequal real and imaginary noise sigma. """ + if disable_jax(): + use_jax = False + noise_map_real = np.asarray(self.noise_map.real) noise_map_imag = np.asarray(self.noise_map.imag) diff --git a/autoarray/inversion/mesh/image_mesh/overlay.py b/autoarray/inversion/mesh/image_mesh/overlay.py index d9f49f9be..c7ebaff91 100644 --- a/autoarray/inversion/mesh/image_mesh/overlay.py +++ b/autoarray/inversion/mesh/image_mesh/overlay.py @@ -8,6 +8,7 @@ from autoarray.geometry import geometry_util from autoarray.structures.grids import grid_2d_util +from autoarray.util.dataset_util import cap_mesh_shape_for_small_datasets from autoarray import numba_util @@ -158,7 +159,7 @@ def overlay_via_unmasked_overlaid_from( class Overlay(AbstractImageMesh): - def __init__(self, shape=(3, 3)): + def __init__(self, shape=(3, 3), respect_small_datasets: bool = True): """ Computes an image-mesh by overlaying a uniform grid of (y,x) coordinates over the masked image that the pixelization is fitting. @@ -176,10 +177,18 @@ def __init__(self, shape=(3, 3)): ---------- shape The 2D shape of the grid which is overlaid over the grid to determine the image mesh. + respect_small_datasets + When `PYAUTO_SMALL_DATASETS=1` is set, `shape` is capped per axis to the small-datasets + cap, matching the cap `Grid2D.uniform` and `Mask2D.circular` apply to the data. Pass + `False` to opt out for an image mesh whose resolution is load-bearing for the script. """ super().__init__() + shape = cap_mesh_shape_for_small_datasets( + shape, respect_small_datasets=respect_small_datasets + ) + self.shape = (int(shape[0]), int(shape[1])) def image_plane_mesh_grid_from( diff --git a/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_density.py b/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_density.py index 5087afa44..5f9204bec 100644 --- a/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_density.py +++ b/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_density.py @@ -10,6 +10,7 @@ class RectangularBilinearAdaptDensity(RectangularRTUAdaptDensity): def __init__( self, shape: Tuple[int, int] = (3, 3), + respect_small_datasets: bool = True, ): """ A rectangular mesh of pixels used to reconstruct a source on a regular @@ -71,7 +72,9 @@ def __init__( If either dimension is less than 3, as a minimum of 3×3 pixels is required to define interior and boundary structure. """ - super().__init__(shape=shape) + super().__init__( + shape=shape, respect_small_datasets=respect_small_datasets + ) @property def interpolator_kwargs(self) -> dict: diff --git a/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_image.py b/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_image.py index 48b6a925b..511003cf1 100644 --- a/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_image.py +++ b/autoarray/inversion/mesh/mesh/rectangular_bilinear_adapt_image.py @@ -12,6 +12,7 @@ def __init__( shape: Tuple[int, int] = (3, 3), weight_power: float = 1.0, weight_floor: float = 0.0, + respect_small_datasets: bool = True, ): """ A rectangular mesh of pixels used to reconstruct a source on a regular @@ -69,6 +70,7 @@ def __init__( shape=shape, weight_power=weight_power, weight_floor=weight_floor, + respect_small_datasets=respect_small_datasets, ) @property diff --git a/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_density.py b/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_density.py index 9d1469499..753faa297 100644 --- a/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_density.py +++ b/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_density.py @@ -9,6 +9,7 @@ from autoarray.inversion.mesh.border_relocator import BorderRelocator from autoarray.structures.grids import grid_2d_util +from autoarray.util.dataset_util import cap_mesh_shape_for_small_datasets from autoarray import exc @@ -71,6 +72,7 @@ def __init__( shape: Tuple[int, int] = (3, 3), bandwidth: Optional[float] = None, n_knots: Optional[int] = None, + respect_small_datasets: bool = True, ): """ A rectangular mesh of pixels used to reconstruct a source on a regular @@ -132,6 +134,11 @@ def __init__( n_knots Size of the fixed knot table used to invert the CDF. Defaults to the kernel default. + respect_small_datasets + When ``PYAUTO_SMALL_DATASETS=1`` is set, `shape` is capped per axis + to the small-datasets cap, matching the cap `Grid2D.uniform` and + `Mask2D.circular` apply to the data. Pass ``False`` to opt out for a + mesh whose resolution is load-bearing for the script. Raises ------ @@ -144,6 +151,10 @@ def __init__( KERNEL_CDF_DEFAULT_KNOTS, ) + shape = cap_mesh_shape_for_small_datasets( + shape, respect_small_datasets=respect_small_datasets + ) + if shape[0] <= 2 or shape[1] <= 2: raise exc.MeshException( "The rectangular pixelization must be at least dimensions 3x3" diff --git a/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_image.py b/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_image.py index 3b1c8537d..5a9e07575 100644 --- a/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_image.py +++ b/autoarray/inversion/mesh/mesh/rectangular_rtu_adapt_image.py @@ -15,6 +15,7 @@ def __init__( weight_floor: float = 0.0, bandwidth: Optional[float] = None, n_knots: Optional[int] = None, + respect_small_datasets: bool = True, ): """ A uniform rectangular mesh of pixels used to reconstruct a source on a @@ -82,7 +83,12 @@ def __init__( the kernel default. """ - super().__init__(shape=shape, bandwidth=bandwidth, n_knots=n_knots) + super().__init__( + shape=shape, + bandwidth=bandwidth, + n_knots=n_knots, + respect_small_datasets=respect_small_datasets, + ) self.weight_power = weight_power self.weight_floor = weight_floor diff --git a/autoarray/util/dataset_util.py b/autoarray/util/dataset_util.py index b03699582..7eb3a145c 100644 --- a/autoarray/util/dataset_util.py +++ b/autoarray/util/dataset_util.py @@ -22,6 +22,27 @@ # Keep in sync with PyAutoNerves#153. SMALL_DATASETS_HEADER_KEY = "SMALLDAT" +# The cap the writing process was capping to, as ``"x"``, written by +# the same PyAutoNerves stamp (``SMALL_DATASETS_SHAPE_HEADER_KEY`` there). +# +# ``SMALLDAT`` records the wrong proposition for the question this module asks. +# ``SMALLDAT = T`` means "the env var was set in the writing process"; the +# question is "capped at *today's* cap", which is why every reader below has to +# corroborate the card against the measured shape of the data. This card answers +# the question directly, so no corroboration is needed -- which is what lets the +# families whose shape *cannot* corroborate (interferometer ``(n_visibilities, +# 2)``, multi_dataset's prefixed FITS, datacube's per-channel FITS) be reused at +# all rather than being deleted and re-simulated on every single run. +# +# Duplicated as a literal for exactly the reason recorded above against +# ``SMALL_DATASETS_HEADER_KEY``, and with more force: an autonerves too old to +# *write* this card is precisely an autonerves too old to export its name, so an +# import would turn "the card is absent" -- the case this reader already handles +# by falling back -- into an ``ImportError`` at module load. +# Keep in sync with PyAutoNerves#159; ``test_dataset_util.py`` pins the two +# together whenever a stamp-aware autonerves is importable. +SMALL_DATASETS_SHAPE_HEADER_KEY = "SMALLSHP" + def cap_array_2d_for_small_datasets(array_2d, pixel_scales): """ @@ -85,6 +106,46 @@ def cap_array_2d_for_small_datasets(array_2d, pixel_scales): ) +def cap_mesh_shape_for_small_datasets(shape, respect_small_datasets=True): + """ + Cap a pixelization / image-mesh ``shape`` to the small-datasets cap when + ``PYAUTO_SMALL_DATASETS=1`` is active, per axis. + + Returns ``shape`` unchanged when the env var is not set to ``"1"``, when + ``respect_small_datasets`` is ``False``, or when the shape is already + at-or-below the cap on both axes. + + This is the mesh-side counterpart of the cap ``Grid2D.uniform`` and + ``Mask2D.circular`` already apply to ``shape_native``, and it exists because + the two were out of step. A capped run reduces the *data* to a 16x16 image + while a pixelization's ``shape=`` stayed at whatever the script asked for, + so the smoke profile reconstructed 1600-2500 source pixels from ~80 image + pixels -- an inversion far larger than the data it is fitting, and the + dominant cost of the four HowTo chapter-3 scripts (43.6 s -> 17.3 s once + capped). + + Capped per axis with ``min``, never rewritten to the cap: a mesh smaller + than the cap on one axis is a mesh the script chose, and shrinking is the + only direction this may move. + + ``respect_small_datasets`` is the same escape hatch ``Grid2D.uniform`` + carries, for the same reason -- a script whose mesh resolution is + load-bearing (a visualization of a specific reconstruction, a test pinning + a pixel count) opts out at the call site rather than by unsetting the env + var for the whole run. + """ + if not respect_small_datasets: + return shape + + if os.environ.get("PYAUTO_SMALL_DATASETS") != "1": + return shape + + return ( + min(int(shape[0]), SMALL_DATASETS_SHAPE_NATIVE[0]), + min(int(shape[1]), SMALL_DATASETS_SHAPE_NATIVE[1]), + ) + + def _on_disk_shape_native(data_path): """ Returns the ``(rows, columns)`` shape of the first 2D image in the FITS file @@ -129,11 +190,21 @@ def _small_datasets_stamp_on_disk(dataset_path): Callers must treat ``None`` as "leave the dataset alone" and fall back to :func:`_is_small_datasets_on_disk`. Unknown must never mean "full". """ - from astropy.io import fits + return _stamp_card_in_file(Path(dataset_path) / "data.fits") - data_path = Path(dataset_path) / "data.fits" - if not data_path.exists(): +def _stamp_card_in_file(data_path): + """ + Returns the ``SMALLDAT`` regime recorded in the FITS file at ``data_path``, + as the same tri-state :func:`_small_datasets_stamp_on_disk` documents. + + Split out from that function so the multi-file dataset families -- whose + data does not live at ``/data.fits`` -- can be read with the + same rules rather than a second, subtly different implementation. + """ + from astropy.io import fits + + if not Path(data_path).exists(): return None try: @@ -148,6 +219,113 @@ def _small_datasets_stamp_on_disk(dataset_path): return None +def _shape_card_in_file(data_path): + """ + Returns the cap recorded by the ``SMALLSHP`` card in the FITS file at + ``data_path`` as a ``(rows, columns)`` tuple, or ``None`` when there is no + usable card. + + ``None`` means "unknown cap" and never "no cap": it is returned for a + missing or unreadable file, a file written before the card existed, a file + written at full resolution (where there is no cap to record), and a card + whose value is not the ``"x"`` string this stack writes. Every + caller must treat it as "fall back to what you did before the card + existed", which for :func:`_is_capped_at_the_current_cap` is the shape + heuristic. + + A malformed card is deliberately read as unknown rather than coerced. The + predicate this feeds ends in ``shutil.rmtree`` in one direction and in + "reuse this data" in the other, and a card this code did not write is not a + card this code can trust in either. + """ + from astropy.io import fits + + if not Path(data_path).exists(): + return None + + try: + with fits.open(data_path) as hdu_list: + for hdu in hdu_list: + value = hdu.header.get(SMALL_DATASETS_SHAPE_HEADER_KEY) + + if not isinstance(value, str): + continue + + rows, separator, columns = value.partition("x") + + if not separator: + return None + + try: + return (int(rows), int(columns)) + except ValueError: + return None + except Exception: + return None + + return None + + +def _small_datasets_shape_on_disk(dataset_path): + """ + Returns the cap recorded in ``/data.fits``'s ``SMALLSHP`` + card, or ``None`` if there is none (see :func:`_shape_card_in_file`). + """ + return _shape_card_in_file(Path(dataset_path) / "data.fits") + + +def _capped_data_paths(dataset_path): + """ + Returns the FITS files that carry ``dataset_path``'s data array(s), as the + list :func:`_is_capped_at_the_current_cap` reads its cards from. + + Three layouts, tried in order, and the first that matches wins: + + - ``/data.fits`` -- the ordinary case, roughly 228 of the 253 + ``should_simulate`` call sites in autolens_workspace. When it exists it is + the only file consulted, so nothing about the single-file case changes. + - ``/{prefix}_data.fits`` -- multi_dataset, which prefixes + every file with its waveband. + - ``/channel_*/data.fits`` -- interferometer datacubes, which + nest one dataset per channel. + + The suffix is matched exactly (``_data.fits``, never ``*.fits``) and only at + the two levels above. That narrowness is the whole safety argument: this + list feeds a predicate whose *false* branch ends in ``shutil.rmtree``, and + every trap recorded in autolens_workspace_test#260 was a widened match + reaching a file it should not have -- a bare ``*.fits`` glob would sweep in + ``psf.fits``, which is legitimately tiny at full resolution. + + Returns ``[]`` for a directory with none of those, which callers must read + as "no verdict" -- exactly the position those datasets were in before this + resolution existed. + """ + path = Path(dataset_path) + + if not path.is_dir(): + return [] + + data_path = path / "data.fits" + + if data_path.exists(): + return [data_path] + + prefixed = sorted( + candidate + for candidate in path.glob("*_data.fits") + if candidate.is_file() + ) + + if prefixed: + return prefixed + + return sorted( + candidate + for candidate in path.glob("channel_*/data.fits") + if candidate.is_file() + ) + + def _is_small_datasets_on_disk(dataset_path): """ Returns True if the dataset on disk at ``dataset_path`` was written by a @@ -213,7 +391,21 @@ def _stamp_contradicted_by_shape(dataset_path): deletion, so failing to read the file must not silently protect a genuinely stale dataset the stamp correctly identified. """ - shape = _on_disk_shape_native(Path(dataset_path) / "data.fits") + return _shape_contradicts_the_cap(Path(dataset_path) / "data.fits") + + +def _shape_contradicts_the_cap(data_path): + """ + Returns True when the FITS file at ``data_path`` is larger than the cap on + **both** axes, i.e. when its shape says it cannot have been written by a + capped write however its provenance cards read. + + Split out of :func:`_stamp_contradicted_by_shape` so both branches of + :func:`should_simulate` can apply the same guard to the same question. Both + axes, never either, and unknown means not contradicted: the reasoning for + both is in that function's docstring. + """ + shape = _on_disk_shape_native(data_path) return ( shape is not None @@ -241,23 +433,71 @@ class of silent-stale-dataset bug the stamp was introduced to prevent, the writer's *environment*, not a measured property of the data. Neither branch may treat it as unfalsifiable. - Interferometer datasets deliberately never satisfy this. Their ``data.fits`` - is ``(n_visibilities, 2)`` -- its shape is fixed by the committed uv file and - does not change under the cap -- so shape cannot corroborate their stamp, and - a capped interferometer dataset is regenerated on every run exactly as it was - before. That is the conservative choice and it is deliberate: the alternative - is trusting the stamp alone for precisely the family whose corruption is - invisible. - - Anything without a readable ``data.fits`` at the top level -- JSON-only - datasets, datacubes nesting theirs in ``channel_XXX/``, multi_dataset's - prefixed names -- also returns False and is regenerated, preserving today's - behaviour for the families this cannot speak about. + The corroboration the second half needs no longer has to come from the + data's shape. A capped writer now also records the cap it used, as + ``SMALLSHP = '16x16'`` (:data:`SMALL_DATASETS_SHAPE_HEADER_KEY`), which is + that proposition stated directly rather than inferred. When the card is + present it is read and the shape is not consulted; when it is absent -- + every file written before it existed -- the shape heuristic runs exactly as + it did before, so a pre-card dataset behaves identically. + + That matters because the shape heuristic is structurally blind to three + families. Interferometer ``data.fits`` is ``(n_visibilities, 2)``, fixed by + the committed uv file and unchanged by the cap; multi_dataset prefixes its + files (``{waveband}_data.fits``); datacubes nest theirs in + ``channel_XXX/``. None of them could ever corroborate a stamp, so all three + were deleted and re-simulated on **every** run -- 5.2-6.5 s per script, and + the dominant cost of several smoke entries in both workspaces. They are + reached now: the cards are read from every file + :func:`_capped_data_paths` resolves, and every one of them must agree. + + The card is read *with* the contradiction guard the full-resolution branch + already applies (:func:`_shape_contradicts_the_cap`), and that pairing is + load-bearing. Both cards record the writing **process**, not a measured + property of the array: a 180x180 image written in a shell exporting + ``PYAUTO_SMALL_DATASETS=1`` -- a user converting real data, or any array + written with ``respect_small_datasets=False`` -- carries ``SMALLDAT = T`` + and ``SMALLSHP = '16x16'`` while being full resolution. Trusting the card + alone would reuse it in a capped run, which is the shape-mismatch class the + capped branch exists to prevent. A file over the cap on **both** axes is + therefore refused however its cards read, which leaves interferometer + ``(n_visibilities, 2)`` -- over on one axis only -- reachable, as intended. + + The safety property is unchanged, and is the one this predicate has always + had to hold: relative to the shape-only rule, this can only ever move a + dataset from *delete* to *keep*, and only when the writing process recorded + **exactly** today's cap. A dataset stamped at a different cap has a + mismatching card and is still deleted -- which is the whole point of the + second half, and the property PyAutoArray#471 / PyAutoNerves#153 + established. An absent or malformed card falls back rather than being + coerced, so "unknown" still never means "capped". + + A dataset with none of the three layouts -- JSON-only datasets, the + FITS-less pair named in :func:`should_simulate`'s "Known gap" -- still + returns False and is regenerated, preserving today's behaviour for the + families this cannot speak about. """ - return ( - _small_datasets_stamp_on_disk(dataset_path) is True - and _is_small_datasets_on_disk(dataset_path) - ) + data_paths = _capped_data_paths(dataset_path) + + if not data_paths: + return False + + for data_path in data_paths: + if _stamp_card_in_file(data_path) is not True: + return False + + if _shape_contradicts_the_cap(data_path): + return False + + recorded = _shape_card_in_file(data_path) + + if recorded is not None: + if recorded != SMALL_DATASETS_SHAPE_NATIVE: + return False + elif _on_disk_shape_native(data_path) != SMALL_DATASETS_SHAPE_NATIVE: + return False + + return True def should_simulate(dataset_path): @@ -331,14 +571,28 @@ def should_simulate(dataset_path): Known gap --------- - This reads ``/data.fits`` and nothing else, which covers - roughly 228 of the 253 ``should_simulate`` call sites in autolens_workspace. - The rest have no file of that name at that level and so get no verdict: + The **capped** branch resolves three layouts (:func:`_capped_data_paths`): + ``/data.fits``, multi_dataset's ``{waveband}_data.fits``, and + a datacube's ``channel_XXX/data.fits``. The **full-resolution** branch below + still reads ``/data.fits`` and nothing else. + + That asymmetry is deliberate, and is the safety property stated as code. + Widening the capped branch can only move a dataset from *delete* to *keep*: + a directory with no ``data.fits`` had no verdict, so it was deleted and + re-simulated on every run, and a resolved, agreeing set of cards is the + only thing that now spares it. Widening the full branch would move datasets + the other way, into ``shutil.rmtree``, and growing the reach of a + destructive predicate is its own change with its own review -- not a rider + on the one that stops re-simulating data that is already correct. + + So under ``PYAUTO_SMALL_DATASETS=1`` the families below are covered; in the + full-resolution regime they still get no verdict, which is *keep*: - interferometer **datacube** datasets, whose FITS sit in ``channel_XXX/`` subdirectories; - **multi_dataset** datasets, which prefix the name (``{waveband}_data.fits``); - - **sample** datasets, which nest under ``dataset_N/``; + - **sample** datasets, which nest under ``dataset_N/`` (covered by neither + branch); - the two FITS-less directories, ``dataset/weak/simple`` and ``dataset/point_source/multiple_sources``, which a FITS-header stamp cannot reach under any placement. diff --git a/test_autoarray/dataset/interferometer/test_dataset.py b/test_autoarray/dataset/interferometer/test_dataset.py index e6562fe94..b7e1e4766 100644 --- a/test_autoarray/dataset/interferometer/test_dataset.py +++ b/test_autoarray/dataset/interferometer/test_dataset.py @@ -294,3 +294,58 @@ def test__different_interferometer_without_mock_objects__customize_constructor_i assert (dataset.data == 1.0 + 1.0j * np.ones((19,))).all() assert (dataset.noise_map == 2.0 + 2.0j * np.ones((19,))).all() assert (dataset.uv_wavelengths == 3.0 * np.ones((19, 2))).all() + + +def test__apply_sparse_operator__disable_jax_overrides_an_explicit_use_jax( + monkeypatch, mask_2d_7x7 +): + # `PYAUTO_DISABLE_JAX=1` is a harness-level override, not a preference: it is + # the documented way to force the NumPy path and the smoke profiles set it so + # a fast run does not pay a JIT compile (2.3-3.2 s per interferometer script). + # A script demonstrating the production path says `use_jax=True`, and that + # must not defeat the harness. + n_visibilities = 5 + rng = np.random.default_rng(seed=0) + data = aa.Visibilities( + visibilities=rng.normal(size=(n_visibilities, 2)).astype(np.float64) + ) + noise_map = aa.VisibilitiesNoiseMap( + visibilities=np.ones((n_visibilities, 2), dtype=np.float64) + ) + uv_wavelengths = rng.normal(size=(n_visibilities, 2)).astype(np.float64) + + recorded = [] + original = aa.Interferometer.psf_precision_operator_from + + def spy(self, *args, use_jax=False, **kwargs): + recorded.append(use_jax) + # Always compute on the NumPy path, so the assertion is about the flag + # that arrives here and never about whether JAX is installed. + return original(self, *args, use_jax=False, **kwargs) + + monkeypatch.setattr(aa.Interferometer, "psf_precision_operator_from", spy) + + def dataset(): + return aa.Interferometer( + data=data, + noise_map=noise_map, + uv_wavelengths=uv_wavelengths, + real_space_mask=mask_2d_7x7, + transformer_class=transformer.TransformerDFT, + ) + + monkeypatch.setenv("PYAUTO_DISABLE_JAX", "1") + dataset().apply_sparse_operator(use_jax=True) + + assert recorded == [False] + + # Every other value of the variable, and its absence, leave the caller's + # choice alone -- the predicate compares against the exact string "1". + for value in ["0", "true", "True"]: + monkeypatch.setenv("PYAUTO_DISABLE_JAX", value) + dataset().apply_sparse_operator(use_jax=True) + + monkeypatch.delenv("PYAUTO_DISABLE_JAX", raising=False) + dataset().apply_sparse_operator(use_jax=True) + + assert recorded == [False, True, True, True, True] diff --git a/test_autoarray/inversion/pixelization/image_mesh/test_overlay.py b/test_autoarray/inversion/pixelization/image_mesh/test_overlay.py index 79b9a31ee..84dcd56e9 100644 --- a/test_autoarray/inversion/pixelization/image_mesh/test_overlay.py +++ b/test_autoarray/inversion/pixelization/image_mesh/test_overlay.py @@ -503,3 +503,22 @@ def test__image_plane_mesh_grid_from__offset_mask__origin_shift_corrects(): ] ) ).all() + + +def test__small_datasets__overlay_shape_is_capped_per_axis(monkeypatch): + # The image mesh is capped for the same reason as the pixelization mesh: the + # data it is overlaid on is 16x16 under PYAUTO_SMALL_DATASETS=1. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + assert aa.image_mesh.Overlay(shape=(50, 40)).shape == (16, 16) + assert aa.image_mesh.Overlay(shape=(8, 30)).shape == (8, 16) + assert ( + aa.image_mesh.Overlay(shape=(50, 40), respect_small_datasets=False).shape + == (50, 40) + ) + + +def test__small_datasets_unset__overlay_shape_is_untouched(monkeypatch): + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False) + + assert aa.image_mesh.Overlay(shape=(50, 40)).shape == (50, 40) diff --git a/test_autoarray/inversion/pixelization/mesh/test_rectangular.py b/test_autoarray/inversion/pixelization/mesh/test_rectangular.py index 3a4a9f127..8c018c7a9 100644 --- a/test_autoarray/inversion/pixelization/mesh/test_rectangular.py +++ b/test_autoarray/inversion/pixelization/mesh/test_rectangular.py @@ -314,3 +314,56 @@ def test__bilinear__split_regularization_not_supported(): ).supports_split_regularization is False ) + + +def test__small_datasets__mesh_shape_is_capped_per_axis(monkeypatch): + # `Grid2D.uniform` and `Mask2D.circular` cap the DATA to 16x16 under + # PYAUTO_SMALL_DATASETS=1 while a pixelization's `shape=` did not, so a + # capped run reconstructed 1600-2500 source pixels from ~80 image pixels -- + # an inversion far larger than the data it fits. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + for cls in [ + aa.mesh.RectangularUniform, + aa.mesh.RectangularRTUAdaptDensity, + aa.mesh.RectangularRTUAdaptImage, + aa.mesh.RectangularBilinearAdaptDensity, + aa.mesh.RectangularBilinearAdaptImage, + ]: + mesh = cls(shape=(50, 40)) + + assert mesh.shape == (16, 16) + assert mesh.pixels == 16 * 16 + + # Per axis with `min`, never rewritten to the cap: a mesh already + # smaller than the cap is a mesh the script chose. + assert cls(shape=(8, 30)).shape == (8, 16) + + # The escape hatch, for a mesh whose resolution is load-bearing. + assert cls(shape=(50, 40), respect_small_datasets=False).shape == (50, 40) + + +def test__small_datasets_unset__mesh_shape_is_untouched(monkeypatch): + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False) + + for cls in [ + aa.mesh.RectangularUniform, + aa.mesh.RectangularRTUAdaptDensity, + aa.mesh.RectangularRTUAdaptImage, + aa.mesh.RectangularBilinearAdaptDensity, + aa.mesh.RectangularBilinearAdaptImage, + ]: + mesh = cls(shape=(50, 40)) + + assert mesh.shape == (50, 40) + assert mesh.pixels == 50 * 40 + + +def test__small_datasets__cap_still_respects_the_3x3_minimum(monkeypatch): + # The cap only ever shrinks, and the mesh's own floor still applies to what + # the caller asked for -- a capped run must not turn a 2x2 request into a + # silently valid mesh. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + with pytest.raises(aa.exc.MeshException): + aa.mesh.RectangularUniform(shape=(2, 2)) diff --git a/test_autoarray/util/test_dataset_util.py b/test_autoarray/util/test_dataset_util.py index 5e80c88ac..3ee2023ae 100644 --- a/test_autoarray/util/test_dataset_util.py +++ b/test_autoarray/util/test_dataset_util.py @@ -1,4 +1,5 @@ import numpy as np +import pytest import autoarray as aa @@ -114,7 +115,11 @@ def test__env_set__non_square_above_cap__center_crops_to_16x16(monkeypatch): _small_datasets_stamp_on_disk, _stamp_contradicted_by_shape, _is_capped_at_the_current_cap, + _capped_data_paths, + _shape_card_in_file, + _small_datasets_shape_on_disk, SMALL_DATASETS_HEADER_KEY, + SMALL_DATASETS_SHAPE_HEADER_KEY, ) def _set_stamp(file_path, stamp): @@ -245,16 +250,18 @@ def test__small_regime__full_resolution_dataset__is_regenerated(monkeypatch, tmp assert not dataset_path.exists() -def test__small_regime__interferometer_dataset__is_always_regenerated( +def test__small_regime__interferometer_dataset__without_a_cap_card__is_regenerated( monkeypatch, tmp_path ): - # Deliberate and conservative. An interferometer data.fits is - # (n_visibilities, 2) -- its shape is fixed by the committed uv file and does - # not change under the cap -- so shape cannot corroborate its stamp. Rather - # than trust the stamp alone for precisely the family whose corruption is - # invisible, this family keeps regenerating every run, as before. + # An interferometer data.fits is (n_visibilities, 2) -- its shape is fixed by + # the committed uv file and does not change under the cap -- so shape cannot + # corroborate its stamp. Without a cap card there is nothing else to consult, + # so this family regenerates every run, as it always did. (With one it does + # not: see `test__cap_card__interferometer_dataset__is_kept_instead_of_resimulated`, + # which is the behaviour PyAutoArray#528 changes.) monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") dataset_path = _write_dataset(tmp_path / "dataset", (360, 2), stamp=True) + _set_shape_card(dataset_path / "data.fits", None) assert _small_datasets_stamp_on_disk(str(dataset_path)) is True assert should_simulate(str(dataset_path)) is True @@ -546,3 +553,307 @@ def test__contradiction_guard__unknown_shape_does_not_block_a_deletion( missing = tmp_path / "gone" assert _stamp_contradicted_by_shape(str(missing)) is False + + +def _set_shape_card(file_path, value): + """ + Force the ``SMALLSHP`` cap card on an already-written FITS file. + + Forced for exactly the reason :func:`_set_stamp` documents for ``SMALLDAT``: + the card is written by PyAutoNerves, this suite owns the *reader*, and a + test that leaned on the writer would pass through the shape fallback in any + environment resolving autonerves from PyPI rather than exercising the card. + + ``value=None`` strips the card, producing a file byte-equivalent to one + written before it existed -- which is what keeps the fallback exercised. + """ + from astropy.io import fits + + with fits.open(file_path, mode="update") as hdu_list: + for hdu in hdu_list: + if value is None: + hdu.header.pop(SMALL_DATASETS_SHAPE_HEADER_KEY, None) + else: + hdu.header[SMALL_DATASETS_SHAPE_HEADER_KEY] = value + + +def _write_fits(file_path, shape, stamp=True, shape_card="16x16"): + """ + Write a single FITS file with the two provenance cards forced onto it. + """ + file_path.parent.mkdir(parents=True, exist_ok=True) + + aa.output_to_fits( + values=np.ones(shape), file_path=str(file_path), overwrite=True + ) + + _set_stamp(file_path, stamp) + _set_shape_card(file_path, shape_card) + + return file_path + + +def test__cap_card__interferometer_dataset__is_kept_instead_of_resimulated( + monkeypatch, tmp_path +): + # THE FIX. Interferometer data.fits is (n_visibilities, 2), a shape the cap + # does not change, so it could never corroborate its own stamp and was + # deleted and re-simulated on EVERY run. The cap card states the + # proposition directly, so no corroboration is needed. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "data.fits", (360, 2)) + + assert _small_datasets_shape_on_disk(str(dataset_path)) == (16, 16) + assert _is_capped_at_the_current_cap(str(dataset_path)) is True + + assert should_simulate(str(dataset_path)) is False + assert (dataset_path / "data.fits").exists() + + +def test__cap_card__at_a_DIFFERENT_cap__is_still_regenerated(monkeypatch, tmp_path): + # The safety property PyAutoArray#471 / PyAutoNerves#153 established, now + # carried by the card rather than by shape: a dataset written under a + # DIFFERENT cap is stale and must still be deleted, even though its stamp + # says T and its shape happens to match today's cap. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits( + dataset_path / "data.fits", SMALL_DATASETS_SHAPE_NATIVE, shape_card="32x32" + ) + + assert _is_capped_at_the_current_cap(str(dataset_path)) is False + + assert should_simulate(str(dataset_path)) is True + assert not dataset_path.exists() + + +def test__cap_card__absent__is_exactly_todays_behaviour(monkeypatch, tmp_path): + # Every file written before the card existed has none, so its absence must + # leave the shape heuristic in charge -- not be read as "capped". + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + at_cap = tmp_path / "at_cap" + _write_fits(at_cap / "data.fits", SMALL_DATASETS_SHAPE_NATIVE, shape_card=None) + assert _small_datasets_shape_on_disk(str(at_cap)) is None + assert _is_capped_at_the_current_cap(str(at_cap)) is True + + interferometer = tmp_path / "interferometer" + _write_fits(interferometer / "data.fits", (360, 2), shape_card=None) + assert _is_capped_at_the_current_cap(str(interferometer)) is False + + +def test__cap_card__malformed__is_unknown_and_falls_back(monkeypatch, tmp_path): + # A card this code did not write is not a card it can trust. Unknown falls + # back to the shape heuristic; it is never coerced into a cap. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + for card in ["16", "sixteen x sixteen", "16x", "", True]: + dataset_path = tmp_path / f"dataset_{str(card)[:6]}" + _write_fits( + dataset_path / "data.fits", SMALL_DATASETS_SHAPE_NATIVE, shape_card=card + ) + + assert _shape_card_in_file(dataset_path / "data.fits") is None + + # Falls back: the shape IS at the cap, so this is kept exactly as it was + # before the card existed. + assert _is_capped_at_the_current_cap(str(dataset_path)) is True + + +def test__cap_card__unstamped_file__is_regenerated_however_the_card_reads( + monkeypatch, tmp_path +): + # The cap card refines the second half of the predicate; it does not replace + # the first. A file with no SMALLDAT stamp is still unknown, and unknown is + # still not "capped". + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "data.fits", (360, 2), stamp=None) + + assert _is_capped_at_the_current_cap(str(dataset_path)) is False + assert should_simulate(str(dataset_path)) is True + + +def test__multi_dataset__prefixed_fits__are_resolved_and_kept(monkeypatch, tmp_path): + # multi_dataset prefixes every file with its waveband, so there is no + # `data.fits` at all and the dataset got no verdict -- deleted and + # re-simulated on every run. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "g_data.fits", (16, 16)) + _write_fits(dataset_path / "r_data.fits", (16, 16)) + _write_fits(dataset_path / "g_psf.fits", (11, 11), stamp=None, shape_card=None) + + assert [path.name for path in _capped_data_paths(str(dataset_path))] == [ + "g_data.fits", + "r_data.fits", + ] + assert _is_capped_at_the_current_cap(str(dataset_path)) is True + + assert should_simulate(str(dataset_path)) is False + assert (dataset_path / "g_data.fits").exists() + + +def test__multi_dataset__one_waveband_disagrees__whole_dataset_is_regenerated( + monkeypatch, tmp_path +): + # Every resolved file must agree. A dataset half-written under a different + # cap is stale as a whole, and a per-file verdict would reuse the half that + # happens to match. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "g_data.fits", (16, 16)) + _write_fits(dataset_path / "r_data.fits", (16, 16), shape_card="32x32") + + assert _is_capped_at_the_current_cap(str(dataset_path)) is False + assert should_simulate(str(dataset_path)) is True + + +def test__datacube__channel_fits__are_resolved_and_kept(monkeypatch, tmp_path): + # Datacubes nest one dataset per channel, so the same "no verdict" applied. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "channel_000" / "data.fits", (360, 2)) + _write_fits(dataset_path / "channel_001" / "data.fits", (360, 2)) + + assert [path.parent.name for path in _capped_data_paths(str(dataset_path))] == [ + "channel_000", + "channel_001", + ] + assert _is_capped_at_the_current_cap(str(dataset_path)) is True + assert should_simulate(str(dataset_path)) is False + + +def test__path_resolution__top_level_data_fits_wins_and_psf_is_never_resolved( + tmp_path, +): + # The resolution is by exact suffix at two known levels, never a `*.fits` + # glob: the trap recorded in autolens_workspace_test#260 was a widened match + # reaching `psf.fits`, which is legitimately tiny at full resolution. + dataset_path = tmp_path / "dataset" + _write_fits(dataset_path / "data.fits", (16, 16)) + _write_fits(dataset_path / "g_data.fits", (16, 16)) + _write_fits(dataset_path / "psf.fits", (11, 11)) + _write_fits(dataset_path / "channel_000" / "data.fits", (16, 16)) + + assert [path.name for path in _capped_data_paths(str(dataset_path))] == [ + "data.fits" + ] + + psf_only = tmp_path / "psf_only" + _write_fits(psf_only / "psf.fits", (11, 11)) + assert _capped_data_paths(str(psf_only)) == [] + + json_only = tmp_path / "json_only" + json_only.mkdir() + (json_only / "dataset.json").write_text(json.dumps({"a": 1})) + assert _capped_data_paths(str(json_only)) == [] + + +def test__safety_property__the_cap_card_only_spares__and_only_at_todays_cap( + monkeypatch, tmp_path +): + # The property stated as a test rather than as prose, over every combination + # of shape, stamp and card: + # + # - a KEEP requires the stamp, a shape that does not contradict the cap, + # and either a card reading exactly today's cap or no card and a shape + # measuring exactly today's cap; + # - a dataset the shape-only rule kept is still kept, unless its card names + # a different cap -- the one case where the card condemns, which is the + # stale-dataset case it exists to catch; + # - a card naming a different cap never keeps. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + shapes = [SMALL_DATASETS_SHAPE_NATIVE, (360, 2), (150, 150)] + stamps = [True, False, None] + cards = ["16x16", "32x32", None, "not-a-shape"] + + for index, shape in enumerate(shapes): + for stamp in stamps: + for card in cards: + dataset_path = tmp_path / f"dataset_{index}_{stamp}_{card}" + _write_fits( + dataset_path / "data.fits", + shape, + stamp=stamp, + shape_card=card, + ) + + kept = _is_capped_at_the_current_cap(str(dataset_path)) + + recorded = _small_datasets_shape_on_disk(str(dataset_path)) + stamped = _small_datasets_stamp_on_disk(str(dataset_path)) is True + contradicted = _stamp_contradicted_by_shape(str(dataset_path)) + measured_at_the_cap = _is_small_datasets_on_disk(str(dataset_path)) + + if kept: + assert stamped + assert not contradicted + assert ( + recorded == SMALL_DATASETS_SHAPE_NATIVE + if recorded is not None + else measured_at_the_cap + ) + + # The shape-only rule as it stood before the cap card. + if stamped and measured_at_the_cap: + assert kept is ( + recorded is None or recorded == SMALL_DATASETS_SHAPE_NATIVE + ) + + if recorded is not None and recorded != SMALL_DATASETS_SHAPE_NATIVE: + assert kept is False + + +def test__cap_card_constants__agree_with_the_writer_in_autonerves(): + # The literal is duplicated on purpose (see the module docstring against + # SMALL_DATASETS_HEADER_KEY); this is the drift guard the duplication needs, + # which only a repo that can see both constants can provide. Skipped on an + # autonerves too old to carry them, which is the case the literal exists for. + autonerves_test_mode = pytest.importorskip("autonerves.test_mode") + autonerves_fitsable = pytest.importorskip("autonerves.fitsable") + + if not hasattr(autonerves_fitsable, "SMALL_DATASETS_SHAPE_HEADER_KEY"): + pytest.skip("autonerves on this path predates the cap card") + + assert ( + SMALL_DATASETS_SHAPE_HEADER_KEY + == autonerves_fitsable.SMALL_DATASETS_SHAPE_HEADER_KEY + ) + assert ( + SMALL_DATASETS_SHAPE_NATIVE + == autonerves_test_mode.SMALL_DATASETS_SHAPE_NATIVE + ) + + +def test__cap_card__written_by_the_stack_is_read_back(monkeypatch, tmp_path): + # End to end through the real writer rather than the forced cards above, so + # the reader is pinned to what `aa.output_to_fits` actually emits under the + # capped regime whenever the installed autonerves is new enough to emit it. + autonerves_fitsable = pytest.importorskip("autonerves.fitsable") + + if not hasattr(autonerves_fitsable, "SMALL_DATASETS_SHAPE_HEADER_KEY"): + pytest.skip("autonerves on this path predates the cap card") + + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + dataset_path = tmp_path / "dataset" + dataset_path.mkdir() + aa.output_to_fits( + values=np.ones((360, 2)), + file_path=str(dataset_path / "data.fits"), + overwrite=True, + ) + + assert _small_datasets_shape_on_disk(str(dataset_path)) == ( + SMALL_DATASETS_SHAPE_NATIVE + ) + assert should_simulate(str(dataset_path)) is False