From a3512d7d8f73d9499c12fd7e8f5f4db873f01d59 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 1 Oct 2026 13:57:48 +0200 Subject: [PATCH 1/2] Add rumbleConfig(), support generators, optionally display jar path. --- README.md | 11 +++++++++-- src/jsoniq/session.py | 27 +++++++++++++++++++++++---- tests/test_bundled_spark.py | 14 ++++++++++++++ tests/test_rumbledb_3_api.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dddf19f..17c99a8 100644 --- a/README.md +++ b/README.md @@ -111,13 +111,18 @@ import pandas as pd # A RumbleSession is a SparkSession that additionally knows about RumbleDB. # SparkSession methods such as createDataFrame() and sql() are also available on RumbleSession. -rumble = RumbleSession.builder.getOrCreate() +# rumbleConfig() applies Rumble settings before startup messages, or to a reused session. +# config() remains available for Spark settings. +rumble = ( + RumbleSession.builder + .rumbleConfig("debug.showErrorInfo", True) + .getOrCreate() +) # Configure subsequent queries using RumbleDB 3.0's dot-separated paths. # The result size cap controls first() and notebook display; json() uses the materialization cap. rumble.getRumbleConf().set("runtime.resultsSizeCap", 100) rumble.getRumbleConf().set("runtime.materializationCap", 100000) -rumble.getRumbleConf().set("debug.showErrorInfo", True) print(rumble.getRumbleConf().getInt("runtime.resultsSizeCap")) # Just to improve readability when invoking Spark methods @@ -203,6 +208,8 @@ print(seq.json()) # and then use it in a query. # In JSONiq, variables and query results are sequences of items. # A Python tuple will be seamlessly converted to a sequence of items by the library. +# Python generators are consumed into tuples and bound as sequences of items too. +# Generators must be finite; their contents are materialized in memory when bound. # Scalars can be strings, ints, floats, booleans, None, or dicts. # Lists represent array items: wrap a list in a singleton tuple to bind one array. # Keyword bindings apply only to this query; they do not persist on the session. diff --git a/src/jsoniq/session.py b/src/jsoniq/session.py index 36c7a5c..bacc8a3 100644 --- a/src/jsoniq/session.py +++ b/src/jsoniq/session.py @@ -7,6 +7,7 @@ import os import re from threading import RLock +from types import GeneratorType import pandas as pd from importlib.resources import files, as_file @@ -17,7 +18,6 @@ jar_path_str = str(jar_path) else: jar_path_str = "file://" + str(jar_path) - print(f"[Info] Using RumbleDB jar file at: {jar_path_str}") def get_spark_version(): if os.environ.get('SPARK_HOME') != None: @@ -36,11 +36,15 @@ def __getattr__(cls, item): return getattr(SparkSession, item) class RumbleSession(object, metaclass=MetaRumbleSession): - def __init__(self, spark_session: SparkSession): + def __init__(self, spark_session: SparkSession, rumble_options=None): self._sparksession = spark_session self._jrumblesession = spark_session._jvm.org.rumbledb.api.Rumble(spark_session._jsparkSession) self._configuration = RumbleConfiguration(self) self._bindings = {} + for path, value in (rumble_options or {}).items(): + self._configuration.set(path, value) + if self._configuration.getBoolean("debug.showErrorInfo"): + print(f"[Info] Using RumbleDB jar file at: {jar_path_str}") def getRumbleConf(self): return self._configuration @@ -82,11 +86,21 @@ def __init__(self): sys.exit(43) self._sparkbuilder = SparkSession.builder.config("spark.jars", jar_path_str) self._use_bundled_spark = True + self._rumble_options = {} self._appendable_keys = { "spark.jars.packages", "spark.sql.extensions", } + def rumbleConfig(self, path, value): + """Set a Rumble configuration path for new or reused sessions. + + Settings are retained by this builder; repeated paths use the last value. + Use config() for Spark settings. + """ + self._rumble_options[path] = value + return self + def withBundledSpark(self, enabled=True): """Use PySpark's bundled Spark by default; pass False to respect SPARK_HOME. @@ -99,10 +113,10 @@ def _create_session(self, method): # Serialize this library's startup calls while changing the process environment. with _spark_creation_lock: if not self._use_bundled_spark: - return RumbleSession(getattr(self._sparkbuilder, method)()) + return RumbleSession(getattr(self._sparkbuilder, method)(), self._rumble_options) previous_spark_home = os.environ.pop("SPARK_HOME", None) try: - return RumbleSession(getattr(self._sparkbuilder, method)()) + return RumbleSession(getattr(self._sparkbuilder, method)(), self._rumble_options) finally: if previous_spark_home is None: os.environ.pop("SPARK_HOME", None) @@ -136,6 +150,9 @@ def getOrCreate(self): sys.stderr.write(f"We would appreciate a bug report with some information about your OS, setup, etc.\n") sys.stderr.write(f"In the meantime, what you could do as a workaround is download the Spark 4.0.0 zip file from spark.apache.org, unzip it to some local directory, and point SPARK_HOME to this directory.\n") raise e + else: + for path, value in self._rumble_options.items(): + RumbleSession._rumbleSession.getRumbleConf().set(path, value) return RumbleSession._rumbleSession def create(self): @@ -259,6 +276,8 @@ def bind(self, name: str, valueToBind): if not name.startswith("$"): raise ValueError("Variable name must start with a dollar symbol ('$').") name = name[1:] + if isinstance(valueToBind, GeneratorType): + valueToBind = tuple(valueToBind) if isinstance(valueToBind, SequenceOfItems): outputs = valueToBind.availableOutputs() if isinstance(outputs, (list, JavaList)) and "DataFrame" in outputs: diff --git a/tests/test_bundled_spark.py b/tests/test_bundled_spark.py index 951c372..958b2a6 100644 --- a/tests/test_bundled_spark.py +++ b/tests/test_bundled_spark.py @@ -99,6 +99,20 @@ def start(): assert os.environ["SPARK_HOME"] == "/external/spark" +def test_rumble_config_applies_to_reused_session(builder, monkeypatch, capsys): + existing = Mock() + monkeypatch.setattr(RumbleSession, "_rumbleSession", existing) + assert builder.rumbleConfig("debug.showErrorInfo", False) is builder + builder.rumbleConfig("debug.showErrorInfo", True) + builder.rumbleConfig("runtime.resultsSizeCap", 2) + assert builder.getOrCreate() is existing + existing.getRumbleConf.return_value.set.assert_any_call("debug.showErrorInfo", True) + existing.getRumbleConf.return_value.set.assert_any_call("runtime.resultsSizeCap", 2) + assert existing.getRumbleConf.return_value.set.call_count == 2 + builder._sparkbuilder.getOrCreate.assert_not_called() + assert "Using RumbleDB jar file" not in capsys.readouterr().out + + def test_bundled_spark_starts_with_invalid_external_home(monkeypatch, tmp_path): monkeypatch.setattr(RumbleSession, "_rumbleSession", None) invalid_home = str(tmp_path / "missing-spark") diff --git a/tests/test_rumbledb_3_api.py b/tests/test_rumbledb_3_api.py index 9966988..7041d94 100644 --- a/tests/test_rumbledb_3_api.py +++ b/tests/test_rumbledb_3_api.py @@ -20,6 +20,28 @@ def rumble(): RumbleSession._rumbleSession = None +@pytest.mark.parametrize("bundled_spark", [False, True]) +@pytest.mark.parametrize("show_error_info", [False, True]) +def test_builder_rumble_config_before_startup(rumble, monkeypatch, capsys, bundled_spark, show_error_info): + monkeypatch.setattr(RumbleSession, "_rumbleSession", None) + builder = ( + RumbleSession.Builder() + .withBundledSpark(bundled_spark) + .rumbleConfig("debug.showErrorInfo", show_error_info) + .rumbleConfig("runtime.resultsSizeCap", 2) + ) + capsys.readouterr() + session = builder.getOrCreate() + assert ("Using RumbleDB jar file" in capsys.readouterr().out) == show_error_info + assert session.getRumbleConf().getBoolean("debug.showErrorInfo") is show_error_info + assert session.getRumbleConf().getInt("runtime.resultsSizeCap") == 2 + assert len(session.jsoniq("1 to 3").first()) == 2 + builder.rumbleConfig("debug.showErrorInfo", not show_error_info) + assert builder.getOrCreate() is session + assert session.getRumbleConf().getBoolean("debug.showErrorInfo") is not show_error_info + assert "Using RumbleDB jar file" not in capsys.readouterr().out + + def test_configuration_updates_preserve_session_and_other_settings(rumble): conf = rumble.getRumbleConf() spark = rumble._sparksession._jsparkSession @@ -61,6 +83,18 @@ def test_python_value_bindings(rumble, value): rumble.jsoniq("$value") +@pytest.mark.parametrize("values", [(), ({"foo": [42]},) * 10, (1, "two", [3], None)]) +def test_generator_bindings(rumble, values): + generated = (value for value in values) + rumble.bind("$generated", generated) + try: + assert rumble.jsoniq("$generated").json() == values + assert tuple(generated) == () + assert rumble.jsoniq("$temporary", temporary=(value for value in values)).json() == values + finally: + rumble.unbind("$generated") + + def test_keyword_bindings_restore_persistent_values_even_on_failure(rumble): rumble.bind("$value", 7) try: From a74b222f29bd907809c49d55c6726971b4da9774 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 1 Oct 2026 13:58:28 +0200 Subject: [PATCH 2/2] Bump version. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 14aa822..ed9189e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jsoniq" -version = "3.0.1" +version = "3.0.2" description = "Python edition of RumbleDB, a JSONiq engine" requires-python = ">=3.11" dependencies = [