From 1817f0689ab17b10bf9e2cf58c74d211fce892d0 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 11:38:29 +0200 Subject: [PATCH 1/3] preserve key order when deep merging `set()` has no order, and python randomises string hashing per process, so every yaml read through `read_nested_yaml()` came back with its keys in a different order on every run. The author info columns in a rendered task README are built by iterating those keys, so each run produced a different README. * `deep_merge()`: keep the keys of the first object, then append the ones only found in the second, like the R implementation does * Add tests for `deep_merge()` and for the key order of `read_nested_yaml()` --- .../src/openproblems/utils/deep_merge.py | 4 ++- .../tests/test_project_read_nested_yaml.py | 34 +++++++++++++++++++ .../tests/test_utils_deep_merge.py | 20 +++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 packages/python/openproblems/tests/test_project_read_nested_yaml.py create mode 100644 packages/python/openproblems/tests/test_utils_deep_merge.py diff --git a/packages/python/openproblems/src/openproblems/utils/deep_merge.py b/packages/python/openproblems/src/openproblems/utils/deep_merge.py index 461d4b3..56ecd53 100644 --- a/packages/python/openproblems/src/openproblems/utils/deep_merge.py +++ b/packages/python/openproblems/src/openproblems/utils/deep_merge.py @@ -5,11 +5,13 @@ def deep_merge(obj1: any, obj2: any) -> dict: obj1 (any): The first dictionary or list. obj2 (any): The second dictionary or list. + Keys keep the order of `obj1`, followed by the keys only found in `obj2`. + Returns: dict: The merged dictionary. """ if isinstance(obj1, dict) and isinstance(obj2, dict): - keys = set(list(obj1.keys()) + list(obj2.keys())) + keys = list(obj1.keys()) + [k for k in obj2 if k not in obj1] out = {} for key in keys: if key in obj1: diff --git a/packages/python/openproblems/tests/test_project_read_nested_yaml.py b/packages/python/openproblems/tests/test_project_read_nested_yaml.py new file mode 100644 index 0000000..d12f885 --- /dev/null +++ b/packages/python/openproblems/tests/test_project_read_nested_yaml.py @@ -0,0 +1,34 @@ +import os +import yaml +from openproblems.project import read_nested_yaml + +EXAMPLE_PROJECT = os.path.normpath( + os.path.join( + os.path.dirname(__file__), + "data/example_project", + ) +) + + +def test_read_nested_yaml_preserves_key_order(): + # the rendered README lists author info in the order the keys appear in + # the yaml, so the merge must not reshuffle them + path = os.path.join(EXAMPLE_PROJECT, "_viash.yaml") + with open(path, "r") as f: + raw = yaml.safe_load(f) + + conf = read_nested_yaml(path) + + assert list(conf.keys()) == list(raw.keys()) + for i, author in enumerate(conf["authors"]): + assert list(author["info"].keys()) == list(raw["authors"][i]["info"].keys()) + + +def test_read_nested_yaml_resolves_merges(): + path = os.path.join(EXAMPLE_PROJECT, "api", "comp_method.yaml") + conf = read_nested_yaml(path) + + train_arg = next(arg for arg in conf["arguments"] if arg["name"] == "--input_train") + # pulled in from file_train.yaml + assert train_arg["type"] == "file" + assert train_arg["label"] == "Training data" diff --git a/packages/python/openproblems/tests/test_utils_deep_merge.py b/packages/python/openproblems/tests/test_utils_deep_merge.py new file mode 100644 index 0000000..3ef61be --- /dev/null +++ b/packages/python/openproblems/tests/test_utils_deep_merge.py @@ -0,0 +1,20 @@ +from openproblems.utils import deep_merge + + +def test_deep_merge_overrides_and_adds(): + out = deep_merge({"a": 1, "b": 2}, {"b": 3, "c": 4}) + assert out == {"a": 1, "b": 3, "c": 4} + + +def test_deep_merge_is_recursive(): + out = deep_merge({"a": {"b": 1, "c": 2}}, {"a": {"c": 3}}) + assert out == {"a": {"b": 1, "c": 3}} + + +def test_deep_merge_appends_lists(): + assert deep_merge([1, 2], [3]) == [1, 2, 3] + + +def test_deep_merge_preserves_key_order(): + out = deep_merge({"b": 1, "a": 2}, {"d": 3, "c": 4, "a": 5}) + assert list(out.keys()) == ["b", "a", "d", "c"] From 477cc5d22189dae52c8ba35cdacf3c5a18cd8c74 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 11:38:29 +0200 Subject: [PATCH 2/3] render tabular file formats `read_file_format()` accepts `tabular` as a format type but the renderer only knew about `csv`, `tsv` and `parquet`, so such a spec rendered an empty Format and Data structure section. --- .../project/docs/render_file_format.py | 4 +-- .../tests/test_docs_render_file_format.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 packages/python/openproblems/tests/test_docs_render_file_format.py diff --git a/packages/python/openproblems/src/openproblems/project/docs/render_file_format.py b/packages/python/openproblems/src/openproblems/project/docs/render_file_format.py index 7049233..cc5fd69 100644 --- a/packages/python/openproblems/src/openproblems/project/docs/render_file_format.py +++ b/packages/python/openproblems/src/openproblems/project/docs/render_file_format.py @@ -100,7 +100,7 @@ def _render_format_example(spec: dict) -> list[str]: lines.append(f" {struct_name}: {', '.join(structs[struct_name])}") return lines - if fmt_type in ("csv", "tsv", "parquet"): + if fmt_type in ("tabular", "csv", "tsv", "parquet"): names = ", ".join(f"'{row['name']}'" for row in expected_format) return [" Tabular data", f" {names}"] @@ -155,7 +155,7 @@ def _clean_desc(row: dict) -> str: ) ] - if fmt_type in ("csv", "tsv", "parquet"): + if fmt_type in ("tabular", "csv", "tsv", "parquet"): rows = [ [ f'`{row["name"]}`', diff --git a/packages/python/openproblems/tests/test_docs_render_file_format.py b/packages/python/openproblems/tests/test_docs_render_file_format.py new file mode 100644 index 0000000..2d510b4 --- /dev/null +++ b/packages/python/openproblems/tests/test_docs_render_file_format.py @@ -0,0 +1,31 @@ +import pytest +from openproblems.project.docs import render_file_format + +COLUMNS = [ + {"name": "cell_id", "type": "string", "required": True, "description": "Cell id"}, + {"name": "score", "type": "double", "required": False, "description": "The score"}, +] + + +def _spec(file_type): + return { + "info": { + "file_name": "file_scores", + "file_type": file_type, + "label": "Scores", + "summary": "A table of scores.", + }, + "expected_format": COLUMNS, + } + + +@pytest.mark.parametrize("file_type", ["tabular", "csv", "tsv", "parquet"]) +def test_render_file_format_tabular(file_type): + result = render_file_format(_spec(file_type)) + + assert "## File format: Scores" in result + assert "Tabular data" in result + assert "'cell_id', 'score'" in result + assert "| Column | Type | Description |" in result + assert "`cell_id`" in result + assert "(_Optional_)" in result From eefa0ce42cbacad89d72d6e2c868246e1bda8ffd Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 11:38:30 +0200 Subject: [PATCH 3/3] update changelog --- packages/python/openproblems/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/python/openproblems/CHANGELOG.md b/packages/python/openproblems/CHANGELOG.md index 4354b69..d771f40 100644 --- a/packages/python/openproblems/CHANGELOG.md +++ b/packages/python/openproblems/CHANGELOG.md @@ -37,6 +37,14 @@ Viash does. `os.path.join()` treats such a path as absolute and silently dropped the project root, so a config containing e.g. `__merge__: /src/api/file_dataset.yaml` failed to read. +* `deep_merge`: Preserve the order of the keys. Since the merged keys were collected in a `set()` + and Python randomises string hashing per process, `read_nested_yaml` returned its keys in a + different order on every run, and a rendered task README differed from one run to the next. + +* `render_file_format`: Render file formats of type `tabular`. `read_file_format` accepts them, + but the renderer only knew about `csv`, `tsv` and `parquet`, so the Format and Data structure + sections came out empty. + # openproblems core Python v0.1.1 ## NEW FUNCTIONALITY