Preserve yaml key order, render tabular file formats

packages/python/openproblems/CHANGELOG.md

@@ -37,6 +37,14 @@
   Viash does. `os.path.join()` treats such a path as absolute and silently dropped the project
   root, so a config containing e.g. `__merge__: /src/api/file_dataset.yaml` failed to read.
 
+* `deep_merge`: Preserve the order of the keys. Since the merged keys were collected in a `set()`
+  and Python randomises string hashing per process, `read_nested_yaml` returned its keys in a
+  different order on every run, and a rendered task README differed from one run to the next.
+
+* `render_file_format`: Render file formats of type `tabular`. `read_file_format` accepts them,
+  but the renderer only knew about `csv`, `tsv` and `parquet`, so the Format and Data structure
+  sections came out empty.
+
 # openproblems core Python v0.1.1
 
 ## NEW FUNCTIONALITY

packages/python/openproblems/src/openproblems/project/docs/render_file_format.py

@@ -100,7 +100,7 @@ def _render_format_example(spec: dict) -> list[str]:
                 lines.append(f"     {struct_name}: {', '.join(structs[struct_name])}")
         return lines
 
-    if fmt_type in ("csv", "tsv", "parquet"):
+    if fmt_type in ("tabular", "csv", "tsv", "parquet"):
         names = ", ".join(f"'{row['name']}'" for row in expected_format)
         return ["    Tabular data", f"     {names}"]
 
@@ -155,7 +155,7 @@ def _clean_desc(row: dict) -> str:
             )
         ]
 
-    if fmt_type in ("csv", "tsv", "parquet"):
+    if fmt_type in ("tabular", "csv", "tsv", "parquet"):
         rows = [
             [
                 f'`{row["name"]}`',

packages/python/openproblems/src/openproblems/utils/deep_merge.py

@@ -5,11 +5,13 @@ def deep_merge(obj1: any, obj2: any) -> dict:
         obj1 (any): The first dictionary or list.
         obj2 (any): The second dictionary or list.
 
+    Keys keep the order of `obj1`, followed by the keys only found in `obj2`.
+
     Returns:
         dict: The merged dictionary.
     """
     if isinstance(obj1, dict) and isinstance(obj2, dict):
-        keys = set(list(obj1.keys()) + list(obj2.keys()))
+        keys = list(obj1.keys()) + [k for k in obj2 if k not in obj1]
         out = {}
         for key in keys:
             if key in obj1:

packages/python/openproblems/tests/test_docs_render_file_format.py

@@ -0,0 +1,31 @@
+import pytest
+from openproblems.project.docs import render_file_format
+
+COLUMNS = [
+    {"name": "cell_id", "type": "string", "required": True, "description": "Cell id"},
+    {"name": "score", "type": "double", "required": False, "description": "The score"},
+]
+
+
+def _spec(file_type):
+    return {
+        "info": {
+            "file_name": "file_scores",
+            "file_type": file_type,
+            "label": "Scores",
+            "summary": "A table of scores.",
+        },
+        "expected_format": COLUMNS,
+    }
+
+
+@pytest.mark.parametrize("file_type", ["tabular", "csv", "tsv", "parquet"])
+def test_render_file_format_tabular(file_type):
+    result = render_file_format(_spec(file_type))
+
+    assert "## File format: Scores" in result
+    assert "Tabular data" in result
+    assert "'cell_id', 'score'" in result
+    assert "| Column | Type | Description |" in result
+    assert "`cell_id`" in result
+    assert "(_Optional_)" in result

packages/python/openproblems/tests/test_project_read_nested_yaml.py

@@ -0,0 +1,34 @@
+import os
+import yaml
+from openproblems.project import read_nested_yaml
+
+EXAMPLE_PROJECT = os.path.normpath(
+    os.path.join(
+        os.path.dirname(__file__),
+        "data/example_project",
+    )
+)
+
+
+def test_read_nested_yaml_preserves_key_order():
+    # the rendered README lists author info in the order the keys appear in
+    # the yaml, so the merge must not reshuffle them
+    path = os.path.join(EXAMPLE_PROJECT, "_viash.yaml")
+    with open(path, "r") as f:
+        raw = yaml.safe_load(f)
+
+    conf = read_nested_yaml(path)
+
+    assert list(conf.keys()) == list(raw.keys())
+    for i, author in enumerate(conf["authors"]):
+        assert list(author["info"].keys()) == list(raw["authors"][i]["info"].keys())
+
+
+def test_read_nested_yaml_resolves_merges():
+    path = os.path.join(EXAMPLE_PROJECT, "api", "comp_method.yaml")
+    conf = read_nested_yaml(path)
+
+    train_arg = next(arg for arg in conf["arguments"] if arg["name"] == "--input_train")
+    # pulled in from file_train.yaml
+    assert train_arg["type"] == "file"
+    assert train_arg["label"] == "Training data"

packages/python/openproblems/tests/test_utils_deep_merge.py

@@ -0,0 +1,20 @@
+from openproblems.utils import deep_merge
+
+
+def test_deep_merge_overrides_and_adds():
+    out = deep_merge({"a": 1, "b": 2}, {"b": 3, "c": 4})
+    assert out == {"a": 1, "b": 3, "c": 4}
+
+
+def test_deep_merge_is_recursive():
+    out = deep_merge({"a": {"b": 1, "c": 2}}, {"a": {"c": 3}})
+    assert out == {"a": {"b": 1, "c": 3}}
+
+
+def test_deep_merge_appends_lists():
+    assert deep_merge([1, 2], [3]) == [1, 2, 3]
+
+
+def test_deep_merge_preserves_key_order():
+    out = deep_merge({"b": 1, "a": 2}, {"d": 3, "c": 4, "a": 5})
+    assert list(out.keys()) == ["b", "a", "d", "c"]