PyDABs: deterministic unit-test generation [WIP]

python/Taskfile.yml

@@ -78,6 +78,8 @@ tasks:
           -exec rm -rf {} \;
       # core/ is hand-written except for the generated wiring under _generated/.
       - rm -rf databricks/bundles/core/_generated
+      # test_resources.py is hand-written except for the generated TestCase data.
+      - rm -rf databricks_tests/core/_generated
       - cd codegen && uv run -m pytest codegen_tests
       - cd codegen && uv run -m codegen.main --output ..
       # Generated code is fixed and formatted by the global ruff (see ../ruff.toml).

python/codegen/codegen/generated_test_cases.py

@@ -0,0 +1,335 @@
+"""
+Generates the per-resource TestCase data driving databricks_tests/core/test_resources.py.
+
+For every wired resource a file _generated/<plural>.py is written (rendered from
+test_case.py.tmpl) exposing _test_case() -> (TestCase, _ResourceType). The generated
+_generated/__init__.py collects them into `test_cases`, which test_resources.py imports
+and parametrizes its per-resource tests off.
+
+dict_example and dataclass_example are synthesized from one value tree and rendered two
+independent ways -- a dict literal and a constructor expression -- so the dict->dataclass
+_transform assertion in test_resources.py stays meaningful (the two forms don't share the
+runtime transform path).
+
+Field policy: all required fields (fully expanded), plus optional composite fields
+(nested dataclass / list / map / enum) on the resource itself; nested objects contribute
+only their required fields, which keeps examples bounded and avoids recursive schemas
+(e.g. jobs Task -> ForEachTask -> Task, reachable only through an optional field). Optional
+scalar, deprecated, and experimental fields are omitted.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from string import Template
+from typing import Union
+
+import codegen.jsonschema as openapi
+import codegen.packages as packages
+from codegen.generated_enum import _camel_to_upper_snake
+from codegen.generated_wiring import _wired_resources, _WiredResource
+
+HEADER = "# Code generated by pydabs-codegen. DO NOT EDIT.\n\n"
+
+_TEST_CASE_TEMPLATE = Template(
+    (Path(__file__).parent / "test_case.py.tmpl").read_text()
+)
+
+
+# Synthesized value tree. Each node renders both as a dict literal (dict_example)
+# and as a constructor expression (dataclass_example).
+
+
+@dataclass
+class _Scalar:
+    dict_src: str
+    dataclass_src: str
+
+
+@dataclass
+class _Enum:
+    value: str
+    class_name: str
+    module: str
+    member: str
+
+
+@dataclass
+class _Object:
+    class_name: str
+    module: str
+    fields: "list[tuple[str, _Value]]"
+
+
+@dataclass
+class _List:
+    item: "_Value"
+
+
+@dataclass
+class _Map:
+    key: str
+    value: "_Value"
+
+
+_Value = Union[_Scalar, _Enum, _Object, _List, _Map]
+
+
+def _ref_name(ref: str) -> str:
+    """Last path segment of a JSON-schema ref -- the schema name.
+
+    :param ref: a JSON-schema reference, e.g. "#/$defs/.../jobs.Task" or "#/$defs/string".
+    """
+    return ref.split("/")[-1]
+
+
+def _is_composite(ref: str) -> bool:
+    """Whether a ref is a composite type (list, map, object, or enum) rather than a scalar.
+
+    :param ref: the JSON-schema reference of a field's type.
+    """
+    if ref.startswith(("#/$defs/slice/", "#/$defs/map/")):
+        return True
+
+    return _ref_name(ref) not in packages.PRIMITIVES
+
+
+def _synth_scalar(name: str, hint: str) -> _Scalar:
+    """Placeholder value for a primitive (str -> hint, int -> 0, float -> 0.0, bool -> True).
+
+    :param name: the primitive's schema name, e.g. "string", "int", "boolean".
+    :param hint: enclosing field name, used as the string placeholder so examples read meaningfully.
+    """
+    if name == "string":
+        return _Scalar(f'"{hint}"', f'"{hint}"')
+    if name in ("integer", "int", "int64"):
+        return _Scalar("0", "0")
+    if name in ("number", "float", "float64"):
+        return _Scalar("0.0", "0.0")
+    if name in ("boolean", "bool"):
+        return _Scalar("True", "True")
+
+    raise ValueError(f"Unknown primitive: {name}")
+
+
+def _synth_ref(
+    namespace: str,
+    ref: str,
+    hint: str,
+    schemas: dict[str, openapi.Schema],
+    visiting: set[str],
+) -> _Value:
+    """Synthesize a value node for whatever type a ref points at: list, map, scalar, enum, or nested object.
+
+    :param namespace: the resource's namespace (e.g. "jobs"); selects the module a referenced type is generated into.
+    :param ref: the JSON-schema reference of the type to synthesize.
+    :param hint: enclosing field name, passed through as the string placeholder.
+    :param schemas: all post-patch schemas keyed by schema name, for looking up nested/enum types.
+    :param visiting: ancestor object names on the current path, used to detect required cycles.
+    """
+    if ref.startswith("#/$defs/slice/"):
+        element_ref = ref.replace("#/$defs/slice/", "#/$defs/")
+
+        return _List(_synth_ref(namespace, element_ref, hint, schemas, visiting))
+
+    if ref.startswith("#/$defs/map/"):
+        # generate_type only ever produces dict[str, str] maps (map/string).
+        if ref != "#/$defs/map/string":
+            raise ValueError(f"Unsupported map ref: {ref}")
+
+        return _Map("key", _Scalar('"value"', '"value"'))
+
+    name = _ref_name(ref)
+    if name in packages.PRIMITIVES:
+        return _synth_scalar(name, hint)
+
+    schema = schemas[name]
+    class_name = packages.get_class_name(ref)
+    module = packages.get_package(namespace, ref)
+    assert module
+
+    if schema.type == openapi.SchemaType.STRING:
+        value = schema.enum[0]
+
+        return _Enum(value, class_name, module, _camel_to_upper_snake(value))
+
+    # Only reachable through required fields at this depth (see _synth_object); a
+    # required cycle has no finite value, so fail loudly instead of looping.
+    if name in visiting:
+        raise ValueError(f"Required-field cycle through '{name}'")
+
+    return _synth_object(namespace, name, schema, schemas, visiting, top_level=False)
+
+
+def _synth_object(
+    namespace: str,
+    schema_name: str,
+    schema: openapi.Schema,
+    schemas: dict[str, openapi.Schema],
+    visiting: set[str],
+    top_level: bool,
+) -> _Object:
+    """Synthesize an object value, choosing fields by policy: all required fields, plus (only at the resource top level) stable optional composite fields.
+
+    :param namespace: the resource's namespace, threaded through to resolve nested types' modules.
+    :param schema_name: this object's schema name (e.g. "resources.Alert").
+    :param schema: the Schema for this object -- its properties and required list.
+    :param schemas: all post-patch schemas, for recursing into nested types.
+    :param visiting: ancestor object names on the current path (cycle guard).
+    :param top_level: True only for the resource itself; when False, all optional fields are dropped.
+    """
+    visiting = visiting | {schema_name}
+    fields: list[tuple[str, _Value]] = []
+
+    for field_name, prop in schema.properties.items():
+        required = field_name in schema.required
+
+        if not required:
+            # Nested objects contribute only required fields; on the resource
+            # itself, also include stable optional composite fields.
+            if not top_level:
+                continue
+            if not _is_composite(prop.ref):
+                continue
+            if prop.deprecated or prop.stage == openapi.LaunchStage.PRIVATE_PREVIEW:
+                continue
+
+        value = _synth_ref(namespace, prop.ref, field_name, schemas, visiting)
+        fields.append((field_name, value))
+
+    return _Object(
+        packages.get_class_name(schema_name), _module_of(namespace, schema_name), fields
+    )
+
+
+def _module_of(namespace: str, schema_name: str) -> str:
+    """Python module a (non-primitive) schema's generated class lives in; asserts it exists.
+
+    :param namespace: the resource's namespace; the type is generated under databricks.bundles.<namespace>._models.
+    :param schema_name: the object/enum schema name to resolve.
+    """
+    module = packages.get_package(namespace, schema_name)
+    assert module
+
+    return module
+
+
+def _render_dict(value: _Value) -> str:
+    """Render a synthesized value as a dict-literal source string (the dict_example form).
+
+    :param value: the synthesized value node to render.
+    """
+    if isinstance(value, _Scalar):
+        return value.dict_src
+    if isinstance(value, _Enum):
+        return f'"{value.value}"'
+    if isinstance(value, _List):
+        return f"[{_render_dict(value.item)}]"
+    if isinstance(value, _Map):
+        return "{" + f'"{value.key}": {_render_dict(value.value)}' + "}"
+
+    fields = ", ".join(
+        f'"{name}": {_render_dict(child)}' for name, child in value.fields
+    )
+
+    return "{" + fields + "}"
+
+
+def _render_dataclass(value: _Value) -> str:
+    """Render a synthesized value as a constructor-expression source string (the dataclass_example form).
+
+    :param value: the synthesized value node to render.
+    """
+    if isinstance(value, _Scalar):
+        return value.dataclass_src
+    if isinstance(value, _Enum):
+        return f"{value.class_name}.{value.member}"
+    if isinstance(value, _List):
+        return f"[{_render_dataclass(value.item)}]"
+    if isinstance(value, _Map):
+        return "{" + f'"{value.key}": {_render_dataclass(value.value)}' + "}"
+
+    fields = ", ".join(
+        f"{name}={_render_dataclass(child)}" for name, child in value.fields
+    )
+
+    return f"{value.class_name}({fields})"
+
+
+def _collect_imports(value: _Value, out: set[tuple[str, str]]) -> None:
+    """Collect (module, class_name) pairs the dataclass_example needs, walking nested objects/enums.
+
+    :param value: the synthesized value node to walk.
+    :param out: set accumulating the (module, class_name) import pairs; mutated in place.
+    """
+    if isinstance(value, _Enum):
+        out.add((value.module, value.class_name))
+    elif isinstance(value, _Object):
+        out.add((value.module, value.class_name))
+        for _, child in value.fields:
+            _collect_imports(child, out)
+    elif isinstance(value, _List):
+        _collect_imports(value.item, out)
+    elif isinstance(value, _Map):
+        _collect_imports(value.value, out)
+
+
+def write_test_cases(output: str, schemas: dict[str, openapi.Schema]):
+    """Write one _generated/<resource_plural>.py per wired resource plus the collector __init__.py.
+
+    :param output: codegen output root (the python/ directory); files land under databricks_tests/core/_generated.
+    :param schemas: all post-patch schemas, used to synthesize each resource's dict/dataclass examples.
+    """
+    resources = _wired_resources()
+
+    generated_path = Path(output) / "databricks_tests" / "core" / "_generated"
+    generated_path.mkdir(parents=True, exist_ok=True)
+
+    plural_to_ref = {ns: ref for ref, ns in packages.RESOURCE_NAMESPACE.items()}
+
+    for r in resources:
+        resource_ref = plural_to_ref[r.plural_name]
+        schema = schemas[resource_ref]
+
+        example = _synth_object(
+            r.plural_name, resource_ref, schema, schemas, set(), top_level=True
+        )
+
+        imports: set[tuple[str, str]] = set()
+        _collect_imports(example, imports)
+        model_imports = "\n".join(
+            f"from {module} import {class_name}"
+            for module, class_name in sorted(imports)
+        )
+
+        code = _TEST_CASE_TEMPLATE.substitute(
+            singular=r.singular_name,
+            plural=r.plural_name,
+            model_imports=model_imports,
+            dict_example=_render_dict(example),
+            dataclass_example=_render_dataclass(example),
+        )
+        (generated_path / f"{r.plural_name}.py").write_text(HEADER + code)
+
+    (generated_path / "__init__.py").write_text(HEADER + _collector_code(resources))
+
+    print(f"Writing test cases into {generated_path}")
+
+
+def _collector_code(resources: list[_WiredResource]) -> str:
+    """Source for _generated/__init__.py: imports the per-resource modules and assembles `test_cases`.
+
+    :param resources: the wired resources, in the order their test cases are collected.
+    """
+    module_imports = "\n".join(f"    {r.plural_name}," for r in resources)
+    entries = "\n".join(f"    {r.plural_name}._test_case()," for r in resources)
+
+    return f"""from databricks_tests.core._generated import (
+{module_imports}
+)
+
+__all__ = ["test_cases"]
+
+test_cases = [
+{entries}
+]
+"""

python/codegen/codegen/main.py

@@ -8,6 +8,7 @@
 import codegen.generated_dataclass_patch as generated_dataclass_patch
 import codegen.generated_enum as generated_enum
 import codegen.generated_imports as generated_imports
+import codegen.generated_test_cases as generated_test_cases
 import codegen.generated_wiring as generated_wiring
 import codegen.jsonschema as openapi
 import codegen.jsonschema_patch as openapi_patch
@@ -52,6 +53,9 @@ def main(output: str):
     # decorators, and the core package __init__).
     generated_wiring.write_wiring(output)
 
+    # Generate the per-resource TestCase data driving test_resources.py.
+    generated_test_cases.write_test_cases(output, schemas)
+
 
 def _transitively_mark_deprecated_and_private(
     roots: list[str],

python/codegen/codegen/test_case.py.tmpl

@@ -0,0 +1,16 @@
+from databricks.bundles.core import Resources, ${singular}_mutator
+from databricks.bundles.core._generated.${plural} import _resource_type
+from databricks_tests.core._resource_test_case import TestCase
+$model_imports
+
+
+def _test_case():
+    return (
+        TestCase(
+            add_resource=Resources.add_${singular},
+            dict_example=$dict_example,
+            dataclass_example=$dataclass_example,
+            mutator=${singular}_mutator,
+        ),
+        _resource_type(),
+    )

python/databricks_tests/.gitattributes

@@ -0,0 +1,4 @@
+# Generated by pydabs-codegen (see python/codegen). The per-resource TestCase
+# data under core/_generated/ drives the parametrized tests in test_resources.py;
+# the rest of databricks_tests/ is hand-written.
+core/_generated/** linguist-generated=true