From 3e2a54336609cf4eae2696b4b2617931369b409c Mon Sep 17 00:00:00 2001 From: Ilya Matiach Date: Thu, 6 Aug 2026 14:18:42 -0400 Subject: [PATCH] feat(agents): add initialization_parameters to evaluator YAML config Enables evaluators that require configuration (e.g. builtin.regex_match patterns) to be fully specified in the optimize YAML config. Add initialization_parameters field to EvaluatorRef (YAML only, not serialized per-evaluator in the API array) and wire it through to EvaluatorInitParamsMap in the OptimizeRequest top-level field. The C# API stores this in Cosmos; the Python optimizer reads it via cosmos_loader._resolve_evaluator_init_params_map. Usage: evaluators: - name: builtin.regex_match version: 4 initialization_parameters: patterns: - (?i)Answer:\s*GT --- .../internal/cmd/optimize_config.go | 21 ++++++- .../internal/cmd/optimize_config_test.go | 55 +++++++++++++++++++ .../internal/pkg/agents/opt_eval/yaml.go | 12 +++- .../internal/pkg/agents/opt_eval/yaml_test.go | 47 ++++++++++++++++ .../pkg/agents/optimize_api/models.go | 15 +++-- 5 files changed, 141 insertions(+), 9 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go index 72c8ddf8db5..420679fb62c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go @@ -139,6 +139,11 @@ func (c *OptimizeConfig) ToRequest() (*optimize_api.OptimizeRequest, []string, e }, } + // Collect per-evaluator initialization_parameters into EvaluatorInitParamsMap. + if evalInitParams := evaluatorInitParamsMap(c.Evaluators); len(evalInitParams) > 0 { + req.EvaluatorInitParamsMap = evalInitParams + } + // Map optimization_config from YAML to API format. if c.Options.OptimizationConfig != nil { req.Options.OptimizationConfig = c.Options.OptimizationConfig @@ -252,7 +257,21 @@ func evaluatorRefs(list opt_eval.EvaluatorList) []optimize_api.EvaluatorRef { return refs } -// mergeEvaluators appends add to base, skipping entries whose name already +// evaluatorInitParamsMap builds the EvaluatorInitParamsMap from evaluators +// that have initialization_parameters set. Returns nil when no evaluator +// has initialization_parameters. +func evaluatorInitParamsMap(list opt_eval.EvaluatorList) map[string]map[string]any { + var result map[string]map[string]any + for _, e := range list { + if len(e.InitializationParameters) > 0 { + if result == nil { + result = make(map[string]map[string]any) + } + result[e.Name] = e.InitializationParameters + } + } + return result +} // exists in base (case-sensitive). Order is preserved: base first, then any // new entries from add. Used to layer --evaluator flags on top of config // evaluators without dropping the config entries. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config_test.go index aa187ae9544..a91b5309f33 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config_test.go @@ -915,3 +915,58 @@ func TestToRequest_MaxStalls(t *testing.T) { require.NotNil(t, req.Options.MaxStalls) assert.Equal(t, 3, *req.Options.MaxStalls) } + +func TestToRequest_EvaluatorInitParamsForwardedToAPI(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + cfg := &OptimizeConfig{ + Config: opt_eval.Config{ + Agent: opt_eval.AgentRef{Name: "agent"}, + Evaluators: opt_eval.EvaluatorList{ + { + Name: "builtin.regex_match", + Version: "4", + InitializationParameters: map[string]any{ + "patterns": []any{"(?i)Answer:\\s*{{ground_truth}}"}, + }, + }, + }, + DatasetFile: writeTestFile(t, dir, "ds.jsonl", `{"query":"hi","ground_truth":"yes"}`), + }, + Options: &opt_eval.Options{ + EvalModel: "gpt-4o-mini", + OptimizationModel: "gpt-5", + }, + } + + req, _, err := cfg.ToRequest() + require.NoError(t, err) + require.NotNil(t, req.EvaluatorInitParamsMap) + params, ok := req.EvaluatorInitParamsMap["builtin.regex_match"] + require.True(t, ok, "expected entry for builtin.regex_match") + patterns, ok := params["patterns"] + require.True(t, ok, "expected patterns key") + assert.NotEmpty(t, patterns) +} + +func TestToRequest_EvaluatorInitParamsNilWhenAbsent(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + cfg := &OptimizeConfig{ + Config: opt_eval.Config{ + Agent: opt_eval.AgentRef{Name: "agent"}, + Evaluators: opt_eval.EvaluatorList{{Name: "builtin.task_adherence"}}, + DatasetFile: writeTestFile(t, dir, "ds.jsonl", `{"query":"hi"}`), + }, + Options: &opt_eval.Options{ + EvalModel: "gpt-4o-mini", + OptimizationModel: "gpt-5", + }, + } + + req, _, err := cfg.ToRequest() + require.NoError(t, err) + assert.Nil(t, req.EvaluatorInitParamsMap) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go index 5fbb6be3a26..ed2d36f9153 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go @@ -90,10 +90,16 @@ func (c *Config) RemoteDatasetReference() *DatasetRef { // EvaluatorRef describes an evaluator. It can be a simple string name or a // structured entry with name, version, and local_uri. +// EvaluatorRef describes an evaluator. It can be a simple string name or a +// structured entry with name, version, local_uri, and initialization_parameters. type EvaluatorRef struct { - Name string `yaml:"name" json:"name"` - Version string `yaml:"version,omitempty" json:"version,omitempty"` - LocalURI string `yaml:"local_uri,omitempty" json:"local_uri,omitempty"` + Name string `yaml:"name" json:"name"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + LocalURI string `yaml:"local_uri,omitempty" json:"local_uri,omitempty"` + // InitializationParameters holds evaluator-specific configuration parameters + // such as regex_match's ``patterns``. Collected into EvaluatorInitParamsMap + // in the API request; not serialized per-evaluator in the evaluators array. + InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"-"` } // EvaluatorList is a list of evaluators that supports mixed YAML: diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml_test.go index d91e2ace357..2e06695ad5f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml_test.go @@ -415,3 +415,50 @@ optimization_config: // model should be the JSON string, not double-quoted. assert.JSONEq(t, `"gpt-4o"`, string(opts.OptimizationConfig["model"])) } + +// TestOptions_EvaluatorInitializationParameters verifies initialization_parameters +// under an evaluator entry is parsed correctly. +func TestOptions_EvaluatorInitializationParameters(t *testing.T) { + t.Parallel() + + input := ` +name: test +agent: + name: my-agent +evaluators: + - name: builtin.regex_match + version: "4" + initialization_parameters: + patterns: + - "(?i)Answer:\\s*{{ground_truth}}" +` + var cfg Config + require.NoError(t, yaml.Unmarshal([]byte(input), &cfg)) + + require.Len(t, cfg.Evaluators, 1) + ref := cfg.Evaluators[0] + assert.Equal(t, "builtin.regex_match", ref.Name) + require.NotNil(t, ref.InitializationParameters) + patterns, ok := ref.InitializationParameters["patterns"] + require.True(t, ok) + assert.NotEmpty(t, patterns) +} + +// TestOptions_EvaluatorInitializationParametersOmitted verifies +// InitializationParameters is nil when absent. +func TestOptions_EvaluatorInitializationParametersOmitted(t *testing.T) { + t.Parallel() + + input := ` +name: test +agent: + name: my-agent +evaluators: + - name: builtin.task_adherence +` + var cfg Config + require.NoError(t, yaml.Unmarshal([]byte(input), &cfg)) + + require.Len(t, cfg.Evaluators, 1) + assert.Nil(t, cfg.Evaluators[0].InitializationParameters) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/optimize_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/optimize_api/models.go index 528846c424d..bb9b64bdf7f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/optimize_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/optimize_api/models.go @@ -48,11 +48,16 @@ func IsTerminal(status string) bool { // OptimizeRequest is the top-level payload sent to POST /optimize. type OptimizeRequest struct { - Agent AgentIdentifier `json:"agent"` - TrainDataset *Dataset `json:"train_dataset,omitempty"` - ValidationDataset *Dataset `json:"validation_dataset,omitempty"` - Evaluators []EvaluatorRef `json:"evaluators,omitempty"` - Options OptimizeOptions `json:"options"` + Agent AgentIdentifier `json:"agent"` + TrainDataset *Dataset `json:"train_dataset,omitempty"` + ValidationDataset *Dataset `json:"validation_dataset,omitempty"` + Evaluators []EvaluatorRef `json:"evaluators,omitempty"` + Options OptimizeOptions `json:"options"` + // EvaluatorInitParamsMap maps evaluator name to its initialization_parameters. + // The C# API stores this in Cosmos; the Python optimizer reads it via + // cosmos_loader._resolve_evaluator_init_params_map to supply required params + // (e.g. regex_match's ``patterns``) when building testing criteria. + EvaluatorInitParamsMap map[string]map[string]any `json:"evaluatorInitParamsMap,omitempty"` } // AgentIdentifier references the agent to optimize by name and optional version.