feat(agents): add initialization_parameters to evaluator YAML config

cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config.go

@@ -139,6 +139,11 @@ func (c *OptimizeConfig) ToRequest() (*optimize_api.OptimizeRequest, []string, e
 		},
 	}
 
+	// Collect per-evaluator initialization_parameters into EvaluatorInitParamsMap.
+	if evalInitParams := evaluatorInitParamsMap(c.Evaluators); len(evalInitParams) > 0 {
+		req.EvaluatorInitParamsMap = evalInitParams
+	}
+
 	// Map optimization_config from YAML to API format.
 	if c.Options.OptimizationConfig != nil {
 		req.Options.OptimizationConfig = c.Options.OptimizationConfig
@@ -252,7 +257,21 @@ func evaluatorRefs(list opt_eval.EvaluatorList) []optimize_api.EvaluatorRef {
 	return refs
 }
 
-// mergeEvaluators appends add to base, skipping entries whose name already
+// evaluatorInitParamsMap builds the EvaluatorInitParamsMap from evaluators
+// that have initialization_parameters set. Returns nil when no evaluator
+// has initialization_parameters.
+func evaluatorInitParamsMap(list opt_eval.EvaluatorList) map[string]map[string]any {
+	var result map[string]map[string]any
+	for _, e := range list {
+		if len(e.InitializationParameters) > 0 {
+			if result == nil {
+				result = make(map[string]map[string]any)
+			}
+			result[e.Name] = e.InitializationParameters
+		}
+	}
+	return result
+}
 // exists in base (case-sensitive). Order is preserved: base first, then any
 // new entries from add. Used to layer --evaluator flags on top of config
 // evaluators without dropping the config entries.

cli/azd/extensions/azure.ai.agents/internal/cmd/optimize_config_test.go

@@ -915,3 +915,58 @@ func TestToRequest_MaxStalls(t *testing.T) {
 	require.NotNil(t, req.Options.MaxStalls)
 	assert.Equal(t, 3, *req.Options.MaxStalls)
 }
+
+func TestToRequest_EvaluatorInitParamsForwardedToAPI(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	cfg := &OptimizeConfig{
+		Config: opt_eval.Config{
+			Agent: opt_eval.AgentRef{Name: "agent"},
+			Evaluators: opt_eval.EvaluatorList{
+				{
+					Name:    "builtin.regex_match",
+					Version: "4",
+					InitializationParameters: map[string]any{
+						"patterns": []any{"(?i)Answer:\\s*{{ground_truth}}"},
+					},
+				},
+			},
+			DatasetFile: writeTestFile(t, dir, "ds.jsonl", `{"query":"hi","ground_truth":"yes"}`),
+		},
+		Options: &opt_eval.Options{
+			EvalModel:         "gpt-4o-mini",
+			OptimizationModel: "gpt-5",
+		},
+	}
+
+	req, _, err := cfg.ToRequest()
+	require.NoError(t, err)
+	require.NotNil(t, req.EvaluatorInitParamsMap)
+	params, ok := req.EvaluatorInitParamsMap["builtin.regex_match"]
+	require.True(t, ok, "expected entry for builtin.regex_match")
+	patterns, ok := params["patterns"]
+	require.True(t, ok, "expected patterns key")
+	assert.NotEmpty(t, patterns)
+}
+
+func TestToRequest_EvaluatorInitParamsNilWhenAbsent(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	cfg := &OptimizeConfig{
+		Config: opt_eval.Config{
+			Agent:       opt_eval.AgentRef{Name: "agent"},
+			Evaluators:  opt_eval.EvaluatorList{{Name: "builtin.task_adherence"}},
+			DatasetFile: writeTestFile(t, dir, "ds.jsonl", `{"query":"hi"}`),
+		},
+		Options: &opt_eval.Options{
+			EvalModel:         "gpt-4o-mini",
+			OptimizationModel: "gpt-5",
+		},
+	}
+
+	req, _, err := cfg.ToRequest()
+	require.NoError(t, err)
+	assert.Nil(t, req.EvaluatorInitParamsMap)
+}

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml.go

@@ -90,10 +90,16 @@ func (c *Config) RemoteDatasetReference() *DatasetRef {
 
 // EvaluatorRef describes an evaluator. It can be a simple string name or a
 // structured entry with name, version, and local_uri.
+// EvaluatorRef describes an evaluator. It can be a simple string name or a
+// structured entry with name, version, local_uri, and initialization_parameters.
 type EvaluatorRef struct {
-	Name     string `yaml:"name" json:"name"`
-	Version  string `yaml:"version,omitempty" json:"version,omitempty"`
-	LocalURI string `yaml:"local_uri,omitempty" json:"local_uri,omitempty"`
+	Name     string         `yaml:"name" json:"name"`
+	Version  string         `yaml:"version,omitempty" json:"version,omitempty"`
+	LocalURI string         `yaml:"local_uri,omitempty" json:"local_uri,omitempty"`
+	// InitializationParameters holds evaluator-specific configuration parameters
+	// such as regex_match's ``patterns``. Collected into EvaluatorInitParamsMap
+	// in the API request; not serialized per-evaluator in the evaluators array.
+	InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"-"`
 }
 
 // EvaluatorList is a list of evaluators that supports mixed YAML:

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/opt_eval/yaml_test.go

@@ -415,3 +415,50 @@ optimization_config:
 	// model should be the JSON string, not double-quoted.
 	assert.JSONEq(t, `"gpt-4o"`, string(opts.OptimizationConfig["model"]))
 }
+
+// TestOptions_EvaluatorInitializationParameters verifies initialization_parameters
+// under an evaluator entry is parsed correctly.
+func TestOptions_EvaluatorInitializationParameters(t *testing.T) {
+	t.Parallel()
+
+	input := `
+name: test
+agent:
+  name: my-agent
+evaluators:
+  - name: builtin.regex_match
+    version: "4"
+    initialization_parameters:
+      patterns:
+        - "(?i)Answer:\\s*{{ground_truth}}"
+`
+	var cfg Config
+	require.NoError(t, yaml.Unmarshal([]byte(input), &cfg))
+
+	require.Len(t, cfg.Evaluators, 1)
+	ref := cfg.Evaluators[0]
+	assert.Equal(t, "builtin.regex_match", ref.Name)
+	require.NotNil(t, ref.InitializationParameters)
+	patterns, ok := ref.InitializationParameters["patterns"]
+	require.True(t, ok)
+	assert.NotEmpty(t, patterns)
+}
+
+// TestOptions_EvaluatorInitializationParametersOmitted verifies
+// InitializationParameters is nil when absent.
+func TestOptions_EvaluatorInitializationParametersOmitted(t *testing.T) {
+	t.Parallel()
+
+	input := `
+name: test
+agent:
+  name: my-agent
+evaluators:
+  - name: builtin.task_adherence
+`
+	var cfg Config
+	require.NoError(t, yaml.Unmarshal([]byte(input), &cfg))
+
+	require.Len(t, cfg.Evaluators, 1)
+	assert.Nil(t, cfg.Evaluators[0].InitializationParameters)
+}

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/optimize_api/models.go

@@ -48,11 +48,16 @@ func IsTerminal(status string) bool {
 
 // OptimizeRequest is the top-level payload sent to POST /optimize.
 type OptimizeRequest struct {
-	Agent             AgentIdentifier `json:"agent"`
-	TrainDataset      *Dataset        `json:"train_dataset,omitempty"`
-	ValidationDataset *Dataset        `json:"validation_dataset,omitempty"`
-	Evaluators        []EvaluatorRef  `json:"evaluators,omitempty"`
-	Options           OptimizeOptions `json:"options"`
+	Agent                  AgentIdentifier           `json:"agent"`
+	TrainDataset           *Dataset                  `json:"train_dataset,omitempty"`
+	ValidationDataset      *Dataset                  `json:"validation_dataset,omitempty"`
+	Evaluators             []EvaluatorRef            `json:"evaluators,omitempty"`
+	Options                OptimizeOptions           `json:"options"`
+	// EvaluatorInitParamsMap maps evaluator name to its initialization_parameters.
+	// The C# API stores this in Cosmos; the Python optimizer reads it via
+	// cosmos_loader._resolve_evaluator_init_params_map to supply required params
+	// (e.g. regex_match's ``patterns``) when building testing criteria.
+	EvaluatorInitParamsMap map[string]map[string]any `json:"evaluatorInitParamsMap,omitempty"`
 }
 
 // AgentIdentifier references the agent to optimize by name and optional version.