Add multi-model support for Copilot engine with wildcard matching and CLI-based selection

.github/workflows/test-multi-model.md

@@ -0,0 +1,22 @@
+---
+name: Test Multi-Model Selection
+description: Test workflow to verify multi-model selection with wildcards works correctly
+on: workflow_dispatch
+engine:
+ id: copilot
+ model:
+ - "gpt-5"
+ - "gpt-4o"
+ - "gpt-*-mini"
+---
+
+# Test Multi-Model Selection
+
+This is a simple test workflow to verify that multi-model selection works correctly.
+
+The workflow should:
+1. Validate secrets
+2. Select the first available model from the list (gpt-5, gpt-4o, or any gpt-*-mini model)
+3. Use the selected model for execution
+
+Please respond with a simple message confirming which model was selected.

pkg/parser/schemas/main_workflow_schema.json

@@ -4766,8 +4766,21 @@
  "examples": ["beta", "stable", 20, 3.11]
  },
  "model": {
- "type": "string",
- "description": "Optional specific LLM model to use (e.g., 'claude-3-5-sonnet-20241022', 'gpt-4'). Has sensible defaults and can typically be omitted."
+ "oneOf": [
+ {
+ "type": "string",
+ "description": "Single LLM model to use (e.g., 'claude-3-5-sonnet-20241022', 'gpt-4')"
+ },
+ {
+ "type": "array",
+ "description": "Array of LLM models in priority order. The first compatible model will be selected. Supports wildcard patterns (e.g., 'sonnet*', 'gpt-*-mini').",
+ "items": {
+ "type": "string"
+ },
+ "minItems": 1
+ }
+ ],
+ "description": "Optional specific LLM model(s) to use. Can be a single model string or an array of models in priority order. Supports wildcard patterns like 'sonnet*' or 'gpt-*-mini'. Has sensible defaults and can typically be omitted."
  },
  "max-turns": {
  "type": "integer",

pkg/workflow/agentic_engine.go

@@ -55,6 +55,16 @@ type CodingAgentEngine interface {
 	// GetInstallationSteps returns the GitHub Actions steps needed to install this engine
 	GetInstallationSteps(workflowData *WorkflowData) []GitHubActionStep
 
+	// GetModelSelectionStep returns a GitHub Actions step that validates and selects a compatible model
+	// from the list of models specified in the workflow configuration. Returns nil if no model selection is needed.
+	// The step should:
+	// 1. Query available models from the engine
+	// 2. Match against the requested models (supporting wildcards like "sonnet*", "gpt-*-mini")
+	// 3. Select the first compatible model
+	// 4. Set the selected model as an output variable
+	// 5. Exit with error if no compatible models are found
+	GetModelSelectionStep(workflowData *WorkflowData) GitHubActionStep
+
 	// GetExecutionSteps returns the GitHub Actions steps for executing this engine
 	GetExecutionSteps(workflowData *WorkflowData, logFile string) []GitHubActionStep
 
@@ -157,6 +167,12 @@ func (e *BaseEngine) GetDeclaredOutputFiles() []string {
 	return []string{}
 }
 
+// GetModelSelectionStep returns nil by default (engines can override to provide model selection)
+// Engines that support multiple models can override this to return a step that validates and selects a model
+func (e *BaseEngine) GetModelSelectionStep(workflowData *WorkflowData) GitHubActionStep {
+	return nil
+}
+
 // GetErrorPatterns returns an empty list by default (engines can override)
 func (e *BaseEngine) GetErrorPatterns() []ErrorPattern {
 	return []ErrorPattern{}

pkg/workflow/copilot_engine.go

@@ -1,6 +1,7 @@
 package workflow
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -41,6 +42,44 @@
 	return constants.DefaultCopilotDetectionModel
 }
 
+// GetModelSelectionStep returns a step that validates and selects a compatible model
+// Returns nil if no model selection is needed (single model or no models specified)
+func (e *CopilotEngine) GetModelSelectionStep(workflowData *WorkflowData) GitHubActionStep {
+	// Only generate model selection step if multiple models are specified
+	if workflowData.EngineConfig == nil || len(workflowData.EngineConfig.Models) <= 1 {
+ return nil
+	}
+
+	copilotLog.Printf("Generating model selection step for %d models", len(workflowData.EngineConfig.Models))
+
+	// Convert models array to JSON
+	requestedModelsJSON, err := json.Marshal(workflowData.EngineConfig.Models)
+	if err != nil {
+ copilotLog.Printf("Error marshaling requested models: %v", err)
+ return nil
+	}
+
+	script := getSelectModelWithCLIScript()
+
+	stepLines := []string{
+ " - name: Select Compatible Model",
+ " id: select_model",
+ fmt.Sprintf(" uses: %s", GetActionPin("actions/github-script")),
+ " with:",
+ " script: |",
+	}
+
+	// Inline the JavaScript code with proper indentation
+	scriptLines := FormatJavaScriptForYAML(script)
+	stepLines = append(stepLines, scriptLines...)
+
+	// Add inputs section
+	stepLines = append(stepLines, " inputs:",
+ fmt.Sprintf(" requested_models: '%s'", string(requestedModelsJSON)))
+
+	return GitHubActionStep(stepLines)
+}
+
 func (e *CopilotEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubActionStep {
 	copilotLog.Printf("Generating installation steps for Copilot engine: workflow=%s", workflowData.Name)
 
@@ -65,6 +104,13 @@
 	)
 	steps = append(steps, secretValidation)
 
+	// Add model selection step if multiple models are specified
+	modelSelectionStep := e.GetModelSelectionStep(workflowData)
+	if modelSelectionStep != nil {
+ copilotLog.Print("Adding model selection step")
+ steps = append(steps, modelSelectionStep)
+	}
+
 	// Determine Copilot version
 	copilotVersion := config.Version
 	if workflowData.EngineConfig != nil && workflowData.EngineConfig.Version != "" {
@@ -195,9 +241,22 @@
 	copilotArgs = append(copilotArgs, "--disable-builtin-mcps")
 
 	// Add model if specified (check if Copilot CLI supports this)
-	if workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" {
- copilotLog.Printf("Using custom model: %s", workflowData.EngineConfig.Model)
- copilotArgs = append(copilotArgs, "--model", workflowData.EngineConfig.Model)
+	if workflowData.EngineConfig != nil {
+ if len(workflowData.EngineConfig.Models) > 1 {
+ // Multiple models specified - use the selected model from the selection step
+ // Only add --model if selected_model is not empty (empty means "*" wildcard was matched)
+ copilotLog.Print("Using model from selection step if not empty: ${{ steps.select_model.outputs.selected_model }}")
+ // Use shell conditional to only add --model if selected_model is not empty
+ // This will be evaluated at runtime in the workflow
+ modelArg := "${{ steps.select_model.outputs.selected_model && format('--model {0}', steps.select_model.outputs.selected_model) || '' }}"
+ if modelArg != "" {
+ copilotArgs = append(copilotArgs, modelArg)
+ }
+ } else if workflowData.EngineConfig.Model != "" {
+ // Single model specified directly
+ copilotLog.Printf("Using custom model: %s", workflowData.EngineConfig.Model)
+ copilotArgs = append(copilotArgs, "--model", workflowData.EngineConfig.Model)
+ }
 	}
 
 	// Add --agent flag if custom agent file is specified (via imports)

pkg/workflow/engine.go

@@ -13,7 +13,8 @@ var engineLog = logger.New("workflow:engine")
 type EngineConfig struct {
 	ID string
 	Version string
-	Model string
+	Model string // Single model (for backward compatibility)
+	Models []string // Array of models in priority order (takes precedence over Model if set)
 	MaxTurns string
 	Concurrency string // Agent job-level concurrency configuration (YAML format)
 	UserAgent string
@@ -69,10 +70,24 @@ func (c *Compiler) ExtractEngineConfig(frontmatter map[string]any) (string, *Eng
  }
  }
 
- // Extract optional 'model' field
+ // Extract optional 'model' field (supports both string and array)
  if model, hasModel := engineObj["model"]; hasModel {
  if modelStr, ok := model.(string); ok {
+ // Single model string
  config.Model = modelStr
+ config.Models = []string{modelStr}
+ } else if modelArray, ok := model.([]any); ok {
+ // Array of models
+ var models []string
+ for _, m := range modelArray {
+ if mStr, ok := m.(string); ok {
+ models = append(models, mStr)
+ }
+ }
+ if len(models) > 0 {
+ config.Models = models
+ config.Model = models[0] // For backward compatibility, set Model to first item
+ }
  }
  }
 

pkg/workflow/engine_config_test.go

@@ -73,7 +73,7 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "codex",
- expectedConfig: &EngineConfig{ID: "codex", Model: "gpt-4o"},
+ expectedConfig: &EngineConfig{ID: "codex", Model: "gpt-4o", Models: []string{"gpt-4o"}},
  },
  {
  name: "object format - complete",
@@ -85,7 +85,7 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "claude",
- expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022"},
+ expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", Models: []string{"claude-3-5-sonnet-20241022"}},
  },
  {
  name: "object format - with max-turns",
@@ -109,7 +109,7 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "claude",
- expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", MaxTurns: "10"},
+ expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", MaxTurns: "10", Models: []string{"claude-3-5-sonnet-20241022"}},
  },
  {
  name: "object format - with env vars",
@@ -140,7 +140,7 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "claude",
- expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", MaxTurns: "5", Env: map[string]string{"AWS_REGION": "us-west-2", "API_ENDPOINT": "https://api.example.com"}},
+ expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", MaxTurns: "5", Env: map[string]string{"AWS_REGION": "us-west-2", "API_ENDPOINT": "https://api.example.com"}, Models: []string{"claude-3-5-sonnet-20241022"}},
  },
  {
  name: "custom engine with steps",
@@ -189,7 +189,7 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "",
- expectedConfig: &EngineConfig{Version: "beta", Model: "gpt-4o"},
+ expectedConfig: &EngineConfig{Version: "beta", Model: "gpt-4o", Models: []string{"gpt-4o"}},
  },
  {
  name: "object format - with user-agent (hyphen)",
@@ -217,7 +217,29 @@ func TestExtractEngineConfig(t *testing.T) {
  },
  },
  expectedEngineSetting: "codex",
- expectedConfig: &EngineConfig{ID: "codex", Version: "beta", Model: "gpt-4o", MaxTurns: "3", UserAgent: "complete-custom-agent", Env: map[string]string{"CUSTOM_VAR": "value1"}},
+ expectedConfig: &EngineConfig{ID: "codex", Version: "beta", Model: "gpt-4o", MaxTurns: "3", UserAgent: "complete-custom-agent", Env: map[string]string{"CUSTOM_VAR": "value1"}, Models: []string{"gpt-4o"}},
+ },
+ {
+ name: "object format - with multiple models (array)",
+ frontmatter: map[string]any{
+ "engine": map[string]any{
+ "id": "copilot",
+ "model": []any{"gpt-5", "gpt-4o", "gpt-*-mini"},
+ },
+ },
+ expectedEngineSetting: "copilot",
+ expectedConfig: &EngineConfig{ID: "copilot", Model: "gpt-5", Models: []string{"gpt-5", "gpt-4o", "gpt-*-mini"}},
+ },
+ {
+ name: "object format - with multiple models (array) for claude",
+ frontmatter: map[string]any{
+ "engine": map[string]any{
+ "id": "claude",
+ "model": []any{"claude-3-opus-20240229", "*sonnet*", "claude-*"},
+ },
+ },
+ expectedEngineSetting: "claude",
+ expectedConfig: &EngineConfig{ID: "claude", Model: "claude-3-opus-20240229", Models: []string{"claude-3-opus-20240229", "*sonnet*", "claude-*"}},
  },
 	}
 
@@ -259,6 +281,17 @@ func TestExtractEngineConfig(t *testing.T) {
  t.Errorf("Expected config.UserAgent '%s', got '%s'", test.expectedConfig.UserAgent, config.UserAgent)
  }
 
+ // Check Models field
+ if len(config.Models) != len(test.expectedConfig.Models) {
+ t.Errorf("Expected config.Models length %d, got %d", len(test.expectedConfig.Models), len(config.Models))
+ } else {
+ for i, expectedModel := range test.expectedConfig.Models {
+ if config.Models[i] != expectedModel {
+ t.Errorf("Expected config.Models[%d] = '%s', got '%s'", i, expectedModel, config.Models[i])
+ }
+ }
+ }
+
  if len(config.Env) != len(test.expectedConfig.Env) {
  t.Errorf("Expected config.Env length %d, got %d", len(test.expectedConfig.Env), len(config.Env))
  } else {
@@ -344,7 +377,7 @@ engine:
 
 This is a test workflow.`,
  expectedAI: "claude",
- expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022"},
+ expectedConfig: &EngineConfig{ID: "claude", Version: "beta", Model: "claude-3-5-sonnet-20241022", Models: []string{"claude-3-5-sonnet-20241022"}},
  },
  {
  name: "object engine format - codex with model",
@@ -364,7 +397,7 @@ engine:
 
 This is a test workflow.`,
  expectedAI: "codex",
- expectedConfig: &EngineConfig{ID: "codex", Model: "gpt-4o"},
+ expectedConfig: &EngineConfig{ID: "codex", Model: "gpt-4o", Models: []string{"gpt-4o"}},
  },
 	}
 

pkg/workflow/js.go

@@ -77,6 +77,12 @@ var validateErrorsScript string
 //go:embed js/missing_tool.cjs
 var missingToolScript string
 
+//go:embed js/select_model.cjs
+var selectModelScript string
+
+//go:embed js/select_model_with_cli.cjs
+var selectModelWithCLIScript string
+
 //go:embed js/safe_outputs_mcp_server.cjs
 var safeOutputsMCPServerScriptSource string
 
@@ -85,6 +91,16 @@ func getSafeOutputsMCPServerScript() string {
 	return DefaultScriptRegistry.GetWithMode("safe_outputs_mcp_server", RuntimeModeGitHubScript)
 }
 
+// getSelectModelScript returns the bundled select_model script
+func getSelectModelScript() string {
+	return selectModelScript
+}
+
+// getSelectModelWithCLIScript returns the bundled select_model_with_cli script
+func getSelectModelWithCLIScript() string {
+	return selectModelWithCLIScript
+}
+
 //go:embed js/safe_outputs_tools.json
 var safeOutputsToolsJSON string