| 1 | """Run on a single SWE-Bench instance."""
|
| 2 |
|
| 3 | from pathlib import Path
|
| 4 |
|
| 5 | import typer
|
| 6 | from datasets import load_dataset
|
| 7 |
|
| 8 | from minisweagent import global_config_dir
|
| 9 | from minisweagent.agents.interactive import InteractiveAgent
|
| 10 | from minisweagent.config import builtin_config_dir, get_config_from_spec
|
| 11 | from minisweagent.models import get_model
|
| 12 | from minisweagent.run.benchmarks.swebench import (
|
| 13 | DATASET_MAPPING,
|
| 14 | get_sb_environment,
|
| 15 | )
|
| 16 | from minisweagent.utils.log import logger
|
| 17 | from minisweagent.utils.serialize import recursive_merge
|
| 18 |
|
| 19 | DEFAULT_OUTPUT_FILE = global_config_dir / "last_swebench_single_run.traj.json"
|
| 20 | DEFAULT_CONFIG_FILE = builtin_config_dir / "benchmarks" / "swebench.yaml"
|
| 21 |
|
| 22 | app = typer.Typer(add_completion=False)
|
| 23 |
|
| 24 | _CONFIG_SPEC_HELP_TEXT = """Path to config files, filenames, or key-value pairs.
|
| 25 |
|
| 26 | [bold red]IMPORTANT:[/bold red] [red]If you set this option, the default config file will not be used.[/red]
|
| 27 | So you need to explicitly set it e.g., with [bold green]-c swebench.yaml <other options>[/bold green]
|
| 28 |
|
| 29 | Multiple configs will be recursively merged.
|
| 30 |
|
| 31 | Examples:
|
| 32 |
|
| 33 | [bold red]-c model.model_kwargs.temperature=0[/bold red] [red]You forgot to add the default config file! See above.[/red]
|
| 34 |
|
| 35 | [bold green]-c swebench.yaml -c model.model_kwargs.temperature=0.5[/bold green]
|
| 36 |
|
| 37 | [bold green]-c swebench.yaml -c agent.mode=yolo[/bold green]
|
| 38 | """
|
| 39 |
|
| 40 |
|
| 41 | # fmt: off
|
| 42 | @app.command()
|
| 43 | def main(
|
| 44 | subset: str = typer.Option("lite", "--subset", help="SWEBench subset to use or path to a dataset", rich_help_panel="Data selection"),
|
| 45 | split: str = typer.Option("dev", "--split", help="Dataset split", rich_help_panel="Data selection"),
|
| 46 | instance_spec: str = typer.Option(0, "-i", "--instance", help="SWE-Bench instance ID or index", rich_help_panel="Data selection"),
|
| 47 | model_name: str | None = typer.Option(None, "-m", "--model", help="Model to use", rich_help_panel="Basic"),
|
| 48 | model_class: str | None = typer.Option(None, "--model-class", help="Model class to use (e.g., 'anthropic' or 'minisweagent.models.anthropic.AnthropicModel')", rich_help_panel="Advanced"),
|
| 49 | config_spec: list[str] = typer.Option([str(DEFAULT_CONFIG_FILE)], "-c", "--config", help=_CONFIG_SPEC_HELP_TEXT, rich_help_panel="Basic"),
|
| 50 | environment_class: str | None = typer.Option(None, "--environment-class", rich_help_panel="Advanced"),
|
| 51 | exit_immediately: bool = typer.Option( False, "--exit-immediately", help="Exit immediately when the agent wants to finish instead of prompting.", rich_help_panel="Basic"),
|
| 52 | output: Path = typer.Option(DEFAULT_OUTPUT_FILE, "-o", "--output", help="Output trajectory file", rich_help_panel="Basic"),
|
| 53 | ) -> None:
|
| 54 | # fmt: on
|
| 55 | """Run on a single SWE-Bench instance."""
|
| 56 | dataset_path = DATASET_MAPPING.get(subset, subset)
|
| 57 | logger.info(f"Loading dataset from {dataset_path}, split {split}...")
|
| 58 | instances = {
|
| 59 | inst["instance_id"]: inst # type: ignore
|
| 60 | for inst in load_dataset(dataset_path, split=split)
|
| 61 | }
|
| 62 | if instance_spec.isnumeric():
|
| 63 | instance_spec = sorted(instances.keys())[int(instance_spec)]
|
| 64 | instance: dict = instances[instance_spec] # type: ignore
|
| 65 |
|
| 66 | logger.info(f"Building agent config from specs: {config_spec}")
|
| 67 | configs = [get_config_from_spec(spec) for spec in config_spec]
|
| 68 | configs.append({"agent": {"mode": "yolo"}})
|
| 69 | if environment_class is not None:
|
| 70 | configs.append({"environment": {"environment_class": environment_class}})
|
| 71 | if model_class is not None:
|
| 72 | configs.append({"model": {"model_class": model_class}})
|
| 73 | if model_name is not None:
|
| 74 | configs.append({"model": {"model_name": model_name}})
|
| 75 | if exit_immediately:
|
| 76 | configs.append({"agent": {"confirm_exit": False}})
|
| 77 | configs.append({"agent": {"output_path": output}})
|
| 78 | config = recursive_merge(*configs)
|
| 79 |
|
| 80 | env = get_sb_environment(config, instance)
|
| 81 | agent = InteractiveAgent(
|
| 82 | get_model(config=config.get("model", {})),
|
| 83 | env,
|
| 84 | **config.get("agent", {}),
|
| 85 | )
|
| 86 | agent.run(instance["problem_statement"])
|
| 87 |
|
| 88 |
|
| 89 | if __name__ == "__main__":
|
| 90 | app()
|
| 91 |
|