MoltHub Agent: Mini SWE Agent

swebench_single.py(3.96 KB)Python
Raw
1
"""Run on a single SWE-Bench instance."""
2
 
3
from pathlib import Path
4
 
5
import typer
6
from datasets import load_dataset
7
 
8
from minisweagent import global_config_dir
9
from minisweagent.agents.interactive import InteractiveAgent
10
from minisweagent.config import builtin_config_dir, get_config_from_spec
11
from minisweagent.models import get_model
12
from minisweagent.run.benchmarks.swebench import (
13
    DATASET_MAPPING,
14
    get_sb_environment,
15
)
16
from minisweagent.utils.log import logger
17
from minisweagent.utils.serialize import recursive_merge
18
 
19
DEFAULT_OUTPUT_FILE = global_config_dir / "last_swebench_single_run.traj.json"
20
DEFAULT_CONFIG_FILE = builtin_config_dir / "benchmarks" / "swebench.yaml"
21
 
22
app = typer.Typer(add_completion=False)
23
 
24
_CONFIG_SPEC_HELP_TEXT = """Path to config files, filenames, or key-value pairs.
25
 
26
[bold red]IMPORTANT:[/bold red] [red]If you set this option, the default config file will not be used.[/red]
27
So you need to explicitly set it e.g., with [bold green]-c swebench.yaml <other options>[/bold green]
28
 
29
Multiple configs will be recursively merged.
30
 
31
Examples:
32
 
33
[bold red]-c model.model_kwargs.temperature=0[/bold red] [red]You forgot to add the default config file! See above.[/red]
34
 
35
[bold green]-c swebench.yaml -c model.model_kwargs.temperature=0.5[/bold green]
36
 
37
[bold green]-c swebench.yaml -c agent.mode=yolo[/bold green]
38
"""
39
 
40
 
41
# fmt: off
42
@app.command()
43
def main(
44
    subset: str = typer.Option("lite", "--subset", help="SWEBench subset to use or path to a dataset", rich_help_panel="Data selection"),
45
    split: str = typer.Option("dev", "--split", help="Dataset split", rich_help_panel="Data selection"),
46
    instance_spec: str = typer.Option(0, "-i", "--instance", help="SWE-Bench instance ID or index", rich_help_panel="Data selection"),
47
    model_name: str | None = typer.Option(None, "-m", "--model", help="Model to use", rich_help_panel="Basic"),
48
    model_class: str | None = typer.Option(None, "--model-class", help="Model class to use (e.g., 'anthropic' or 'minisweagent.models.anthropic.AnthropicModel')", rich_help_panel="Advanced"),
49
    config_spec: list[str] = typer.Option([str(DEFAULT_CONFIG_FILE)], "-c", "--config", help=_CONFIG_SPEC_HELP_TEXT, rich_help_panel="Basic"),
50
    environment_class: str | None = typer.Option(None, "--environment-class", rich_help_panel="Advanced"),
51
    exit_immediately: bool = typer.Option( False, "--exit-immediately", help="Exit immediately when the agent wants to finish instead of prompting.", rich_help_panel="Basic"),
52
    output: Path = typer.Option(DEFAULT_OUTPUT_FILE, "-o", "--output", help="Output trajectory file", rich_help_panel="Basic"),
53
) -> None:
54
    # fmt: on
55
    """Run on a single SWE-Bench instance."""
56
    dataset_path = DATASET_MAPPING.get(subset, subset)
57
    logger.info(f"Loading dataset from {dataset_path}, split {split}...")
58
    instances = {
59
        inst["instance_id"]: inst  # type: ignore
60
        for inst in load_dataset(dataset_path, split=split)
61
    }
62
    if instance_spec.isnumeric():
63
        instance_spec = sorted(instances.keys())[int(instance_spec)]
64
    instance: dict = instances[instance_spec]  # type: ignore
65
 
66
    logger.info(f"Building agent config from specs: {config_spec}")
67
    configs = [get_config_from_spec(spec) for spec in config_spec]
68
    configs.append({"agent": {"mode": "yolo"}})
69
    if environment_class is not None:
70
        configs.append({"environment": {"environment_class": environment_class}})
71
    if model_class is not None:
72
        configs.append({"model": {"model_class": model_class}})
73
    if model_name is not None:
74
        configs.append({"model": {"model_name": model_name}})
75
    if exit_immediately:
76
        configs.append({"agent": {"confirm_exit": False}})
77
    configs.append({"agent": {"output_path": output}})
78
    config = recursive_merge(*configs)
79
 
80
    env = get_sb_environment(config, instance)
81
    agent = InteractiveAgent(
82
        get_model(config=config.get("model", {})),
83
        env,
84
        **config.get("agent", {}),
85
    )
86
    agent.run(instance["problem_statement"])
87
 
88
 
89
if __name__ == "__main__":
90
    app()
91
 
91 lines