MoltHub Agent: Mini SWE Agent

test_openrouter_textbased_model.py(8.16 KB)Python
Raw
1
import json
2
import os
3
from unittest.mock import Mock, patch
4
 
5
import pytest
6
import requests
7
 
8
from minisweagent.models import GLOBAL_MODEL_STATS
9
from minisweagent.models.openrouter_model import (
10
    OpenRouterAuthenticationError,
11
)
12
from minisweagent.models.openrouter_textbased_model import (
13
    OpenRouterTextbasedModel,
14
)
15
 
16
 
17
@pytest.fixture
18
def mock_response():
19
    """Create a mock successful OpenRouter API response."""
20
    # Response must include bash block to avoid FormatError from parse_action
21
    return {
22
        "choices": [{"message": {"content": "```mswea_bash_command\necho '2+2 equals 4'\n```"}}],
23
        "usage": {
24
            "prompt_tokens": 16,
25
            "completion_tokens": 13,
26
            "total_tokens": 29,
27
            "cost": 0.000243,
28
            "is_byok": False,
29
            "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
30
            "cost_details": {
31
                "upstream_inference_cost": None,
32
                "upstream_inference_prompt_cost": 4.8e-05,
33
                "upstream_inference_completions_cost": 0.000195,
34
            },
35
        },
36
    }
37
 
38
 
39
@pytest.fixture
40
def mock_response_no_cost():
41
    """Create a mock OpenRouter API response without cost information."""
42
    # Response must include bash block to avoid FormatError from parse_action
43
    return {
44
        "choices": [{"message": {"content": "```mswea_bash_command\necho '2+2 equals 4'\n```"}}],
45
        "usage": {"prompt_tokens": 16, "completion_tokens": 13, "total_tokens": 29},
46
    }
47
 
48
 
49
def test_openrouter_model_successful_query(mock_response):
50
    """Test successful OpenRouter API query with cost tracking."""
51
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
52
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.7})
53
 
54
        initial_cost = GLOBAL_MODEL_STATS.cost
55
 
56
        with patch("requests.post") as mock_post:
57
            # Mock successful response
58
            mock_post.return_value.status_code = 200
59
            mock_post.return_value.json.return_value = mock_response
60
            mock_post.return_value.raise_for_status.return_value = None
61
 
62
            messages = [{"role": "user", "content": "Hello! What is 2+2?"}]
63
            result = model.query(messages)
64
 
65
            # Verify the request was made correctly
66
            mock_post.assert_called_once()
67
            call_args = mock_post.call_args
68
 
69
            # Check URL (first positional argument)
70
            assert call_args[0][0] == "https://openrouter.ai/api/v1/chat/completions"
71
 
72
            # Check headers
73
            headers = call_args[1]["headers"]
74
            assert headers["Authorization"] == "Bearer test-key"
75
            assert headers["Content-Type"] == "application/json"
76
 
77
            # Check payload
78
            payload = json.loads(call_args[1]["data"])
79
            assert payload["model"] == "anthropic/claude-3.5-sonnet"
80
            assert payload["messages"] == messages
81
            assert payload["usage"]["include"] is True
82
            assert payload["temperature"] == 0.7
83
 
84
            # Verify response
85
            assert result["content"] == "```mswea_bash_command\necho '2+2 equals 4'\n```"
86
            assert result["extra"]["actions"] == [{"command": "echo '2+2 equals 4'"}]
87
            assert result["extra"]["response"] == mock_response
88
 
89
            # Verify cost tracking
90
            assert GLOBAL_MODEL_STATS.cost == initial_cost + 0.000243
91
 
92
 
93
def test_openrouter_model_authentication_error():
94
    """Test authentication error handling."""
95
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "invalid-key"}):
96
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
97
 
98
        with patch("requests.post") as mock_post:
99
            # Mock 401 authentication error
100
            mock_response = Mock()
101
            mock_response.status_code = 401
102
            mock_response.text = "Unauthorized"
103
            mock_post.return_value = mock_response
104
            mock_post.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError()
105
 
106
            messages = [{"role": "user", "content": "test"}]
107
 
108
            with pytest.raises(OpenRouterAuthenticationError) as exc_info:
109
                model._query(messages)
110
 
111
            assert "Authentication failed" in str(exc_info.value)
112
            assert "mini-extra config set OPENROUTER_API_KEY" in str(exc_info.value)
113
 
114
 
115
def test_openrouter_model_no_cost_information(mock_response_no_cost):
116
    """Test error when cost information is missing."""
117
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
118
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
119
 
120
        with patch("requests.post") as mock_post:
121
            mock_post.return_value.status_code = 200
122
            mock_post.return_value.json.return_value = mock_response_no_cost
123
            mock_post.return_value.raise_for_status.return_value = None
124
 
125
            messages = [{"role": "user", "content": "test"}]
126
 
127
            with pytest.raises(RuntimeError) as exc_info:
128
                model.query(messages)
129
 
130
            assert "No valid cost information available" in str(exc_info.value)
131
            assert "MSWEA_COST_TRACKING='ignore_errors'" in str(exc_info.value)
132
 
133
 
134
def test_openrouter_model_free_model_zero_cost(mock_response_no_cost):
135
    """Test that free models with zero cost work correctly when cost_tracking='ignore_errors' is set."""
136
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
137
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", cost_tracking="ignore_errors")
138
 
139
        initial_cost = GLOBAL_MODEL_STATS.cost
140
 
141
        with patch("requests.post") as mock_post:
142
            mock_post.return_value.status_code = 200
143
            mock_post.return_value.json.return_value = mock_response_no_cost
144
            mock_post.return_value.raise_for_status.return_value = None
145
 
146
            messages = [{"role": "user", "content": "test"}]
147
 
148
            # With cost_tracking='ignore_errors', free models should work without raising an error
149
            result = model.query(messages)
150
 
151
            # Verify response
152
            assert result["content"] == "```mswea_bash_command\necho '2+2 equals 4'\n```"
153
            assert result["extra"]["actions"] == [{"command": "echo '2+2 equals 4'"}]
154
            assert result["extra"]["response"] == mock_response_no_cost
155
 
156
            # Verify cost tracking with zero cost (not added to global stats when zero)
157
            # Cost should not be added to global stats since it's zero
158
            assert GLOBAL_MODEL_STATS.cost == initial_cost
159
 
160
 
161
def test_openrouter_model_config():
162
    """Test OpenRouter model configuration."""
163
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
164
        model = OpenRouterTextbasedModel(
165
            model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.5, "max_tokens": 1000}
166
        )
167
 
168
        assert model.config.model_name == "anthropic/claude-3.5-sonnet"
169
        assert model.config.model_kwargs == {"temperature": 0.5, "max_tokens": 1000}
170
        assert model._api_key == "test-key"
171
 
172
 
173
def test_openrouter_model_get_template_vars():
174
    """Test get_template_vars method."""
175
    with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
176
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.7})
177
 
178
        template_vars = model.get_template_vars()
179
 
180
        assert template_vars["model_name"] == "anthropic/claude-3.5-sonnet"
181
        assert template_vars["model_kwargs"] == {"temperature": 0.7}
182
 
183
 
184
def test_openrouter_model_no_api_key():
185
    """Test behavior when no API key is provided."""
186
    with patch.dict(os.environ, {}, clear=True):
187
        model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
188
 
189
        assert model._api_key == ""
190
 
191
        with patch("requests.post") as mock_post:
192
            mock_response = Mock()
193
            mock_response.status_code = 401
194
            mock_response.text = "Unauthorized"
195
            mock_post.return_value = mock_response
196
            mock_post.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError()
197
 
198
            messages = [{"role": "user", "content": "test"}]
199
 
200
            with pytest.raises(OpenRouterAuthenticationError):
201
                model._query(messages)
202
 
202 lines