| 1 | import json
|
| 2 | import os
|
| 3 | from unittest.mock import Mock, patch
|
| 4 |
|
| 5 | import pytest
|
| 6 | import requests
|
| 7 |
|
| 8 | from minisweagent.models import GLOBAL_MODEL_STATS
|
| 9 | from minisweagent.models.openrouter_model import (
|
| 10 | OpenRouterAuthenticationError,
|
| 11 | )
|
| 12 | from minisweagent.models.openrouter_textbased_model import (
|
| 13 | OpenRouterTextbasedModel,
|
| 14 | )
|
| 15 |
|
| 16 |
|
| 17 | @pytest.fixture
|
| 18 | def mock_response():
|
| 19 | """Create a mock successful OpenRouter API response."""
|
| 20 | # Response must include bash block to avoid FormatError from parse_action
|
| 21 | return {
|
| 22 | "choices": [{"message": {"content": "```mswea_bash_command\necho '2+2 equals 4'\n```"}}],
|
| 23 | "usage": {
|
| 24 | "prompt_tokens": 16,
|
| 25 | "completion_tokens": 13,
|
| 26 | "total_tokens": 29,
|
| 27 | "cost": 0.000243,
|
| 28 | "is_byok": False,
|
| 29 | "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
|
| 30 | "cost_details": {
|
| 31 | "upstream_inference_cost": None,
|
| 32 | "upstream_inference_prompt_cost": 4.8e-05,
|
| 33 | "upstream_inference_completions_cost": 0.000195,
|
| 34 | },
|
| 35 | },
|
| 36 | }
|
| 37 |
|
| 38 |
|
| 39 | @pytest.fixture
|
| 40 | def mock_response_no_cost():
|
| 41 | """Create a mock OpenRouter API response without cost information."""
|
| 42 | # Response must include bash block to avoid FormatError from parse_action
|
| 43 | return {
|
| 44 | "choices": [{"message": {"content": "```mswea_bash_command\necho '2+2 equals 4'\n```"}}],
|
| 45 | "usage": {"prompt_tokens": 16, "completion_tokens": 13, "total_tokens": 29},
|
| 46 | }
|
| 47 |
|
| 48 |
|
| 49 | def test_openrouter_model_successful_query(mock_response):
|
| 50 | """Test successful OpenRouter API query with cost tracking."""
|
| 51 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
| 52 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.7})
|
| 53 |
|
| 54 | initial_cost = GLOBAL_MODEL_STATS.cost
|
| 55 |
|
| 56 | with patch("requests.post") as mock_post:
|
| 57 | # Mock successful response
|
| 58 | mock_post.return_value.status_code = 200
|
| 59 | mock_post.return_value.json.return_value = mock_response
|
| 60 | mock_post.return_value.raise_for_status.return_value = None
|
| 61 |
|
| 62 | messages = [{"role": "user", "content": "Hello! What is 2+2?"}]
|
| 63 | result = model.query(messages)
|
| 64 |
|
| 65 | # Verify the request was made correctly
|
| 66 | mock_post.assert_called_once()
|
| 67 | call_args = mock_post.call_args
|
| 68 |
|
| 69 | # Check URL (first positional argument)
|
| 70 | assert call_args[0][0] == "https://openrouter.ai/api/v1/chat/completions"
|
| 71 |
|
| 72 | # Check headers
|
| 73 | headers = call_args[1]["headers"]
|
| 74 | assert headers["Authorization"] == "Bearer test-key"
|
| 75 | assert headers["Content-Type"] == "application/json"
|
| 76 |
|
| 77 | # Check payload
|
| 78 | payload = json.loads(call_args[1]["data"])
|
| 79 | assert payload["model"] == "anthropic/claude-3.5-sonnet"
|
| 80 | assert payload["messages"] == messages
|
| 81 | assert payload["usage"]["include"] is True
|
| 82 | assert payload["temperature"] == 0.7
|
| 83 |
|
| 84 | # Verify response
|
| 85 | assert result["content"] == "```mswea_bash_command\necho '2+2 equals 4'\n```"
|
| 86 | assert result["extra"]["actions"] == [{"command": "echo '2+2 equals 4'"}]
|
| 87 | assert result["extra"]["response"] == mock_response
|
| 88 |
|
| 89 | # Verify cost tracking
|
| 90 | assert GLOBAL_MODEL_STATS.cost == initial_cost + 0.000243
|
| 91 |
|
| 92 |
|
| 93 | def test_openrouter_model_authentication_error():
|
| 94 | """Test authentication error handling."""
|
| 95 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "invalid-key"}):
|
| 96 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
|
| 97 |
|
| 98 | with patch("requests.post") as mock_post:
|
| 99 | # Mock 401 authentication error
|
| 100 | mock_response = Mock()
|
| 101 | mock_response.status_code = 401
|
| 102 | mock_response.text = "Unauthorized"
|
| 103 | mock_post.return_value = mock_response
|
| 104 | mock_post.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError()
|
| 105 |
|
| 106 | messages = [{"role": "user", "content": "test"}]
|
| 107 |
|
| 108 | with pytest.raises(OpenRouterAuthenticationError) as exc_info:
|
| 109 | model._query(messages)
|
| 110 |
|
| 111 | assert "Authentication failed" in str(exc_info.value)
|
| 112 | assert "mini-extra config set OPENROUTER_API_KEY" in str(exc_info.value)
|
| 113 |
|
| 114 |
|
| 115 | def test_openrouter_model_no_cost_information(mock_response_no_cost):
|
| 116 | """Test error when cost information is missing."""
|
| 117 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
| 118 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
|
| 119 |
|
| 120 | with patch("requests.post") as mock_post:
|
| 121 | mock_post.return_value.status_code = 200
|
| 122 | mock_post.return_value.json.return_value = mock_response_no_cost
|
| 123 | mock_post.return_value.raise_for_status.return_value = None
|
| 124 |
|
| 125 | messages = [{"role": "user", "content": "test"}]
|
| 126 |
|
| 127 | with pytest.raises(RuntimeError) as exc_info:
|
| 128 | model.query(messages)
|
| 129 |
|
| 130 | assert "No valid cost information available" in str(exc_info.value)
|
| 131 | assert "MSWEA_COST_TRACKING='ignore_errors'" in str(exc_info.value)
|
| 132 |
|
| 133 |
|
| 134 | def test_openrouter_model_free_model_zero_cost(mock_response_no_cost):
|
| 135 | """Test that free models with zero cost work correctly when cost_tracking='ignore_errors' is set."""
|
| 136 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
| 137 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", cost_tracking="ignore_errors")
|
| 138 |
|
| 139 | initial_cost = GLOBAL_MODEL_STATS.cost
|
| 140 |
|
| 141 | with patch("requests.post") as mock_post:
|
| 142 | mock_post.return_value.status_code = 200
|
| 143 | mock_post.return_value.json.return_value = mock_response_no_cost
|
| 144 | mock_post.return_value.raise_for_status.return_value = None
|
| 145 |
|
| 146 | messages = [{"role": "user", "content": "test"}]
|
| 147 |
|
| 148 | # With cost_tracking='ignore_errors', free models should work without raising an error
|
| 149 | result = model.query(messages)
|
| 150 |
|
| 151 | # Verify response
|
| 152 | assert result["content"] == "```mswea_bash_command\necho '2+2 equals 4'\n```"
|
| 153 | assert result["extra"]["actions"] == [{"command": "echo '2+2 equals 4'"}]
|
| 154 | assert result["extra"]["response"] == mock_response_no_cost
|
| 155 |
|
| 156 | # Verify cost tracking with zero cost (not added to global stats when zero)
|
| 157 | # Cost should not be added to global stats since it's zero
|
| 158 | assert GLOBAL_MODEL_STATS.cost == initial_cost
|
| 159 |
|
| 160 |
|
| 161 | def test_openrouter_model_config():
|
| 162 | """Test OpenRouter model configuration."""
|
| 163 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
| 164 | model = OpenRouterTextbasedModel(
|
| 165 | model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.5, "max_tokens": 1000}
|
| 166 | )
|
| 167 |
|
| 168 | assert model.config.model_name == "anthropic/claude-3.5-sonnet"
|
| 169 | assert model.config.model_kwargs == {"temperature": 0.5, "max_tokens": 1000}
|
| 170 | assert model._api_key == "test-key"
|
| 171 |
|
| 172 |
|
| 173 | def test_openrouter_model_get_template_vars():
|
| 174 | """Test get_template_vars method."""
|
| 175 | with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
| 176 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet", model_kwargs={"temperature": 0.7})
|
| 177 |
|
| 178 | template_vars = model.get_template_vars()
|
| 179 |
|
| 180 | assert template_vars["model_name"] == "anthropic/claude-3.5-sonnet"
|
| 181 | assert template_vars["model_kwargs"] == {"temperature": 0.7}
|
| 182 |
|
| 183 |
|
| 184 | def test_openrouter_model_no_api_key():
|
| 185 | """Test behavior when no API key is provided."""
|
| 186 | with patch.dict(os.environ, {}, clear=True):
|
| 187 | model = OpenRouterTextbasedModel(model_name="anthropic/claude-3.5-sonnet")
|
| 188 |
|
| 189 | assert model._api_key == ""
|
| 190 |
|
| 191 | with patch("requests.post") as mock_post:
|
| 192 | mock_response = Mock()
|
| 193 | mock_response.status_code = 401
|
| 194 | mock_response.text = "Unauthorized"
|
| 195 | mock_post.return_value = mock_response
|
| 196 | mock_post.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError()
|
| 197 |
|
| 198 | messages = [{"role": "user", "content": "test"}]
|
| 199 |
|
| 200 | with pytest.raises(OpenRouterAuthenticationError):
|
| 201 | model._query(messages)
|
| 202 |
|