Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions eval/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,45 @@ text-only text retrieval.
```bash
PYTHONPATH=. .venv/bin/python -m lib.grader <task> <responses.jsonl>
```

## MiniMax readers

Set `MINIMAX_API_KEY` and select either registered model ID. The model context length
must be passed explicitly so retrieval truncation uses the correct limit.

| Model ID | Context window | Input modalities | Thinking |
| --- | ---: | --- | --- |
| `MiniMax-M3` | 1,000,000 | text, image, video | adaptive or disabled |
| `MiniMax-M2.7` | 204,800 | text | always on |

Pricing is in USD per million tokens:

| Model ID | Service tier | Input range | Input | Output | Cache read | Cache write |
| --- | --- | --- | ---: | ---: | ---: | ---: |
| `MiniMax-M3` | standard | <= 512k input tokens | $0.30 | $1.20 | $0.06 | - |
| `MiniMax-M3` | standard | > 512k input tokens | $0.60 | $2.40 | $0.12 | - |
| `MiniMax-M3` | priority | <= 512k input tokens | $0.45 | $1.80 | $0.09 | - |
| `MiniMax-M3` | priority | > 512k input tokens | $0.90 | $3.60 | $0.18 | - |
| `MiniMax-M2.7` | standard | all | $0.30 | $1.20 | $0.06 | $0.375 |

```bash
export MINIMAX_API_KEY=your-api-key
uv run python run_bench.py \
--task simpleqa \
--model MiniMax-M3 \
--model-context-length 1000000 \
--num-examples 1
```

`run_bench.py` uses the global OpenAI-compatible endpoint by default. Set
`MINIMAX_API_BASE=https://api.minimaxi.com/v1` to use the China endpoint.

The official API entry points are:

| Region | OpenAI-compatible Base URL | Anthropic-compatible Base URL |
| --- | --- | --- |
| Global | `https://api.minimax.io/v1` | `https://api.minimax.io/anthropic` |
| China | `https://api.minimaxi.com/v1` | `https://api.minimaxi.com/anthropic` |

For Anthropic SDK integrations, pass the `/anthropic` Base URL unchanged. The SDK
appends `/v1/messages` when creating a message.
15 changes: 15 additions & 0 deletions eval/lib/model_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
from typing import Dict, Optional


MINIMAX_MODELS = {
"minimax-m3": "MiniMax-M3",
"minimax-m2.7": "MiniMax-M2.7",
}


def get_model_config(model_name: str) -> Dict[str, Optional[str]]:
"""
Get model configuration based on model name.
Expand All @@ -19,6 +25,15 @@ def get_model_config(model_name: str) -> Dict[str, Optional[str]]:
"""
model_lower = model_name.lower()

# MiniMax models (OpenAI-compatible API)
minimax_model = MINIMAX_MODELS.get(model_lower.rsplit("/", 1)[-1])
if minimax_model:
return {
"api_base": os.getenv("MINIMAX_API_BASE", "https://api.minimax.io/v1"),
"api_key": os.getenv("MINIMAX_API_KEY", os.getenv("API_KEY", "dummy")),
"model": minimax_model,
}

# Gemini models
if "gemini" in model_lower:
# Check for Vertex AI first
Expand Down
51 changes: 51 additions & 0 deletions tests/test_eval_model_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
from pathlib import Path
from runpy import run_path

import pytest


get_model_config = run_path(
str(Path(__file__).parents[1] / "eval" / "lib" / "model_config.py")
)["get_model_config"]


@pytest.mark.parametrize(
("model_name", "model_id"),
[
("MiniMax-M3", "MiniMax-M3"),
("MiniMax/MiniMax-M3", "MiniMax-M3"),
("MiniMax-M2.7", "MiniMax-M2.7"),
("MiniMax/MiniMax-M2.7", "MiniMax-M2.7"),
],
)
def test_minimax_model_config_uses_canonical_model_id(
monkeypatch, model_name, model_id
):
monkeypatch.delenv("MINIMAX_API_BASE", raising=False)
monkeypatch.delenv("MINIMAX_API_KEY", raising=False)
monkeypatch.setenv("API_KEY", "fallback-key")

assert get_model_config(model_name) == {
"api_base": "https://api.minimax.io/v1",
"api_key": "fallback-key",
"model": model_id,
}


def test_minimax_model_config_supports_endpoint_and_key_overrides(monkeypatch):
monkeypatch.setenv("MINIMAX_API_BASE", "https://api.minimaxi.com/v1")
monkeypatch.setenv("MINIMAX_API_KEY", "provider-key")

config = get_model_config("MiniMax-M3")

assert config["api_base"] == "https://api.minimaxi.com/v1"
assert config["api_key"] == "provider-key"


def test_minimax_model_config_does_not_match_unregistered_models(monkeypatch):
monkeypatch.setenv("API_BASE", "http://localhost:9000/v1")

config = get_model_config("MiniMax-M2.7-highspeed")

assert config["api_base"] == "http://localhost:9000/v1"
assert config["model"] == "MiniMax-M2.7-highspeed"
Loading