Skip to content

Commit d8c6210

Browse files
juliendenizeywang96mickaelseznec
authored
Add Mistral Large 3 and Ministral 3 (#29757)
Signed-off-by: Julien Denize <[email protected]> Signed-off-by: Julien Denize <[email protected]> Signed-off-by: Mickael Seznec <[email protected]> Signed-off-by: Roger Wang <[email protected]> Co-authored-by: Roger Wang <[email protected]> Co-authored-by: Mickael Seznec <[email protected]>
1 parent 8bbcf8b commit d8c6210

File tree

16 files changed

+724
-30
lines changed

16 files changed

+724
-30
lines changed

‎docs/models/supported_models.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,8 @@ th {
417417
| `MiniCPMForCausalLM` | MiniCPM | `openbmb/MiniCPM-2B-sft-bf16`, `openbmb/MiniCPM-2B-dpo-bf16`, `openbmb/MiniCPM-S-1B-sft`, etc. | ✅︎ | ✅︎ |
418418
| `MiniCPM3ForCausalLM` | MiniCPM3 | `openbmb/MiniCPM3-4B`, etc. | ✅︎ | ✅︎ |
419419
| `MiniMaxM2ForCausalLM` | MiniMax-M2 |`MiniMaxAI/MiniMax-M2`, etc. | | ✅︎ |
420-
| `MistralForCausalLM` | Mistral, Mistral-Instruct | `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ |
420+
| `MistralForCausalLM` | Ministral-3, Mistral, Mistral-Instruct | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ |
421+
| `MistralLarge3ForCausalLM` | Mistral-Large-3-675B-Base-2512, Mistral-Large-3-675B-Instruct-2512 | `mistralai/Mistral-Large-3-675B-Base-2512`, `mistralai/Mistral-Large-3-675B-Instruct-2512`, etc. | ✅︎ | ✅︎ |
421422
| `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ |
422423
| `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ |
423424
| `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ |
@@ -711,7 +712,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
711712
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
712713
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
713714
| `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ |
714-
| `PixtralForConditionalGeneration` | Mistral 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Pixtral-12B-2409`, etc. | | ✅︎ |
715+
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
715716
| `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
716717
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
717718
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |

‎tests/models/registry.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,9 @@ def check_available_online(
358358
trust_remote_code=True,
359359
),
360360
"MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
361+
"MistralLarge3ForCausalLM": _HfExamplesInfo(
362+
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", is_available_online=False
363+
),
361364
"MixtralForCausalLM": _HfExamplesInfo(
362365
"mistralai/Mixtral-8x7B-Instruct-v0.1",
363366
{"tiny": "TitanML/tiny-mixtral"},
@@ -770,7 +773,13 @@ def check_available_online(
770773
),
771774
"PixtralForConditionalGeneration": _HfExamplesInfo(
772775
"mistralai/Pixtral-12B-2409",
776+
extras={
777+
"mistral-large-3": "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
778+
"ministral-3": "mistralai/Ministral-3-3B-Instruct-2512",
779+
},
773780
tokenizer_mode="mistral",
781+
# TODO: revert once Mistral-Large-3 and Ministral-3 are publicly available.
782+
is_available_online=False,
774783
),
775784
"QwenVLForConditionalGeneration": _HfExamplesInfo(
776785
"Qwen/Qwen-VL",
@@ -870,6 +879,11 @@ def check_available_online(
870879
use_original_num_layers=True,
871880
max_model_len=10240,
872881
),
882+
"EagleMistralLarge3ForCausalLM": _HfExamplesInfo(
883+
"mistralai/Mistral-Large-3-675B-Instruct-2512",
884+
speculative_model="mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle",
885+
is_available_online=False,
886+
),
873887
"LlamaForCausalLMEagle3": _HfExamplesInfo(
874888
"Qwen/Qwen3-8B",
875889
trust_remote_code=True,

‎tests/tokenizers_/test_mistral.py‎

Lines changed: 151 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,118 @@
9191
],
9292
),
9393
),
94+
(
95+
{
96+
"messages": [
97+
{
98+
"role": "user",
99+
"content": "What is the current local date and time?",
100+
}
101+
],
102+
"tools": [
103+
{
104+
"type": "function",
105+
"function": {
106+
"description": "Fetch the current local date and time.",
107+
"unsupported_field": False,
108+
"name": "get_current_time",
109+
"parameters": {},
110+
},
111+
},
112+
{
113+
"type": "function",
114+
"function": {
115+
"description": "Fetch the current local date and time.",
116+
"unsupported_field2": False,
117+
"name": "get_current_time",
118+
"parameters": {},
119+
},
120+
},
121+
],
122+
},
123+
(
124+
[
125+
{
126+
"role": "user",
127+
"content": "What is the current local date and time?",
128+
}
129+
],
130+
[
131+
{
132+
"type": "function",
133+
"function": {
134+
"description": "Fetch the current local date and time.",
135+
"name": "get_current_time",
136+
"parameters": {},
137+
},
138+
},
139+
{
140+
"type": "function",
141+
"function": {
142+
"description": "Fetch the current local date and time.",
143+
"name": "get_current_time",
144+
"parameters": {},
145+
},
146+
},
147+
],
148+
),
149+
),
150+
(
151+
{
152+
"messages": [
153+
{
154+
"role": "user",
155+
"content": "What is the current local date and time?",
156+
}
157+
],
158+
"tools": [
159+
{
160+
"type": "function",
161+
"unsupported_field": False,
162+
"function": {
163+
"description": "Fetch the current local date and time.",
164+
"name": "get_current_time",
165+
"parameters": {},
166+
},
167+
},
168+
{
169+
"type": "function",
170+
"unsupported_field2": False,
171+
"function": {
172+
"description": "Fetch the current local date and time 2.",
173+
"name": "get_current_time2",
174+
"parameters": {"a": "1"},
175+
},
176+
},
177+
],
178+
},
179+
(
180+
[
181+
{
182+
"role": "user",
183+
"content": "What is the current local date and time?",
184+
}
185+
],
186+
[
187+
{
188+
"type": "function",
189+
"function": {
190+
"description": "Fetch the current local date and time.",
191+
"name": "get_current_time",
192+
"parameters": {},
193+
},
194+
},
195+
{
196+
"type": "function",
197+
"function": {
198+
"description": "Fetch the current local date and time 2.",
199+
"name": "get_current_time2",
200+
"parameters": {"a": "1"},
201+
},
202+
},
203+
],
204+
),
205+
),
94206
],
95207
)
96208
def test_prepare_apply_chat_template_tools_and_messages(
@@ -1108,13 +1220,6 @@ def test_decode(
11081220
)
11091221
== expected_tokens[mistral_tokenizer.is_tekken]
11101222
)
1111-
assert (
1112-
mistral_tokenizer.decode(
1113-
ids[mistral_tokenizer.is_tekken],
1114-
skip_special_tokens=skip_special_tokens,
1115-
)
1116-
== expected_tokens[mistral_tokenizer.is_tekken]
1117-
)
11181223

11191224
def test_decode_empty(
11201225
self,
@@ -1140,6 +1245,45 @@ def test_decode_int(
11401245
== "<s>"
11411246
)
11421247

1248+
@pytest.mark.parametrize(
1249+
"skip_special_tokens,expected_tokens",
1250+
(
1251+
(
1252+
False,
1253+
(
1254+
["<s>[INST]▁Hello▁world▁![/INST]▁Hello</s>"],
1255+
["<s>[INST]Hello world ![/INST]Hello</s>"],
1256+
),
1257+
),
1258+
(True, (["Hello world ! Hello"], ["Hello world !Hello"])),
1259+
),
1260+
)
1261+
def test_batch_decode(
1262+
self,
1263+
mistral_tokenizer: MistralTokenizer,
1264+
skip_special_tokens: bool,
1265+
expected_tokens: tuple[str, str],
1266+
):
1267+
ids = (
1268+
[[1, 3, 23325, 2294, 1686, 4, 23325, 2]],
1269+
[[1, 3, 22177, 4304, 2662, 4, 22177, 2]],
1270+
)
1271+
assert (
1272+
mistral_tokenizer.batch_decode(
1273+
ids[mistral_tokenizer.is_tekken],
1274+
skip_special_tokens=skip_special_tokens,
1275+
)
1276+
== expected_tokens[mistral_tokenizer.is_tekken]
1277+
)
1278+
1279+
def test_batch_decode_empty(
1280+
self,
1281+
mistral_tokenizer: MistralTokenizer,
1282+
):
1283+
assert mistral_tokenizer.batch_decode(
1284+
[[]],
1285+
) == [""]
1286+
11431287
def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer):
11441288
tokens = (
11451289
[

‎vllm/config/speculative.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ def compute_hash(self) -> str:
167167

168168
@staticmethod
169169
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
170+
initial_architecture = hf_config.architectures[0]
170171
if hf_config.model_type in ("deepseek_v3", "deepseek_v32"):
171172
hf_config.model_type = "deepseek_mtp"
172173
if hf_config.model_type == "deepseek_mtp":
@@ -226,6 +227,9 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
226227
{"n_predict": n_predict, "architectures": ["LongCatFlashMTPModel"]}
227228
)
228229

230+
if initial_architecture == "MistralLarge3ForCausalLM":
231+
hf_config.update({"architectures": ["EagleMistralLarge3ForCausalLM"]})
232+
229233
return hf_config
230234

231235
def __post_init__(self):

‎vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ def __init__(self, tokenizer: TokenizerLike):
8080
self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
8181
if _is_fn_name_regex_support(self.model_tokenizer):
8282
self.fn_name_regex = re.compile(
83-
r"([a-zA-Z0-9_-]+)(\{[\s\S]*?\})(?=\s*$|,|\s)", re.DOTALL
83+
r"([a-zA-Z0-9_-]+)(\{[\s\S]*?\})(?=\s*$|,|\s)?", re.DOTALL
8484
)
8585
else:
8686
self.fn_name_regex = None

0 commit comments

Comments
 (0)