Qwen3.5-397B-A17B-nvfp4

pip install git+https://github.com/ml-explore/mlx-lm

from mlx_lm import load, generate

model, tokenizer = load("mlx-community/Qwen3.5-397B-A17B-nvfp4")

messages = [{"role": "user", "content": "Explain the Riemann hypothesis in simple terms."}]
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True)

response = generate(
    model,
    tokenizer,
    prompt=prompt,
    max_tokens=4096,
    verbose=True,
    temp=0.6,
    top_p=0.95,
)

from mlx_lm import load, generate

model, tokenizer = load("mlx-community/Qwen3.5-397B-A17B-nvfp4")

messages = [
    {"role": "user", "content": "How many r's are in the word 'strawberry'?"}
]
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True)

response = generate(
    model,
    tokenizer,
    prompt=prompt,
    max_tokens=8192,
    verbose=True,
    temp=0.6,
    top_p=0.95,
)

from mlx_lm import load, generate

model, tokenizer = load("mlx-community/Qwen3.5-397B-A17B-nvfp4")

messages = [
    {"role": "user", "content": "Write a haiku about machine learning."}
]
prompt = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    enable_thinking=False,
)

response = generate(
    model,
    tokenizer,
    prompt=prompt,
    max_tokens=2048,
    verbose=True,
    temp=0.7,
    top_p=0.8,
)

# Thinking mode (default)
mlx_lm.generate \
    --model mlx-community/Qwen3.5-397B-A17B-nvfp4 \
    --prompt "What are the key differences between TCP and UDP?" \
    --max-tokens 4096 \
    --temp 0.6 \
    --top-p 0.95

# Start a local chat server (OpenAI-compatible)
mlx_lm.server --model mlx-community/Qwen3.5-397B-A17B-nvfp4

mlx_lm.server --model mlx-community/Qwen3.5-397B-A17B-nvfp4 --port 8080

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="unused")

response = client.chat.completions.create(
    model="mlx-community/Qwen3.5-397B-A17B-nvfp4",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Write a Python function to find all prime numbers up to n using the Sieve of Eratosthenes."},
    ],
    max_tokens=4096,
    temperature=0.6,
    top_p=0.95,
)
print(response.choices[0].message.content)

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen3.5-397B-A17B-nvfp4",
    "messages": [{"role": "user", "content": "Hello!"}],
    "max_tokens": 512,
    "temperature": 0.6
  }'

@misc{qwen3.5,
    title  = {{Qwen3.5}: Towards Native Multimodal Agents},
    author = {{Qwen Team}},
    month  = {February},
    year   = {2026},
    url    = {https://qwen.ai/blog?id=qwen3.5}
}