Molmo2 is a family of open vision-language models developed by the Allen Institute for AI (Ai2) that support image, video and multi-image understanding and grounding. Molmo2 models are trained on publicly available third party datasets as referenced in our technical report and Molmo2 data, a collection of datasets with highly-curated image-text and video-text pairs. It has state-of-the-art performance among multimodal models with a similar size. You can find all models in the Molmo2 family here.
Molmo2-O-7B is based on Olmo3-7B-Instruct and uses SigLIP 2 as vision backbone. It outperforms others in the class of open weight and data models on short videos, counting, and captioning, and is competitive on long-videos.
Ai2 is commited to open science. The Molmo2 datasets are available here. All other artifacts used in creating Molmo2 (training code, evaluations, intermediate checkpoints) will be made available at a later date, furthering our commitment to open-source AI development and reproducibility.
from transformers import AutoProcessor, AutoModelForImageTextToText
import torch
model_id="allenai/Molmo2-O-7B"
# load the processor
processor = AutoProcessor.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
# load the model
model = AutoModelForImageTextToText.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
# process the video and text
messages = [
{
"role": "user",
"content": [
dict(type="text", text="Which animal appears in the video?"),
dict(type="video", video="https://storage.googleapis.com/oe-training-public/demo_videos/many_penguins.mp4"),
],
}
]
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_tensors="pt",
return_dict=True,
)
inputs = {k: v.to(model.device) for k, v in inputs.items()}
# generate output
with torch.inference_mode():
generated_ids = model.generate(**inputs, max_new_tokens=2048)
# only get generated tokens; decode them to text
generated_tokens = generated_ids[0, inputs['input_ids'].size(1):]
generated_text = processor.tokenizer.decode(generated_tokens, skip_special_tokens=True)
# print the generated text
print(generated_text)
from transformers import AutoProcessor, AutoModelForImageTextToText
import torch
from molmo_utils import process_vision_info
import re
model_id="allenai/Molmo2-O-7B"
# load the processor
processor = AutoProcessor.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
# load the model
model = AutoModelForImageTextToText.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
COORD_REGEX = re.compile(rf"<(?:points|tracks).*? coords=\"([0-9\t:;, .]+)\"/?>")
FRAME_REGEX = re.compile(rf"(?:^|\t|:|,|;)([0-9\.]+) ([0-9\. ]+)")
POINTS_REGEX = re.compile(r"([0-9]+) ([0-9]{3,4}) ([0-9]{3,4})")
def _points_from_num_str(text, image_w, image_h, extract_ids=False):
all_points = []
for points in POINTS_REGEX.finditer(text):
ix, x, y = points.group(1), points.group(2), points.group(3)
# our points format assume coordinates are scaled by 1000
x, y = float(x)/1000*image_w, float(y)/1000*image_h
if 0 <= x <= image_w and 0 <= y <= image_h:
yield ix, x, y
def extract_video_points(text, image_w, image_h, extract_ids=False):
"""Extract video pointing coordinates as a flattened list of (t, x, y) triplets from model output text."""
all_points = []
for coord in COORD_REGEX.finditer(text):
for point_grp in FRAME_REGEX.finditer(coord.group(1)):
frame_id = float(point_grp.group(1))
w, h = (image_w, image_h)
for idx, x, y in _points_from_num_str(point_grp.group(2), w, h):
if extract_ids:
all_points.append((frame_id, idx, x, y))
else:
all_points.append((frame_id, x, y))
return all_points
messages = [
{
"role": "user",
"content": [
dict(type="text", text="Point to the penguins."),
dict(type="video", video="https://storage.googleapis.com/oe-training-public/demo_videos/many_penguins.mp4"),
],
}
]
# process the video using `molmo_utils.process_vision_info`
_, videos, video_kwargs = process_vision_info(messages)
videos, video_metadatas = zip(*videos)
videos, video_metadatas = list(videos), list(video_metadatas)
# apply the chat template to the input messages
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
# process the video and text
inputs = processor(
videos=videos,
video_metadata=video_metadatas,
text=text,
padding=True,
return_tensors="pt",
**video_kwargs,
)
inputs = {k: v.to(model.device) for k, v in inputs.items()}
# generate output
with torch.inference_mode():
generated_ids = model.generate(**inputs, max_new_tokens=2048)
# only get generated tokens; decode them to text
generated_tokens = generated_ids[0, inputs['input_ids'].size(1):]
generated_text = processor.tokenizer.decode(generated_tokens, skip_special_tokens=True)
# decode video pointing outputs
points = extract_video_points(generated_text, image_w=video_metadatas[0]["width"], image_h=video_metadatas[0]["height"])
print(points)
from transformers import AutoProcessor, AutoModelForImageTextToText
import torch
from molmo_utils import process_vision_info
import re
model_id="allenai/Molmo2-O-7B"
# load the processor
processor = AutoProcessor.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
# load the model
model = AutoModelForImageTextToText.from_pretrained(
model_id,
trust_remote_code=True,
dtype="auto",
device_map="auto"
)
COORD_REGEX = re.compile(rf"<(?:points|tracks).*? coords=\"([0-9\t:;, .]+)\"/?>")
FRAME_REGEX = re.compile(rf"(?:^|\t|:|,|;)([0-9\.]+) ([0-9\. ]+)")
POINTS_REGEX = re.compile(r"([0-9]+) ([0-9]{3,4}) ([0-9]{3,4})")
def _points_from_num_str(text, image_w, image_h, extract_ids=False):
all_points = []
for points in POINTS_REGEX.finditer(text):
ix, x, y = points.group(1), points.group(2), points.group(3)
# our points format assume coordinates are scaled by 1000
x, y = float(x)/1000*image_w, float(y)/1000*image_h
if 0 <= x <= image_w and 0 <= y <= image_h:
yield ix, x, y
def extract_video_points(text, image_w, image_h, extract_ids=False):
"""Extract video pointing coordinates as a flattened list of (t, x, y) triplets from model output text."""
all_points = []
for coord in COORD_REGEX.finditer(text):
for point_grp in FRAME_REGEX.finditer(coord.group(1)):
frame_id = float(point_grp.group(1))
w, h = (image_w, image_h)
for idx, x, y in _points_from_num_str(point_grp.group(2), w, h):
if extract_ids:
all_points.append((frame_id, idx, x, y))
else:
all_points.append((frame_id, x, y))
return all_points
# use higher max fps for tracking
messages = [
{
"role": "user",
"content": [
dict(type="text", text="Track the player who is dunking"),
dict(type="video", video="https://storage.googleapis.com/oe-training-public/demo_videos/arena_basketball.mp4", max_fps=8),
],
}
]
# process the video using `molmo_utils.process_vision_info`
_, videos, video_kwargs = process_vision_info(messages)
videos, video_metadatas = zip(*videos)
videos, video_metadatas = list(videos), list(video_metadatas)
# apply the chat template to the input messages
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
# process the video and text
inputs = processor(
videos=videos,
video_metadata=video_metadatas,
text=text,
padding=True,
return_tensors="pt",
**video_kwargs,
)
inputs = {k: v.to(model.device) for k, v in inputs.items()}
# generate output
with torch.inference_mode():
generated_ids = model.generate(**inputs, max_new_tokens=2048)
# only get generated tokens; decode them to text
generated_tokens = generated_ids[0, inputs['input_ids'].size(1):]
generated_text = processor.tokenizer.decode(generated_tokens, skip_special_tokens=True)
# decode video pointing outputs
points = extract_video_points(generated_text, image_w=video_metadatas[0]["width"], image_h=video_metadatas[0]["height"])
print(points)
This model is licensed under Apache 2.0. It is intended for research and educational use in accordance with Ai2’s Responsible Use Guidelines. This model is trained on third party datasets that are subject to academic and non-commercial research use only. Please review the sources to determine if this model is appropriate for your use case.
Files
28 files · 31.1 GBWeights 31.0 GB (safetensors). Sizes and sha256 from the signed manifest.
allenai <huggingface.co/allenai>Newsgroups: alt.freeweights.olmoSubject: 
