openMaMMUT-ViT-B-32-512x512-pt_DFN2B-ft_DFN512x512-s293M-b73k

from PIL import Image
import requests
from transformers import CLIPProcessor, AutoModel, CLIPTokenizer

model_path = "laion/openMaMMUT-ViT-B-32-512x512-pt_DFN2B-ft_DFN512x512-s293M-b73k"
tokenizer = CLIPTokenizer.from_pretrained(model_path)

model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
processor = CLIPProcessor.from_pretrained(model_path)

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

# Image captioning
inputs = processor(images=image, return_tensors="pt", padding=True)
outputs = model.generate(pixel_values=inputs["pixel_values"], top_p=0.1, do_sample=True).sequences
decoded_outputs = tokenizer.batch_decode(outputs)
print("HuggingFace outputs:", decoded_outputs) # prints: ['<|startoftext|>cats on couch']

# Get image-text similarity (just like CLIP)
text  = ["a photo of a cat", "a photo of a dog"]
inputs = processor(images=image, text=text, return_tensors="pt", padding=True)

outputs = model(pixel_values=inputs["pixel_values"], input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], contrastive_only=True)

logits_per_image = outputs.logits_per_image # this is the image-text similarity score
probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities

print("Label probabilities:", probs)
print("Logits per image:", logits_per_image)

# Compute image and text embeddings separately

text_features = model.get_text_features(inputs["input_ids"], inputs["attention_mask"])
image_features = model.get_image_features(inputs["pixel_values"])

print("Text features shape:", text_features.shape)  # prints: [batch_size, feature_dim]
print("Image features shape:", image_features.shape)  # prints: [batch_size, feature_dim]
text_features /= text_features.norm(dim=-1, keepdim=True)
image_features /= image_features.norm(dim=-1, keepdim=True)
text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
print("Label probs:", text_probs)  # prints: [[1., 0.]] or similar, depending on the image and text

git clone https://github.com/LAION-AI/open_clip_mammut
cd open_clip_mammut
python -m pip install .

import torch
from PIL import Image
import open_clip

model, _, transform = open_clip.create_model_and_transforms('openMaMMUT-ViT-B-32-512x512-pt_DFN2B-ft_DFN512x512-s293M-b73k')
model.eval()  # model in train mode by default, impacts some models with BatchNorm or stochastic depth active
tokenizer = open_clip.get_tokenizer('hf-hub:laion/openMaMMUT-ViT-B-32-512x512-pt_DFN2B-ft_DFN512x512-s293M-b73k')

image = transform(Image.open("docs/CLIP.png")).unsqueeze(0)
text = tokenizer(["a diagram", "a dog", "a cat"])

with torch.no_grad(), torch.amp.autocast('cuda'):
    image_features = model.encode_image(image)
    text_features = model.encode_text(text)
    image_features /= image_features.norm(dim=-1, keepdim=True)
    text_features /= text_features.norm(dim=-1, keepdim=True)

    text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)

print("Label probs:", text_probs)  # prints: [[1., 0., 0.]]

import open_clip
import torch
from PIL import Image

model, _, transform = open_clip.create_model_and_transforms('hf-hub:laion/openMaMMUT-ViT-B-32-512x512-pt_DFN2B-ft_DFN512x512-s293M-b73k')

im = Image.open("docs/CLIP.png").convert("RGB")
im = transform(im).unsqueeze(0)

with torch.no_grad(), torch.amp.autocast('cuda'):
  generated = model.generate(im)

print(open_clip.decode(generated[0]).split("<end_of_text>")[0].replace("<start_of_text>", ""))

@article{nezhurina2025scaling,
  title={Scaling Laws for Robust Comparison of Open Foundation Language-Vision Models and Datasets},
  author={Marianna Nezhurina, Tomer Porian, Giovanni Pucceti, Tommie Kerssies, Romain Beaumont, Mehdi Cherti, Jenia Jitsev},
  journal={arXiv:2506.04598},
  url={https://arxiv.org/abs/2506.04598},
  year={2025}
}

@article{gadre2023datacomp,
  title={Datacomp: In search of the next generation of multimodal datasets},
  author={Gadre, Samir Yitzhak and Ilharco, Gabriel and Fang, Alex and Hayase, Jonathan and Smyrnis, Georgios and Nguyen, Thao and Marten, Ryan and Wortsman, Mitchell and Ghosh, Dhruba and Zhang, Jieyu and others},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={27092--27112},
  year={2023}
}

@article{
kuo2023mammut,
title={Ma{MMUT}: A Simple Architecture for Joint Learning for MultiModal Tasks},
author={Weicheng Kuo and AJ Piergiovanni and Dahun Kim and xiyang luo and Benjamin Caine and Wei Li and Abhijit Ogale and Luowei Zhou and Andrew M. Dai and Zhifeng Chen and Claire Cui and Anelia Angelova},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2023},
url={https://openreview.net/forum?id=FqOG4osY7C},
}

@inproceedings{Cherti2023,
  title={Reproducible scaling laws for contrastive language-image learning},
  author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={2818--2829},
  year={2023}
}

@software{ilharco_gabriel_2021_5143773,
  author       = {Ilharco, Gabriel and
                  Wortsman, Mitchell and
                  Wightman, Ross and
                  Gordon, Cade and
                  Carlini, Nicholas and
                  Taori, Rohan and
                  Dave, Achal and
                  Shankar, Vaishaal and
                  Namkoong, Hongseok and
                  Miller, John and
                  Hajishirzi, Hannaneh and
                  Farhadi, Ali and
                  Schmidt, Ludwig},
  title        = {OpenCLIP},
  month        = jul,
  year         = 2021,
  note         = {If you use this software, please cite it as below.},
  publisher    = {Zenodo},
  version      = {0.1},
  doi          = {10.5281/zenodo.5143773},
  url          = {https://doi.org/10.5281/zenodo.5143773}
}

@software{cherti_2025_15403103,
  author       = {Cherti, Mehdi and
                  Beaumont, Romain},
  title        = {CLIP benchmark},
  month        = may,
  year         = 2025,
  publisher    = {Zenodo},
  doi          = {10.5281/zenodo.15403103},
  url          = {https://doi.org/10.5281/zenodo.15403103},
  swhid        = {swh:1:dir:8cf49a5dd06f59224844a1e767337a1d14ee56c2
                   ;origin=https://doi.org/10.5281/zenodo.15403102;vi
                   sit=swh:1:snp:dd153b26f702d614346bf814f723d59fef3d
                   77a2;anchor=swh:1:rel:cff2aeb98f42583b44fdab5374e9
                   fa71793f2cff;path=CLIP\\_benchmark-main
                  },
}