tipsv1-so400m14

pip install transformers torch torchvision sentencepiece scikit-learn requests

from transformers import AutoModel

model = AutoModel.from_pretrained("google/tipsv1-so400m14", trust_remote_code=True)
model.eval()

import requests
from PIL import Image
from torchvision import transforms

url = "https://huggingface.co/spaces/google/TIPSv2/resolve/main/examples/zeroseg/pascal_context_00049_image.png"
image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
transform = transforms.Compose([transforms.Resize((448, 448)), transforms.ToTensor()])
pixel_values = transform(image).unsqueeze(0)

out = model.encode_image(pixel_values)
print(out.cls_token.shape)     # (1, 1, 1152) — global image embedding
print(out.patch_tokens.shape)  # (1, 1024, 1152) — per-patch spatial features

text_emb = model.encode_text(["a photo of a bus", "a photo of a dog"])
print(text_emb.shape)  # (2, 1152) — one embedding per query

import torch.nn.functional as F

classes = ["bus", "car", "dog", "cat"]
cls = F.normalize(out.cls_token[:, 0, :], dim=-1)
text_emb = F.normalize(model.encode_text(classes), dim=-1)
similarity = cls @ text_emb.T
print(classes[similarity.argmax()])  # predicted class

import numpy as np
from sklearn.decomposition import PCA

feat = out.patch_tokens[0].detach().cpu().numpy()
rgb = PCA(n_components=3, whiten=True).fit_transform(feat).reshape(32, 32, 3)
rgb = 1 / (1 + np.exp(-2.0 * rgb))  # sigmoid for [0, 1] range with good contrast

@inproceedings{maninis2025tips,
  title     = {{TIPS: Text-Image Pretraining with Spatial Awareness}},
  author    = {Maninis, Kevis-Kokitsi and Chen, Kaifeng and Ghosh, Soham and Karpur, Arjun and Chen, Koert and Xia, Ye and Cao, Bingyi and Salz, Daniel and Han, Guangxing and Dlabal, Jan and Gnanapragasam, Dan and Seyedhosseini, Mojtaba and Zhou, Howard and Araujo, Andre},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year      = {2025},
  url       = {https://arxiv.org/abs/2410.16512}
}