import torch
from sentence_transformers import SentenceTransformer
model = SentenceTransformer(
"LCO-Embedding/LCO-Embedding-Omni-7B",
model_kwargs={
"torch_dtype": torch.bfloat16,
"attn_implementation": "flash_attention_2", # pip install kernels; recommended but not mandatory
},
)
The same "Summarize the above in one word:" instruction used in the paper is baked into the chat template, so encode() takes plain text, file paths, URLs, or multimodal dicts directly.
query = "What is the tallest mountain in the world?"
documents = [
"Mount Everest is Earth's highest mountain above sea level, located in the Mahalangur Himal sub-range of the Himalayas. Its elevation of 8,848.86 metres was established by a joint Chinese-Nepali survey in 2020.",
"K2, at 8,611 metres above sea level, is the second-highest mountain on Earth, after Mount Everest. It lies in the Karakoram range on the China-Pakistan border.",
"Mount Kilimanjaro is a dormant volcano in Tanzania. It is the highest mountain in Africa, with its summit about 5,895 metres above sea level.",
]
query_embedding = model.encode(query)
document_embeddings = model.encode(documents)
print(model.similarity(query_embedding, document_embeddings))
# tensor([[0.6456, 0.4331, 0.4788]])
To embed a document that combines multiple modalities, pass a dict with any combination of "text", "image", "audio", and "video" keys instead of a single path or string:
from transformers import Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor
from qwen_omni_utils import process_mm_info
processor = Qwen2_5OmniProcessor.from_pretrained("LCO-Embedding/LCO-Embedding-Omni-7B") # or add a `max_pixels = 1280*28*28' for efficient encoding
model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained("LCO-Embedding/LCO-Embedding-Omni-7B",
torch_dtype=torch.bfloat16,
device_map="auto")
texts = ["some random text", "a second random text", "a third random text"] * 30
batch_size = 8
text_prompt = "{}\nSummarize the above text in one word:"
all_text_embeddings = []
with torch.no_grad():
for i in tqdm(range(0, len(texts), batch_size)):
batch_texts = texts[i : i + batch_size]
batch_texts = [text_prompt.format(text) for text in batch_texts]
messages = [[
{
"role": "user",
"content": [
{"type": "text", "text":text},
],
}
] for text in batch_texts]
text_inputs = processor.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
text_inputs = processor(
text = text_inputs,
padding = True,
return_tensors = "pt",
)
text_inputs = text_inputs.to("cuda")
text_outputs = model(
**text_inputs, output_hidden_states=True, return_dict=True
).hidden_states[-1][:, -1, :]
all_text_embeddings.append(text_outputs.to(torch.float16).cpu())
all_text_embeddings = torch.cat(all_text_embeddings, dim=0)
We introduce LCO-Embedding, a language-centric omnimodal representation learning method and the LCO-Embedding model families, setting a new state-of-the-art on MIEB (Massive Image Embedding Benchmark), while supporting audio and videos.
This work also introduces the Generation-Representation Scaling Law, connecting models' generative capabilities and their representation upper bound. Furthermore, we introduce SeaDoc, a challenging visual document retrieval task in Southeast Asian languages, and show that continual generative pretraining before contrastive learning raises the representation upper bound.
We evaluate LCO-Embedding with state-of-the-art embedding models, including E5-V, Voyage Multimodal 3, mmE5, and GME, on a MIEB-Lite benchmark (51 tasks) broken down by task categories.
import os
os.environ["HF_ENDPOINT"] = "https://freeweights.allternit.com"
from transformers import AutoModel, AutoTokenizer
tok = AutoTokenizer.from_pretrained("LCO-Embedding/LCO-Embedding-Omni-7B", revision="108f6f1a5de3b2eedd1d1c7b7005aaca6ed3802c")
model = AutoModel.from_pretrained("LCO-Embedding/LCO-Embedding-Omni-7B", revision="108f6f1a5de3b2eedd1d1c7b7005aaca6ed3802c", device_map="auto")
sentence-transformersEmbed text with sentence-transformers.
import os
os.environ["HF_ENDPOINT"] = "https://freeweights.allternit.com"
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("LCO-Embedding/LCO-Embedding-Omni-7B", revision="108f6f1a5de3b2eedd1d1c7b7005aaca6ed3802c")
vectors = model.encode(["Hello"])
unsloth 
mlx-community