gme-Qwen2-VL-7B-Instruct

from transformers import AutoModel
from transformers.utils.versions import require_version


require_version(
    "transformers<4.52.0",
    "The remote code has some issues with transformers>=4.52.0, please downgrade: pip install transformers==4.51.3"
)


t2i_prompt = 'Find an image that matches the given text.'
texts = [
    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
    "Alibaba office.",
]
images = [
    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
]


gme = AutoModel.from_pretrained(
    "Alibaba-NLP/gme-Qwen2-VL-7B-Instruct",
    torch_dtype="float16", device_map='cuda', trust_remote_code=True
)


# Single-modal embedding
e_text = gme.get_text_embeddings(texts=texts)
e_image = gme.get_image_embeddings(images=images)
print('Single-modal', (e_text @ e_image.T).tolist())
## Single-modal [[0.279296875, 0.0002658367156982422], [0.06427001953125, 0.304443359375]]

# How to set embedding instruction
e_query = gme.get_text_embeddings(texts=texts, instruction=t2i_prompt)
# If is_query=False, we always use the default instruction.
e_corpus = gme.get_image_embeddings(images=images, is_query=False)
print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
## Single-modal with instruction [[0.32861328125, 0.026336669921875], [0.09466552734375, 0.3134765625]]

# Fused-modal embedding
e_fused = gme.get_fused_embeddings(texts=texts, images=images)
print('Fused-modal', (e_fused @ e_fused.T).tolist())
## Fused-modal [[1.0, 0.0308685302734375], [0.0308685302734375, 1.0]]

from sentence_transformers import SentenceTransformer


t2i_prompt = 'Find an image that matches the given text.'
texts = [
    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
    "Alibaba office.",
]
images = [
    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
]


gme_st = SentenceTransformer("Alibaba-NLP/gme-Qwen2-VL-7B-Instruct")

# Single-modal embedding
e_text = gme_st.encode(texts, convert_to_tensor=True)
e_image = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
print('Single-modal', (e_text @ e_image.T).tolist())
## Single-modal [[0.27880859375, 0.0005745887756347656], [0.06500244140625, 0.306640625]]

# How to set embedding instruction
e_query = gme_st.encode([dict(text=t, prompt=t2i_prompt) for t in texts], convert_to_tensor=True)
# If no prompt, we always use the default instruction.
e_corpus = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
## Single-modal with instruction [[0.328369140625, 0.0269927978515625], [0.09521484375, 0.316162109375]]

# Fused-modal embedding
e_fused = gme_st.encode([dict(text=t, image=i) for t, i in zip(texts, images)], convert_to_tensor=True)
print('Fused-modal', (e_fused @ e_fused.T).tolist())
## Fused-modal [[0.99951171875, 0.0311737060546875], [0.0311737060546875, 1.0009765625]]

pip install ms-swift -U

# MAX_PIXELS settings to reduce memory usage
# check: https://swift.readthedocs.io/en/latest/BestPractices/Embedding.html
nproc_per_node=8
MAX_PIXELS=1003520 \
USE_HF=1 \
NPROC_PER_NODE=$nproc_per_node \
swift sft \
    --model Alibaba-NLP/gme-Qwen2-VL-7B-Instruct \
    --train_type lora \
    --dataset 'HuggingFaceM4/TextCaps:emb' \
    --torch_dtype bfloat16 \
    --num_train_epochs 1 \
    --per_device_train_batch_size 2 \
    --per_device_eval_batch_size 2 \
    --gradient_accumulation_steps $(expr 64 / $nproc_per_node) \
    --eval_steps 100 \
    --save_steps 100 \
    --eval_strategy steps \
    --save_total_limit 5 \
    --logging_steps 5 \
    --output_dir output \
    --lazy_tokenize true \
    --warmup_ratio 0.05 \
    --learning_rate 5e-6 \
    --deepspeed zero3 \
    --dataloader_num_workers 4 \
    --task_type embedding \
    --loss_type infonce \
    --dataloader_drop_last true

@misc{zhang2024gme,
      title={GME: Improving Universal Multimodal Retrieval by Multimodal LLMs}, 
      author={Zhang, Xin and Zhang, Yanzhao and Xie, Wen and Li, Mingxin and Dai, Ziqi and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Li, Wenjie and Zhang, Min},
      year={2024},
      eprint={2412.16855},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={http://arxiv.org/abs/2412.16855}, 
}