Update README.md

README.md

@@ -3691,44 +3691,96 @@ The `GME` models support three types of input: **text**, **image**, and **image-
 |[`gme-Qwen2-VL-2B`](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct) | 2.21B | 32768 | 1536 | 65.27 | 66.92 | 64.45 |
 |[`gme-Qwen2-VL-7B`](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct) | 8.29B | 32768 | 3584 | 67.48 | 69.73 | 67.44 |
 
-## Usage
-### Transformers
+## Usage 
+
+
+**Transformers**
+
 ```python
-from transformers import AutoModel
+t2i_prompt = 'Find an image that matches the given text.'
 texts = [
-    "What kind of car is this?",
-    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023."
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
 ]
 images = [
-    'https://en.wikipedia.org/wiki/File:Tesla_Cybertruck_damaged_window.jpg',
-    'https://en.wikipedia.org/wiki/File:2024_Tesla_Cybertruck_Foundation_Series,_front_left_(Greenwich).jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
 ]
 
-gme = AutoModel("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", trust_remote_code=True)
+
+gme = AutoModel.from_pretrained(
+    model_path,
+    torch_dtype="float16", device_map='cuda', trust_remote_code=True
+)
+
 
 # Single-modal embedding
 e_text = gme.get_text_embeddings(texts=texts)
 e_image = gme.get_image_embeddings(images=images)
-print((e_text * e_image).sum(-1))
-## tensor([0.2281, 0.6001], dtype=torch.float16)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## Single-modal [[0.359619140625, 0.0655517578125], [0.04180908203125, 0.374755859375]]
 
 # How to set embedding instruction
-e_query = gme.get_text_embeddings(texts=texts, instruction='Find an image that matches the given text.')
+e_query = gme.get_text_embeddings(texts=texts, instruction=t2i_prompt)
 # If is_query=False, we always use the default instruction.
 e_corpus = gme.get_image_embeddings(images=images, is_query=False)
-print((e_query * e_corpus).sum(-1))
-## tensor([0.2433, 0.7051], dtype=torch.float16)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## Single-modal with instruction [[0.429931640625, 0.11505126953125], [0.049835205078125, 0.409423828125]]
 
 # Fused-modal embedding
 e_fused = gme.get_fused_embeddings(texts=texts, images=images)
-print((e_fused[0] * e_fused[1]).sum())
-## tensor(0.6108, dtype=torch.float16)
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## Fused-modal [[1.0, 0.05511474609375], [0.05511474609375, 1.0]]
+```
+
+
+**sentence_transformers**
+
+The `encode` function accept `str` or `dict` with key(s) in `{'text', 'image', 'prompt'}`.
+
+**Do not pass `prompt` as the argument to `encode`**, pass as the input as a `dict` with a `prompt` key.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+
+t2i_prompt = 'Find an image that matches the given text.'
+texts = [
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
+]
+images = [
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
+]
 
+
+gme_st = SentenceTransformer("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct")
+
+# Single-modal embedding
+e_text = gme_st.encode(texts, convert_to_tensor=True)
+e_image = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## Single-modal [[0.356201171875, 0.06536865234375], [0.041717529296875, 0.37890625]]
+
+# How to set embedding instruction
+e_query = gme_st.encode([dict(text=t, prompt=t2i_prompt) for t in texts], convert_to_tensor=True)
+# If no prompt, we always use the default instruction.
+e_corpus = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## Single-modal with instruction [[0.425537109375, 0.1158447265625], [0.049835205078125, 0.413818359375]]
+
+# Fused-modal embedding
+e_fused = gme_st.encode([dict(text=t, image=i) for t, i in zip(texts, images)], convert_to_tensor=True)
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## Fused-modal [[0.99951171875, 0.0556640625], [0.0556640625, 0.99951171875]]
 ```
 
+
+
 ## Evaluation
 
-We validated the performance on our universal multimodal retrieval benchmark (**UMRB**) among others.
+We validated the performance on our universal multimodal retrieval benchmark (**UMRB**, see [Release UMRB](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct/discussions/2)) among others.
 
 |                    |      | Single-modal |           | Cross-modal |             |           | Fused-modal |            |            |             |  Avg.      |
 |--------------------|------|:------------:|:---------:|:-----------:|:-----------:|:---------:|:-----------:|:----------:|:----------:|:-----------:|:----------:|
@@ -3745,6 +3797,48 @@ The [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) English t
 
 **More detailed experimental results can be found in the [paper](http://arxiv.org/abs/2412.16855)**.
 
+## Community support
+
+### Fine-tuning
+
+GME models can be fine-tuned by SWIFT：
+
+```shell
+pip install ms-swift -U
+```
+
+```shell
+# MAX_PIXELS settings to reduce memory usage
+# check: https://swift.readthedocs.io/en/latest/BestPractices/Embedding.html
+nproc_per_node=8
+MAX_PIXELS=1003520 \
+USE_HF=1 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Alibaba-NLP/gme-Qwen2-VL-2B-Instruct \
+    --train_type lora \
+    --dataset 'HuggingFaceM4/TextCaps:emb' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps $(expr 64 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --eval_strategy steps \
+    --save_total_limit 5 \
+    --logging_steps 5 \
+    --output_dir output \
+    --lazy_tokenize true \
+    --warmup_ratio 0.05 \
+    --learning_rate 5e-6 \
+    --deepspeed zero3 \
+    --dataloader_num_workers 4 \
+    --task_type embedding \
+    --loss_type infonce \
+    --dataloader_drop_last true
+```
+
 
 ## Limitations
 
@@ -3790,4 +3884,4 @@ If you find our paper or models helpful, please consider cite:
       primaryClass={cs.CL},
       url={http://arxiv.org/abs/2412.16855}, 
 }
-```
+```