Update alignment/data.py

alignment/data.py

@@ -18,6 +18,7 @@ import re
 from typing import List, Literal, Optional
 
 from datasets import DatasetDict, concatenate_datasets, load_dataset
+from googletrans import Translator
 
 from .configs import DataArguments
 
@@ -34,6 +35,14 @@ def apply_chat_template(
 
     if task in ["sft", "generation"]:
         messages = example["messages"]
+        print("Message Length: ", len(messages))
+        for i in range(len(messages)):
+            try:
+                translator = Translator()
+                messages[i]["content"] = translator.translate(messages[i]["content"], dest='hi').text
+            except Exception as e:
+                print(e, messages[i]["content"])
+                messages[i]["content"] = ""
         # We add an empty system message if there is none
         if messages[0]["role"] != "system":
             messages.insert(0, {"role": "system", "content": ""})