Update app.py

Try ONNX runtime at 8 bit quantization.

app.py

@@ -1,13 +1,27 @@
-import gradio as gr
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
+# import gradio as gr
+# import torch
+# from transformers import AutoTokenizer, AutoModelForCausalLM
 
-MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+# MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
+# DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+# print("Loading tokenizer & model…")
+# tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+# model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
+
+#########
 
 print("Loading tokenizer & model…")
+import gradio as gr
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForCausalLM
+
+MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
+model = ORTModelForCausalLM.from_pretrained(MODEL_ID, export=True, quantize=True)
+
+#########
+
 
 # -------------------------------------------------
 # Optional tool(s)