Update app.py

Nested generation in inference mode.

app.py

@@ -69,18 +69,18 @@ def chat_fn(history, enable_thinking, temperature, top_p, top_k, repetition_pena
         # xml_tools=TOOLS
     )
     inputs = tokenizer(text, return_tensors="pt").to(DEVICE)
-
-    streamer = model.generate(
-        **inputs,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        repetition_penalty=repetition_penalty,
-        pad_token_id=tokenizer.eos_token_id,
-        streamer=None          # we'll yield manually
-    )
+    with torch.inference_mode():
+        streamer = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=True,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            repetition_penalty=repetition_penalty,
+            pad_token_id=tokenizer.eos_token_id,
+            streamer=None          # we'll yield manually
+        )
     output_ids = streamer[0][len(inputs.input_ids[0]):]
     response = tokenizer.decode(output_ids, skip_special_tokens=True)
     if isinstance(response, str):