add endpoints to extract text

api/docs.py

@@ -648,3 +648,20 @@ async def extract_requirements_from_docs(req: ExtractRequirementsRequest, llm_ro
         yield progress_update(ProgressUpdate(status="complete", data=final_response.model_dump(), total_docs=n_docs, processed_docs=n_processed))
 
     return StreamingResponse(_stream_generator(process_futures), media_type="text/event-stream")
+
+# ======================================================================================================================================================================================
+
+@router.post("/extract_text_from_url")
+async def extract_text_from_url(url: str, http_client: AsyncClient = Depends(get_http_client)) -> dict:
+    """Extract text from a given document URL and return the text content."""
+
+    logging.info(f"Extracting text from URL: {url}")
+
+    try:
+        filename, ext, bytes = await get_doc_archive(url, http_client)
+        text_lines = await extract_text_contents(filename, ext, bytes)
+        content = "\n".join(text_lines)
+        return {"document": filename, "content": content}
+    except Exception as e:
+        logging.error(f"Failed to extract text from URL '{url}': {e}")
+        raise HTTPException(status_code=500, detail=f"Text extraction failed: {e}")