Update pipeline.py

pipeline.py

@@ -459,22 +459,32 @@ def pipeline_with_gemini(accessions):
       chunks_filename = "document_chunks.json"
       lookup_filename = "structured_lookup.json"
       print("name of faiss: ", faiss_filename)  
-      
-      # Save in temporary local directory
-      faiss_index_path = os.path.join(tmp_dir, faiss_filename)
-      document_chunks_path = os.path.join(tmp_dir, chunks_filename)
-      structured_lookup_path = os.path.join(tmp_dir, lookup_filename)
+          
+      faiss_index_path = os.path.join(LOCAL_TEMP_DIR, faiss_filename)
+      document_chunks_path = os.path.join(LOCAL_TEMP_DIR, chunks_filename)
+      structured_lookup_path = os.path.join(LOCAL_TEMP_DIR, lookup_filename)
       print("name if faiss path: ", faiss_index_path)  
       # 🔥 Remove the local file first if it exists
-      if os.path.exists(faiss_index_path):
-        os.remove(faiss_index_path)
-      if os.path.exists(document_chunks_path):
-        os.remove(document_chunks_path)
-      if os.path.exists(structured_lookup_path):
-        os.remove(structured_lookup_path)    
-      download_file_from_drive(faiss_filename, sample_folder_id, faiss_index_path)
-      download_file_from_drive(chunks_filename, sample_folder_id, document_chunks_path)
-      download_file_from_drive(lookup_filename, sample_folder_id, structured_lookup_path)
+      faiss_id = find_drive_file(faiss_filename, sample_folder_id)
+      document_id = find_drive_file(chunks_filename, sample_folder_id)
+      structure_id = find_drive_file(lookup_filename, sample_folder_id)  
+      if faiss_id and document_id and structure_id:
+        print("✅ 3 Files already exist in Google Drive. Downloading them...")
+        download_file_from_drive(faiss_filename, sample_folder_id, faiss_index_path)
+        download_file_from_drive(chunks_filename, sample_folder_id, document_chunks_path)
+        download_file_from_drive(lookup_filename, sample_folder_id, structured_lookup_path)  
+        # Read and parse these into `chunk` and `all_output`
+      else:
+        if os.path.exists(faiss_index_path):
+            os.remove(faiss_index_path)
+        if os.path.exists(document_chunks_path):
+            os.remove(document_chunks_path)
+        if os.path.exists(structured_lookup_path):
+            os.remove(structured_lookup_path)    
+        download_file_from_drive(faiss_filename, sample_folder_id, faiss_index_path)
+        download_file_from_drive(chunks_filename, sample_folder_id, document_chunks_path)
+        download_file_from_drive(lookup_filename, sample_folder_id, structured_lookup_path)
+      
       print("move to load rag")
       master_structured_lookup, faiss_index, document_chunks = model.load_rag_assets(
           faiss_index_path, document_chunks_path, structured_lookup_path