feat: remove pdfs and enable online pdf readings (microsoft#183)

* remove pdfs and enable online pdf readings

* update doc format

* use url as key

Author: peteryang1

rdagent/app/general_model/general_model.py

@@ -20,6 +20,22 @@
 def extract_models_and_implement(
     report_file_path: str,
 ) -> None:
+    """
+    Extracts models from a given PDF report file and implements the necessary operations.
+
+    Parameters:
+    report_file_path (str): The path to the report file. The file must be a PDF file.
+
+    Example URLs of PDF reports:
+    - https://arxiv.org/pdf/2210.09789
+    - https://arxiv.org/pdf/2305.10498
+    - https://arxiv.org/pdf/2110.14446
+    - https://arxiv.org/pdf/2205.12454
+    - https://arxiv.org/pdf/2210.16518
+
+    Returns:
+    None
+    """
     with logger.tag("init"):
         scenario = GeneralModelScenario()
         logger.log_object(scenario, tag="scenario")

rdagent/app/qlib_rd_loop/factor_from_report_w_sc.py

@@ -85,7 +85,7 @@ def extract_hypothesis_and_exp_from_reports(report_file_path: str) -> Tuple[Qlib
             pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
             logger.log_object(pdf_screenshot)
 
-    docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
+    docs_dict = load_and_process_pdfs_by_langchain(report_file_path)
 
     factor_result = {
         task.factor_name: {

rdagent/components/coder/model_coder/task_loader.py

@@ -115,23 +115,12 @@ def load(self, model_dict: dict) -> list:
 
 
 class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader):
-    def load(self, file_or_folder_path: Path) -> dict:
-        docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))  # dict{file_path:content}
+    def load(self, file_or_folder_path: str) -> dict:
+        docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path)  # dict{file_path:content}
         model_dict = extract_model_from_docs(
             docs_dict
         )  # dict{file_name: dict{model_name: dict{description, formulation, variables}}}
         model_dict = merge_file_to_model_dict_to_model_dict(
             model_dict
         )  # dict {model_name: dict{description, formulation, variables}}
         return ModelExperimentLoaderFromDict().load(model_dict)
-
-
-def main(path="../test_doc"):
-    doc_dict = load_and_process_pdfs_by_langchain(Path(path))
-    print(doc_dict.keys())  # if you run code like "python -u", the print content will be truncated
-
-
-import fire
-
-if __name__ == "__main__":
-    fire.Fire(main)

rdagent/components/document_reader/document_reader.py

@@ -6,7 +6,11 @@
 import fitz
 from azure.ai.formrecognizer import DocumentAnalysisClient
 from azure.core.credentials import AzureKeyCredential
-from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader
+from langchain.document_loaders import (
+    OnlinePDFLoader,
+    PyPDFDirectoryLoader,
+    PyPDFLoader,
+)
 from PIL import Image
 
 if TYPE_CHECKING:
@@ -15,7 +19,7 @@
 from rdagent.core.conf import RD_AGENT_SETTINGS
 
 
-def load_documents_by_langchain(path: Path) -> list:
+def load_documents_by_langchain(path: str) -> list:
     """Load documents from the specified path.
 
     Args:
@@ -24,7 +28,10 @@ def load_documents_by_langchain(path: Path) -> list:
     Returns:
         list: A list of loaded documents.
     """
-    loader = PyPDFDirectoryLoader(str(path), silent_errors=True) if path.is_dir() else PyPDFLoader(str(path))
+    if Path(path).is_dir():
+        loader = PyPDFDirectoryLoader(path, silent_errors=True)
+    else:
+        loader = PyPDFLoader(path)
     return loader.load()
 
 
@@ -41,7 +48,10 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     content_dict = {}
 
     for doc in docs:
-        doc_name = str(Path(doc.metadata["source"]).resolve())
+        if Path(doc.metadata["source"]).exists():
+            doc_name = str(Path(doc.metadata["source"]).resolve())
+        else:
+            doc_name = doc.metadata["source"]
         doc_content = doc.page_content
 
         if doc_name not in content_dict:
@@ -52,7 +62,7 @@ def process_documents_by_langchain(docs: list[Document]) -> dict[str, str]:
     return content_dict
 
 
-def load_and_process_pdfs_by_langchain(path: Path) -> dict[str, str]:
+def load_and_process_pdfs_by_langchain(path: str) -> dict[str, str]:
     return process_documents_by_langchain(load_documents_by_langchain(path))
 
 

rdagent/scenarios/general_model/test_doc1.pdf

@@ -509,9 +509,9 @@ def deduplicate_factors_by_llm(  # noqa: C901, PLR0912
 
 
 class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader):
-    def load(self, file_or_folder_path: Path) -> dict:
+    def load(self, file_or_folder_path: str) -> dict:
         with logger.tag("docs"):
-            docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))
+            docs_dict = load_and_process_pdfs_by_langchain(file_or_folder_path)
             logger.log_object(docs_dict)
 
         selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1)