feat: finished basic version of deep scraper

Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com>

examples/openai/depth_search_graph_openai.py

@@ -1,22 +1,28 @@
 """
 depth_search_graph_opeani example
 """
+import os
+from dotenv import load_dotenv
 from scrapegraphai.graphs import DepthSearchGraph
 
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
 graph_config = {
     "llm": {
-        "api_key":"YOUR_API_KEY",
+        "api_key": openai_key,
         "model": "openai/gpt-4o-mini",
     },
     "verbose": True,
     "headless": False,
     "depth": 2,
-    "only_inside_links": True,
+    "only_inside_links": False,
 }
 
 search_graph = DepthSearchGraph(
     prompt="List me all the projects with their description",
-    source="https://perinim.github.io/projects/",
+    source="https://perinim.github.io",
     config=graph_config
 )
 

pyproject.toml

@@ -31,7 +31,9 @@ dependencies = [
     "google>=3.0.0",
     "langchain-ollama>=0.1.3",
     "semchunk==2.2.0",
-    "transformers==4.44.2"
+    "transformers==4.44.2",
+    "qdrant-client>=1.11.3",
+    "fastembed>=0.3.6"
 ]
 
 license = "MIT"
@@ -99,7 +101,7 @@ screenshot_scraper = [
     "pillow>=10.4.0",
 ]
 
-# Group 5: Faiss CPU
+# Group 5: qdrant
 qdrant = [
     "qdrant-client>=1.11.3",
     "fastembed>=0.3.6"

requirements-dev.lock

@@ -64,6 +64,8 @@ click==8.1.7
     # via burr
     # via streamlit
     # via uvicorn
+coloredlogs==15.0.1
+    # via onnxruntime
 contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
@@ -84,9 +86,13 @@ fastapi==0.112.0
     # via burr
 fastapi-pagination==0.12.26
     # via burr
+fastembed==0.3.6
+    # via scrapegraphai
 filelock==3.15.4
     # via huggingface-hub
     # via transformers
+flatbuffers==24.3.25
+    # via onnxruntime
 fonttools==4.53.1
     # via matplotlib
 free-proxy==1.1.1
@@ -132,11 +138,19 @@ greenlet==3.0.3
 grpcio==1.65.4
     # via google-api-core
     # via grpcio-status
+    # via grpcio-tools
+    # via qdrant-client
 grpcio-status==1.62.3
     # via google-api-core
+grpcio-tools==1.62.3
+    # via qdrant-client
 h11==0.14.0
     # via httpcore
     # via uvicorn
+h2==4.1.0
+    # via httpx
+hpack==4.0.0
+    # via h2
 html2text==2024.2.26
     # via scrapegraphai
 httpcore==1.0.5
@@ -149,11 +163,17 @@ httpx==0.27.0
     # via langsmith
     # via ollama
     # via openai
+    # via qdrant-client
 httpx-sse==0.4.0
     # via langchain-mistralai
 huggingface-hub==0.24.5
+    # via fastembed
     # via tokenizers
     # via transformers
+humanfriendly==10.0
+    # via coloredlogs
+hyperframe==6.0.1
+    # via h2
 idna==3.7
     # via anyio
     # via httpx
@@ -218,6 +238,7 @@ langsmith==0.1.121
     # via langchain-core
 loguru==0.7.2
     # via burr
+    # via fastembed
 lxml==5.3.0
     # via free-proxy
 markdown-it-py==3.0.0
@@ -236,8 +257,12 @@ minify-html==0.15.0
     # via scrapegraphai
 mistral-common==1.4.1
     # via scrapegraphai
+mmh3==4.1.0
+    # via fastembed
 mpire==2.10.2
     # via semchunk
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
@@ -249,19 +274,27 @@ narwhals==1.3.0
     # via altair
 numpy==1.26.4
     # via contourpy
+    # via fastembed
     # via langchain
     # via langchain-aws
     # via langchain-community
     # via matplotlib
+    # via onnx
+    # via onnxruntime
     # via opencv-python-headless
     # via pandas
     # via pyarrow
     # via pydeck
+    # via qdrant-client
     # via sf-hamilton
     # via streamlit
     # via transformers
 ollama==0.3.2
     # via langchain-ollama
+onnx==1.17.0
+    # via fastembed
+onnxruntime==1.19.2
+    # via fastembed
 openai==1.40.3
     # via burr
     # via langchain-openai
@@ -275,6 +308,7 @@ packaging==24.1
     # via langchain-core
     # via marshmallow
     # via matplotlib
+    # via onnxruntime
     # via pytest
     # via sphinx
     # via streamlit
@@ -284,6 +318,7 @@ pandas==2.2.2
     # via sf-hamilton
     # via streamlit
 pillow==10.4.0
+    # via fastembed
     # via matplotlib
     # via mistral-common
     # via streamlit
@@ -294,6 +329,8 @@ playwright==1.45.1
     # via undetected-playwright
 pluggy==1.5.0
     # via pytest
+portalocker==2.10.1
+    # via qdrant-client
 proto-plus==1.24.0
     # via google-ai-generativelanguage
     # via google-api-core
@@ -303,6 +340,9 @@ protobuf==4.25.4
     # via google-generativeai
     # via googleapis-common-protos
     # via grpcio-status
+    # via grpcio-tools
+    # via onnx
+    # via onnxruntime
     # via proto-plus
     # via streamlit
 pyarrow==17.0.0
@@ -326,6 +366,7 @@ pydantic==2.8.2
     # via mistral-common
     # via openai
     # via pydantic-settings
+    # via qdrant-client
 pydantic-core==2.20.1
     # via pydantic
 pydantic-settings==2.5.2
@@ -343,6 +384,8 @@ pylint==3.2.6
 pyparsing==3.1.2
     # via httplib2
     # via matplotlib
+pystemmer==2.2.0.1
+    # via fastembed
 pytest==8.0.0
     # via pytest-mock
 pytest-mock==3.14.0
@@ -361,6 +404,8 @@ pyyaml==6.0.2
     # via langchain-community
     # via langchain-core
     # via transformers
+qdrant-client==1.11.3
+    # via scrapegraphai
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
@@ -369,6 +414,7 @@ regex==2024.7.24
     # via transformers
 requests==2.32.3
     # via burr
+    # via fastembed
     # via free-proxy
     # via google-api-core
     # via huggingface-hub
@@ -395,6 +441,8 @@ semchunk==2.2.0
     # via scrapegraphai
 sentencepiece==0.2.0
     # via mistral-common
+setuptools==75.1.0
+    # via grpcio-tools
 sf-hamilton==1.73.1
     # via burr
 six==1.16.0
@@ -406,6 +454,7 @@ sniffio==1.3.1
     # via httpx
     # via openai
 snowballstemmer==2.2.0
+    # via fastembed
     # via sphinx
 soupsieve==2.5
     # via beautifulsoup4
@@ -434,6 +483,8 @@ starlette==0.37.2
     # via fastapi
 streamlit==1.37.1
     # via burr
+sympy==1.13.3
+    # via onnxruntime
 tenacity==8.5.0
     # via langchain
     # via langchain-community
@@ -444,6 +495,7 @@ tiktoken==0.7.0
     # via mistral-common
     # via scrapegraphai
 tokenizers==0.19.1
+    # via fastembed
     # via langchain-mistralai
     # via transformers
 toml==0.10.2
@@ -456,6 +508,7 @@ tomlkit==0.13.0
 tornado==6.4.1
     # via streamlit
 tqdm==4.66.5
+    # via fastembed
     # via google-generativeai
     # via huggingface-hub
     # via mpire
@@ -495,6 +548,7 @@ uritemplate==4.1.1
     # via google-api-python-client
 urllib3==1.26.19
     # via botocore
+    # via qdrant-client
     # via requests
 uvicorn==0.30.5
     # via burr

requirements.lock

@@ -41,6 +41,8 @@ certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2
     # via requests
+coloredlogs==15.0.1
+    # via onnxruntime
 dataclasses-json==0.6.7
     # via langchain-community
 dill==0.3.8
@@ -49,9 +51,13 @@ distro==1.9.0
     # via openai
 exceptiongroup==1.2.2
     # via anyio
+fastembed==0.3.6
+    # via scrapegraphai
 filelock==3.15.4
     # via huggingface-hub
     # via transformers
+flatbuffers==24.3.25
+    # via onnxruntime
 free-proxy==1.1.1
     # via scrapegraphai
 frozenlist==1.4.1
@@ -87,10 +93,18 @@ greenlet==3.0.3
 grpcio==1.65.1
     # via google-api-core
     # via grpcio-status
+    # via grpcio-tools
+    # via qdrant-client
 grpcio-status==1.62.2
     # via google-api-core
+grpcio-tools==1.62.3
+    # via qdrant-client
 h11==0.14.0
     # via httpcore
+h2==4.1.0
+    # via httpx
+hpack==4.0.0
+    # via h2
 html2text==2024.2.26
     # via scrapegraphai
 httpcore==1.0.5
@@ -103,11 +117,17 @@ httpx==0.27.0
     # via langsmith
     # via ollama
     # via openai
+    # via qdrant-client
 httpx-sse==0.4.0
     # via langchain-mistralai
 huggingface-hub==0.24.1
+    # via fastembed
     # via tokenizers
     # via transformers
+humanfriendly==10.0
+    # via coloredlogs
+hyperframe==6.0.1
+    # via h2
 idna==3.7
     # via anyio
     # via httpx
@@ -156,6 +176,8 @@ langsmith==0.1.121
     # via langchain
     # via langchain-community
     # via langchain-core
+loguru==0.7.2
+    # via fastembed
 lxml==5.2.2
     # via free-proxy
 marshmallow==3.21.3
@@ -164,8 +186,12 @@ minify-html==0.15.0
     # via scrapegraphai
 mistral-common==1.4.1
     # via scrapegraphai
+mmh3==4.1.0
+    # via fastembed
 mpire==2.10.2
     # via semchunk
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
@@ -174,14 +200,22 @@ multiprocess==0.70.16
 mypy-extensions==1.0.0
     # via typing-inspect
 numpy==1.26.4
+    # via fastembed
     # via langchain
     # via langchain-aws
     # via langchain-community
+    # via onnx
+    # via onnxruntime
     # via opencv-python-headless
     # via pandas
+    # via qdrant-client
     # via transformers
 ollama==0.3.2
     # via langchain-ollama
+onnx==1.17.0
+    # via fastembed
+onnxruntime==1.19.2
+    # via fastembed
 openai==1.41.0
     # via langchain-openai
 opencv-python-headless==4.10.0.84
@@ -192,14 +226,18 @@ packaging==24.1
     # via huggingface-hub
     # via langchain-core
     # via marshmallow
+    # via onnxruntime
     # via transformers
 pandas==2.2.2
     # via scrapegraphai
 pillow==10.4.0
+    # via fastembed
     # via mistral-common
 playwright==1.45.1
     # via scrapegraphai
     # via undetected-playwright
+portalocker==2.10.1
+    # via qdrant-client
 proto-plus==1.24.0
     # via google-ai-generativelanguage
     # via google-api-core
@@ -209,6 +247,9 @@ protobuf==4.25.3
     # via google-generativeai
     # via googleapis-common-protos
     # via grpcio-status
+    # via grpcio-tools
+    # via onnx
+    # via onnxruntime
     # via proto-plus
 pyasn1==0.6.0
     # via pyasn1-modules
@@ -226,6 +267,7 @@ pydantic==2.8.2
     # via mistral-common
     # via openai
     # via pydantic-settings
+    # via qdrant-client
 pydantic-core==2.20.1
     # via pydantic
 pydantic-settings==2.5.2
@@ -236,6 +278,8 @@ pygments==2.18.0
     # via mpire
 pyparsing==3.1.2
     # via httplib2
+pystemmer==2.2.0.1
+    # via fastembed
 python-dateutil==2.9.0.post0
     # via botocore
     # via pandas
@@ -250,6 +294,8 @@ pyyaml==6.0.1
     # via langchain-community
     # via langchain-core
     # via transformers
+qdrant-client==1.11.3
+    # via scrapegraphai
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
@@ -257,6 +303,7 @@ regex==2024.5.15
     # via tiktoken
     # via transformers
 requests==2.32.3
+    # via fastembed
     # via free-proxy
     # via google-api-core
     # via huggingface-hub
@@ -279,17 +326,23 @@ semchunk==2.2.0
     # via scrapegraphai
 sentencepiece==0.2.0
     # via mistral-common
+setuptools==75.1.0
+    # via grpcio-tools
 six==1.16.0
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
     # via httpx
     # via openai
+snowballstemmer==2.2.0
+    # via fastembed
 soupsieve==2.5
     # via beautifulsoup4
 sqlalchemy==2.0.31
     # via langchain
     # via langchain-community
+sympy==1.13.3
+    # via onnxruntime
 tenacity==8.5.0
     # via langchain
     # via langchain-community
@@ -299,9 +352,11 @@ tiktoken==0.7.0
     # via mistral-common
     # via scrapegraphai
 tokenizers==0.19.1
+    # via fastembed
     # via langchain-mistralai
     # via transformers
 tqdm==4.66.4
+    # via fastembed
     # via google-generativeai
     # via huggingface-hub
     # via mpire
@@ -333,6 +388,7 @@ uritemplate==4.1.1
     # via google-api-python-client
 urllib3==1.26.19
     # via botocore
+    # via qdrant-client
     # via requests
 yarl==1.9.4
     # via aiohttp

scrapegraphai/graphs/depth_search_graph.py

@@ -146,6 +146,6 @@ class DepthSearchGraph(AbstractGraph):
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 
-        docs = self.final_state.get("docs", "No docs")
+        docs = self.final_state.get("answer", "No answer")
 
-        return docs
+        return docs

scrapegraphai/nodes/description_node.py

@@ -44,34 +44,25 @@ class DescriptionNode(BaseNode):
     def execute(self, state: dict) -> dict:
         self.logger.info(f"--- Executing {self.node_name} Node ---")
 
-        input_keys = self.get_input_keys(state)
+        docs = [elem for elem in state.get("docs")]
-        input_data = [state[key] for key in input_keys]
-        docs = input_data[1]
 
         chains_dict = {}
 
         for i, chunk in enumerate(tqdm(docs, desc="Processing chunks", disable=not self.verbose)):
             prompt = PromptTemplate(
                 template=DESCRIPTION_NODE_PROMPT,
-                partial_variables={"context": chunk,
+                partial_variables={"content": chunk.get("document")}
-                                   "chunk_id": i + 1
-                                 }
             )
             chain_name = f"chunk{i+1}"
             chains_dict[chain_name] = prompt | self.llm_model
 
         async_runner = RunnableParallel(**chains_dict)
-        batch_results = async_runner.invoke()
+        batch_results = async_runner.invoke({})
 
-        temp_res = {}
 
-        for i, (summary, document) in enumerate(zip(batch_results, docs)):
+        for i in range(1, len(docs)+1):
-            temp_res[summary] = {
+            docs[i-1]["summary"] = batch_results.get(f"chunk{i}").content
-                "id": i,
-                "summary": summary,
-                "document": document
-            }
 
-        state["descriptions"] = temp_res
+        state.update({self.output[0]: docs})
 
         return state

scrapegraphai/nodes/generate_answer_node_k_level.py

@@ -52,9 +52,9 @@ class GenerateAnswerNodeKLevel(BaseNode):
         self.additional_info = node_config.get("additional_info")
 
     def execute(self, state: dict) -> dict:
-        input_keys = self.get_input_keys(state)
+        self.logger.info(f"--- Executing {self.node_name} Node ---")
-        input_data = [state[key] for key in input_keys]
+
-        user_prompt = input_data[0]
+        user_prompt = state.get("user_prompt")
 
         if self.node_config.get("schema", None) is not None:
             if isinstance(self.llm_model, (ChatOpenAI, ChatMistralAI)):
@@ -113,19 +113,18 @@ class GenerateAnswerNodeKLevel(BaseNode):
         else:
             answer_db = client.query(
                 collection_name="vectorial_collection",
-                query_text=state["question"]
+                query_text=user_prompt
             )
 
-        ## TODO: from the id get the data
-        results_db = [elem for elem in state[answer_db]]
-
         chains_dict = {}
-        for i, chunk in enumerate(tqdm(results_db,
+        elems =[state.get("docs")[elem.id-1] for elem in answer_db if elem.score>0.5]
+
+        for i, chunk in enumerate(tqdm(elems,
                                        desc="Processing chunks", disable=not self.verbose)):
             prompt = PromptTemplate(
                         template=template_chunks_prompt,
-                        input_variables=["question"],
+                        input_variables=["format_instructions"],
-                        partial_variables={"context": chunk,
+                        partial_variables={"context": chunk.get("document"),
                                         "chunk_id": i + 1,
                                      }
                 )
@@ -133,7 +132,7 @@ class GenerateAnswerNodeKLevel(BaseNode):
             chains_dict[chain_name] = prompt | self.llm_model
 
         async_runner = RunnableParallel(**chains_dict)
-        batch_results = async_runner.invoke({"question": user_prompt})
+        batch_results = async_runner.invoke({"format_instructions": user_prompt})
 
         merge_prompt = PromptTemplate(
             template=template_merge_prompt,

scrapegraphai/nodes/rag_node.py

@@ -40,8 +40,9 @@ class RAGNode(BaseNode):
         )
 
     def execute(self, state: dict) -> dict:
-
+        self.logger.info(f"--- Executing {self.node_name} Node ---")
-        if self.node_config.get("client_type") == "memory":
+        
+        if self.node_config.get("client_type") in ["memory", None]:
             client = QdrantClient(":memory:")
         elif self.node_config.get("client_type") == "local_db":
             client = QdrantClient(path="path/to/db")
@@ -50,8 +51,8 @@ class RAGNode(BaseNode):
         else:
             raise ValueError("client_type provided not correct")
 
-        docs = [elem.get("summary") for elem in state.get("descriptions", {})]
+        docs = [elem.get("summary") for elem in state.get("docs")]
-        ids = [elem.get("id") for elem in state.get("descriptions", {})]
+        ids = [i for i in range(1, len(state.get("docs"))+1)]
 
         if state.get("embeddings"):
             import openai

scrapegraphai/prompts/description_node_prompts.py

@@ -5,6 +5,6 @@ description node prompts
 DESCRIPTION_NODE_PROMPT = """
 You are a  scraper and you have just scraped the
 following content from a website. \n
-Please provide a description summary of maximum of 10 words 
+Please provide a description summary of maximum of 20 words 
 Content of the website: {content}
 """

scrapegraphai/prompts/generate_answer_node_prompts.py

@@ -2,6 +2,7 @@
 Generate answer node prompts
 """
 
+
 TEMPLATE_CHUNKS_MD = """
 You are a website scraper and you have just scraped the
 following content from a website converted in markdown format.
@@ -32,6 +33,7 @@ following content from a website converted in markdown format.
 You are now asked to answer a user question about the content you have scraped.\n 
 You have scraped many chunks since the website is big and now you are asked to merge them into a single answer without repetitions (if there are any).\n
 Make sure that if a maximum number of items is specified in the instructions that you get that maximum number and do not exceed it. \n
+The structure should be coherent. \n
 Make sure the output format is a valid JSON and does not contain errors. \n
 OUTPUT INSTRUCTIONS: {format_instructions}\n 
 USER QUESTION: {question}\n