From 85cb9572971719f9f7c66171f5e2246376b6aed2 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 3 Oct 2024 13:13:04 +0200 Subject: [PATCH] feat: finished basic version of deep scraper Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com> --- examples/openai/depth_search_graph_openai.py | 12 +++- pyproject.toml | 6 +- requirements-dev.lock | 54 ++++++++++++++++++ requirements.lock | 56 +++++++++++++++++++ scrapegraphai/graphs/depth_search_graph.py | 4 +- scrapegraphai/nodes/description_node.py | 21 ++----- .../nodes/generate_answer_node_k_level.py | 21 ++++--- scrapegraphai/nodes/rag_node.py | 9 +-- .../prompts/description_node_prompts.py | 2 +- .../prompts/generate_answer_node_prompts.py | 2 + 10 files changed, 149 insertions(+), 38 deletions(-) diff --git a/examples/openai/depth_search_graph_openai.py b/examples/openai/depth_search_graph_openai.py index 7cde7865..dff07ad4 100644 --- a/examples/openai/depth_search_graph_openai.py +++ b/examples/openai/depth_search_graph_openai.py @@ -1,22 +1,28 @@ """ depth_search_graph_opeani example """ +import os +from dotenv import load_dotenv from scrapegraphai.graphs import DepthSearchGraph +load_dotenv() + +openai_key = os.getenv("OPENAI_APIKEY") + graph_config = { "llm": { - "api_key":"YOUR_API_KEY", + "api_key": openai_key, "model": "openai/gpt-4o-mini", }, "verbose": True, "headless": False, "depth": 2, - "only_inside_links": True, + "only_inside_links": False, } search_graph = DepthSearchGraph( prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", + source="https://perinim.github.io", config=graph_config ) diff --git a/pyproject.toml b/pyproject.toml index dde97395..deacd437 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,9 @@ dependencies = [ "google>=3.0.0", "langchain-ollama>=0.1.3", "semchunk==2.2.0", - "transformers==4.44.2" + "transformers==4.44.2", + "qdrant-client>=1.11.3", + "fastembed>=0.3.6" ] license = "MIT" @@ -99,7 +101,7 @@ screenshot_scraper = [ "pillow>=10.4.0", ] -# Group 5: Faiss CPU +# Group 5: qdrant qdrant = [ "qdrant-client>=1.11.3", "fastembed>=0.3.6" diff --git a/requirements-dev.lock b/requirements-dev.lock index 1d9d469a..3423cef0 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -64,6 +64,8 @@ click==8.1.7 # via burr # via streamlit # via uvicorn +coloredlogs==15.0.1 + # via onnxruntime contourpy==1.2.1 # via matplotlib cycler==0.12.1 @@ -84,9 +86,13 @@ fastapi==0.112.0 # via burr fastapi-pagination==0.12.26 # via burr +fastembed==0.3.6 + # via scrapegraphai filelock==3.15.4 # via huggingface-hub # via transformers +flatbuffers==24.3.25 + # via onnxruntime fonttools==4.53.1 # via matplotlib free-proxy==1.1.1 @@ -132,11 +138,19 @@ greenlet==3.0.3 grpcio==1.65.4 # via google-api-core # via grpcio-status + # via grpcio-tools + # via qdrant-client grpcio-status==1.62.3 # via google-api-core +grpcio-tools==1.62.3 + # via qdrant-client h11==0.14.0 # via httpcore # via uvicorn +h2==4.1.0 + # via httpx +hpack==4.0.0 + # via h2 html2text==2024.2.26 # via scrapegraphai httpcore==1.0.5 @@ -149,11 +163,17 @@ httpx==0.27.0 # via langsmith # via ollama # via openai + # via qdrant-client httpx-sse==0.4.0 # via langchain-mistralai huggingface-hub==0.24.5 + # via fastembed # via tokenizers # via transformers +humanfriendly==10.0 + # via coloredlogs +hyperframe==6.0.1 + # via h2 idna==3.7 # via anyio # via httpx @@ -218,6 +238,7 @@ langsmith==0.1.121 # via langchain-core loguru==0.7.2 # via burr + # via fastembed lxml==5.3.0 # via free-proxy markdown-it-py==3.0.0 @@ -236,8 +257,12 @@ minify-html==0.15.0 # via scrapegraphai mistral-common==1.4.1 # via scrapegraphai +mmh3==4.1.0 + # via fastembed mpire==2.10.2 # via semchunk +mpmath==1.3.0 + # via sympy multidict==6.0.5 # via aiohttp # via yarl @@ -249,19 +274,27 @@ narwhals==1.3.0 # via altair numpy==1.26.4 # via contourpy + # via fastembed # via langchain # via langchain-aws # via langchain-community # via matplotlib + # via onnx + # via onnxruntime # via opencv-python-headless # via pandas # via pyarrow # via pydeck + # via qdrant-client # via sf-hamilton # via streamlit # via transformers ollama==0.3.2 # via langchain-ollama +onnx==1.17.0 + # via fastembed +onnxruntime==1.19.2 + # via fastembed openai==1.40.3 # via burr # via langchain-openai @@ -275,6 +308,7 @@ packaging==24.1 # via langchain-core # via marshmallow # via matplotlib + # via onnxruntime # via pytest # via sphinx # via streamlit @@ -284,6 +318,7 @@ pandas==2.2.2 # via sf-hamilton # via streamlit pillow==10.4.0 + # via fastembed # via matplotlib # via mistral-common # via streamlit @@ -294,6 +329,8 @@ playwright==1.45.1 # via undetected-playwright pluggy==1.5.0 # via pytest +portalocker==2.10.1 + # via qdrant-client proto-plus==1.24.0 # via google-ai-generativelanguage # via google-api-core @@ -303,6 +340,9 @@ protobuf==4.25.4 # via google-generativeai # via googleapis-common-protos # via grpcio-status + # via grpcio-tools + # via onnx + # via onnxruntime # via proto-plus # via streamlit pyarrow==17.0.0 @@ -326,6 +366,7 @@ pydantic==2.8.2 # via mistral-common # via openai # via pydantic-settings + # via qdrant-client pydantic-core==2.20.1 # via pydantic pydantic-settings==2.5.2 @@ -343,6 +384,8 @@ pylint==3.2.6 pyparsing==3.1.2 # via httplib2 # via matplotlib +pystemmer==2.2.0.1 + # via fastembed pytest==8.0.0 # via pytest-mock pytest-mock==3.14.0 @@ -361,6 +404,8 @@ pyyaml==6.0.2 # via langchain-community # via langchain-core # via transformers +qdrant-client==1.11.3 + # via scrapegraphai referencing==0.35.1 # via jsonschema # via jsonschema-specifications @@ -369,6 +414,7 @@ regex==2024.7.24 # via transformers requests==2.32.3 # via burr + # via fastembed # via free-proxy # via google-api-core # via huggingface-hub @@ -395,6 +441,8 @@ semchunk==2.2.0 # via scrapegraphai sentencepiece==0.2.0 # via mistral-common +setuptools==75.1.0 + # via grpcio-tools sf-hamilton==1.73.1 # via burr six==1.16.0 @@ -406,6 +454,7 @@ sniffio==1.3.1 # via httpx # via openai snowballstemmer==2.2.0 + # via fastembed # via sphinx soupsieve==2.5 # via beautifulsoup4 @@ -434,6 +483,8 @@ starlette==0.37.2 # via fastapi streamlit==1.37.1 # via burr +sympy==1.13.3 + # via onnxruntime tenacity==8.5.0 # via langchain # via langchain-community @@ -444,6 +495,7 @@ tiktoken==0.7.0 # via mistral-common # via scrapegraphai tokenizers==0.19.1 + # via fastembed # via langchain-mistralai # via transformers toml==0.10.2 @@ -456,6 +508,7 @@ tomlkit==0.13.0 tornado==6.4.1 # via streamlit tqdm==4.66.5 + # via fastembed # via google-generativeai # via huggingface-hub # via mpire @@ -495,6 +548,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via botocore + # via qdrant-client # via requests uvicorn==0.30.5 # via burr diff --git a/requirements.lock b/requirements.lock index 84e25a0f..8949648a 100644 --- a/requirements.lock +++ b/requirements.lock @@ -41,6 +41,8 @@ certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests +coloredlogs==15.0.1 + # via onnxruntime dataclasses-json==0.6.7 # via langchain-community dill==0.3.8 @@ -49,9 +51,13 @@ distro==1.9.0 # via openai exceptiongroup==1.2.2 # via anyio +fastembed==0.3.6 + # via scrapegraphai filelock==3.15.4 # via huggingface-hub # via transformers +flatbuffers==24.3.25 + # via onnxruntime free-proxy==1.1.1 # via scrapegraphai frozenlist==1.4.1 @@ -87,10 +93,18 @@ greenlet==3.0.3 grpcio==1.65.1 # via google-api-core # via grpcio-status + # via grpcio-tools + # via qdrant-client grpcio-status==1.62.2 # via google-api-core +grpcio-tools==1.62.3 + # via qdrant-client h11==0.14.0 # via httpcore +h2==4.1.0 + # via httpx +hpack==4.0.0 + # via h2 html2text==2024.2.26 # via scrapegraphai httpcore==1.0.5 @@ -103,11 +117,17 @@ httpx==0.27.0 # via langsmith # via ollama # via openai + # via qdrant-client httpx-sse==0.4.0 # via langchain-mistralai huggingface-hub==0.24.1 + # via fastembed # via tokenizers # via transformers +humanfriendly==10.0 + # via coloredlogs +hyperframe==6.0.1 + # via h2 idna==3.7 # via anyio # via httpx @@ -156,6 +176,8 @@ langsmith==0.1.121 # via langchain # via langchain-community # via langchain-core +loguru==0.7.2 + # via fastembed lxml==5.2.2 # via free-proxy marshmallow==3.21.3 @@ -164,8 +186,12 @@ minify-html==0.15.0 # via scrapegraphai mistral-common==1.4.1 # via scrapegraphai +mmh3==4.1.0 + # via fastembed mpire==2.10.2 # via semchunk +mpmath==1.3.0 + # via sympy multidict==6.0.5 # via aiohttp # via yarl @@ -174,14 +200,22 @@ multiprocess==0.70.16 mypy-extensions==1.0.0 # via typing-inspect numpy==1.26.4 + # via fastembed # via langchain # via langchain-aws # via langchain-community + # via onnx + # via onnxruntime # via opencv-python-headless # via pandas + # via qdrant-client # via transformers ollama==0.3.2 # via langchain-ollama +onnx==1.17.0 + # via fastembed +onnxruntime==1.19.2 + # via fastembed openai==1.41.0 # via langchain-openai opencv-python-headless==4.10.0.84 @@ -192,14 +226,18 @@ packaging==24.1 # via huggingface-hub # via langchain-core # via marshmallow + # via onnxruntime # via transformers pandas==2.2.2 # via scrapegraphai pillow==10.4.0 + # via fastembed # via mistral-common playwright==1.45.1 # via scrapegraphai # via undetected-playwright +portalocker==2.10.1 + # via qdrant-client proto-plus==1.24.0 # via google-ai-generativelanguage # via google-api-core @@ -209,6 +247,9 @@ protobuf==4.25.3 # via google-generativeai # via googleapis-common-protos # via grpcio-status + # via grpcio-tools + # via onnx + # via onnxruntime # via proto-plus pyasn1==0.6.0 # via pyasn1-modules @@ -226,6 +267,7 @@ pydantic==2.8.2 # via mistral-common # via openai # via pydantic-settings + # via qdrant-client pydantic-core==2.20.1 # via pydantic pydantic-settings==2.5.2 @@ -236,6 +278,8 @@ pygments==2.18.0 # via mpire pyparsing==3.1.2 # via httplib2 +pystemmer==2.2.0.1 + # via fastembed python-dateutil==2.9.0.post0 # via botocore # via pandas @@ -250,6 +294,8 @@ pyyaml==6.0.1 # via langchain-community # via langchain-core # via transformers +qdrant-client==1.11.3 + # via scrapegraphai referencing==0.35.1 # via jsonschema # via jsonschema-specifications @@ -257,6 +303,7 @@ regex==2024.5.15 # via tiktoken # via transformers requests==2.32.3 + # via fastembed # via free-proxy # via google-api-core # via huggingface-hub @@ -279,17 +326,23 @@ semchunk==2.2.0 # via scrapegraphai sentencepiece==0.2.0 # via mistral-common +setuptools==75.1.0 + # via grpcio-tools six==1.16.0 # via python-dateutil sniffio==1.3.1 # via anyio # via httpx # via openai +snowballstemmer==2.2.0 + # via fastembed soupsieve==2.5 # via beautifulsoup4 sqlalchemy==2.0.31 # via langchain # via langchain-community +sympy==1.13.3 + # via onnxruntime tenacity==8.5.0 # via langchain # via langchain-community @@ -299,9 +352,11 @@ tiktoken==0.7.0 # via mistral-common # via scrapegraphai tokenizers==0.19.1 + # via fastembed # via langchain-mistralai # via transformers tqdm==4.66.4 + # via fastembed # via google-generativeai # via huggingface-hub # via mpire @@ -333,6 +388,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via botocore + # via qdrant-client # via requests yarl==1.9.4 # via aiohttp diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py index a93d8fcf..13b39129 100644 --- a/scrapegraphai/graphs/depth_search_graph.py +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -146,6 +146,6 @@ class DepthSearchGraph(AbstractGraph): inputs = {"user_prompt": self.prompt, self.input_key: self.source} self.final_state, self.execution_info = self.graph.execute(inputs) - docs = self.final_state.get("docs", "No docs") + docs = self.final_state.get("answer", "No answer") - return docs \ No newline at end of file + return docs diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py index 97ef2e8f..60c56cec 100644 --- a/scrapegraphai/nodes/description_node.py +++ b/scrapegraphai/nodes/description_node.py @@ -44,34 +44,25 @@ class DescriptionNode(BaseNode): def execute(self, state: dict) -> dict: self.logger.info(f"--- Executing {self.node_name} Node ---") - input_keys = self.get_input_keys(state) - input_data = [state[key] for key in input_keys] - docs = input_data[1] + docs = [elem for elem in state.get("docs")] chains_dict = {} for i, chunk in enumerate(tqdm(docs, desc="Processing chunks", disable=not self.verbose)): prompt = PromptTemplate( template=DESCRIPTION_NODE_PROMPT, - partial_variables={"context": chunk, - "chunk_id": i + 1 - } + partial_variables={"content": chunk.get("document")} ) chain_name = f"chunk{i+1}" chains_dict[chain_name] = prompt | self.llm_model async_runner = RunnableParallel(**chains_dict) - batch_results = async_runner.invoke() + batch_results = async_runner.invoke({}) - temp_res = {} - for i, (summary, document) in enumerate(zip(batch_results, docs)): - temp_res[summary] = { - "id": i, - "summary": summary, - "document": document - } + for i in range(1, len(docs)+1): + docs[i-1]["summary"] = batch_results.get(f"chunk{i}").content - state["descriptions"] = temp_res + state.update({self.output[0]: docs}) return state diff --git a/scrapegraphai/nodes/generate_answer_node_k_level.py b/scrapegraphai/nodes/generate_answer_node_k_level.py index 8dea5c98..291109f2 100644 --- a/scrapegraphai/nodes/generate_answer_node_k_level.py +++ b/scrapegraphai/nodes/generate_answer_node_k_level.py @@ -52,9 +52,9 @@ class GenerateAnswerNodeKLevel(BaseNode): self.additional_info = node_config.get("additional_info") def execute(self, state: dict) -> dict: - input_keys = self.get_input_keys(state) - input_data = [state[key] for key in input_keys] - user_prompt = input_data[0] + self.logger.info(f"--- Executing {self.node_name} Node ---") + + user_prompt = state.get("user_prompt") if self.node_config.get("schema", None) is not None: if isinstance(self.llm_model, (ChatOpenAI, ChatMistralAI)): @@ -113,19 +113,18 @@ class GenerateAnswerNodeKLevel(BaseNode): else: answer_db = client.query( collection_name="vectorial_collection", - query_text=state["question"] + query_text=user_prompt ) - ## TODO: from the id get the data - results_db = [elem for elem in state[answer_db]] - chains_dict = {} - for i, chunk in enumerate(tqdm(results_db, + elems =[state.get("docs")[elem.id-1] for elem in answer_db if elem.score>0.5] + + for i, chunk in enumerate(tqdm(elems, desc="Processing chunks", disable=not self.verbose)): prompt = PromptTemplate( template=template_chunks_prompt, - input_variables=["question"], - partial_variables={"context": chunk, + input_variables=["format_instructions"], + partial_variables={"context": chunk.get("document"), "chunk_id": i + 1, } ) @@ -133,7 +132,7 @@ class GenerateAnswerNodeKLevel(BaseNode): chains_dict[chain_name] = prompt | self.llm_model async_runner = RunnableParallel(**chains_dict) - batch_results = async_runner.invoke({"question": user_prompt}) + batch_results = async_runner.invoke({"format_instructions": user_prompt}) merge_prompt = PromptTemplate( template=template_merge_prompt, diff --git a/scrapegraphai/nodes/rag_node.py b/scrapegraphai/nodes/rag_node.py index 3f861478..b67c50e9 100644 --- a/scrapegraphai/nodes/rag_node.py +++ b/scrapegraphai/nodes/rag_node.py @@ -40,8 +40,9 @@ class RAGNode(BaseNode): ) def execute(self, state: dict) -> dict: - - if self.node_config.get("client_type") == "memory": + self.logger.info(f"--- Executing {self.node_name} Node ---") + + if self.node_config.get("client_type") in ["memory", None]: client = QdrantClient(":memory:") elif self.node_config.get("client_type") == "local_db": client = QdrantClient(path="path/to/db") @@ -50,8 +51,8 @@ class RAGNode(BaseNode): else: raise ValueError("client_type provided not correct") - docs = [elem.get("summary") for elem in state.get("descriptions", {})] - ids = [elem.get("id") for elem in state.get("descriptions", {})] + docs = [elem.get("summary") for elem in state.get("docs")] + ids = [i for i in range(1, len(state.get("docs"))+1)] if state.get("embeddings"): import openai diff --git a/scrapegraphai/prompts/description_node_prompts.py b/scrapegraphai/prompts/description_node_prompts.py index 5cd78d7f..20df481a 100644 --- a/scrapegraphai/prompts/description_node_prompts.py +++ b/scrapegraphai/prompts/description_node_prompts.py @@ -5,6 +5,6 @@ description node prompts DESCRIPTION_NODE_PROMPT = """ You are a scraper and you have just scraped the following content from a website. \n -Please provide a description summary of maximum of 10 words +Please provide a description summary of maximum of 20 words Content of the website: {content} """ \ No newline at end of file diff --git a/scrapegraphai/prompts/generate_answer_node_prompts.py b/scrapegraphai/prompts/generate_answer_node_prompts.py index 7c098fe2..1b336fb4 100644 --- a/scrapegraphai/prompts/generate_answer_node_prompts.py +++ b/scrapegraphai/prompts/generate_answer_node_prompts.py @@ -2,6 +2,7 @@ Generate answer node prompts """ + TEMPLATE_CHUNKS_MD = """ You are a website scraper and you have just scraped the following content from a website converted in markdown format. @@ -32,6 +33,7 @@ following content from a website converted in markdown format. You are now asked to answer a user question about the content you have scraped.\n You have scraped many chunks since the website is big and now you are asked to merge them into a single answer without repetitions (if there are any).\n Make sure that if a maximum number of items is specified in the instructions that you get that maximum number and do not exceed it. \n +The structure should be coherent. \n Make sure the output format is a valid JSON and does not contain errors. \n OUTPUT INSTRUCTIONS: {format_instructions}\n USER QUESTION: {question}\n