mirror of
https://github.com/VinciGit00/Scrapegraph-ai.git
synced 2026-07-12 21:01:56 +08:00
feat: finished basic version of deep scraper
Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com>
This commit is contained in:
Marco Vinciguerra
parent
4b371f4d94
commit
85cb957297
@ -1,22 +1,28 @@
|
||||
"""
|
||||
depth_search_graph_opeani example
|
||||
"""
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from scrapegraphai.graphs import DepthSearchGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key":"YOUR_API_KEY",
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o-mini",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
"depth": 2,
|
||||
"only_inside_links": True,
|
||||
"only_inside_links": False,
|
||||
}
|
||||
|
||||
search_graph = DepthSearchGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
source="https://perinim.github.io/projects/",
|
||||
source="https://perinim.github.io",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
|
||||
@ -31,7 +31,9 @@ dependencies = [
|
||||
"google>=3.0.0",
|
||||
"langchain-ollama>=0.1.3",
|
||||
"semchunk==2.2.0",
|
||||
"transformers==4.44.2"
|
||||
"transformers==4.44.2",
|
||||
"qdrant-client>=1.11.3",
|
||||
"fastembed>=0.3.6"
|
||||
]
|
||||
|
||||
license = "MIT"
|
||||
@ -99,7 +101,7 @@ screenshot_scraper = [
|
||||
"pillow>=10.4.0",
|
||||
]
|
||||
|
||||
# Group 5: Faiss CPU
|
||||
# Group 5: qdrant
|
||||
qdrant = [
|
||||
"qdrant-client>=1.11.3",
|
||||
"fastembed>=0.3.6"
|
||||
|
||||
@ -64,6 +64,8 @@ click==8.1.7
|
||||
# via burr
|
||||
# via streamlit
|
||||
# via uvicorn
|
||||
coloredlogs==15.0.1
|
||||
# via onnxruntime
|
||||
contourpy==1.2.1
|
||||
# via matplotlib
|
||||
cycler==0.12.1
|
||||
@ -84,9 +86,13 @@ fastapi==0.112.0
|
||||
# via burr
|
||||
fastapi-pagination==0.12.26
|
||||
# via burr
|
||||
fastembed==0.3.6
|
||||
# via scrapegraphai
|
||||
filelock==3.15.4
|
||||
# via huggingface-hub
|
||||
# via transformers
|
||||
flatbuffers==24.3.25
|
||||
# via onnxruntime
|
||||
fonttools==4.53.1
|
||||
# via matplotlib
|
||||
free-proxy==1.1.1
|
||||
@ -132,11 +138,19 @@ greenlet==3.0.3
|
||||
grpcio==1.65.4
|
||||
# via google-api-core
|
||||
# via grpcio-status
|
||||
# via grpcio-tools
|
||||
# via qdrant-client
|
||||
grpcio-status==1.62.3
|
||||
# via google-api-core
|
||||
grpcio-tools==1.62.3
|
||||
# via qdrant-client
|
||||
h11==0.14.0
|
||||
# via httpcore
|
||||
# via uvicorn
|
||||
h2==4.1.0
|
||||
# via httpx
|
||||
hpack==4.0.0
|
||||
# via h2
|
||||
html2text==2024.2.26
|
||||
# via scrapegraphai
|
||||
httpcore==1.0.5
|
||||
@ -149,11 +163,17 @@ httpx==0.27.0
|
||||
# via langsmith
|
||||
# via ollama
|
||||
# via openai
|
||||
# via qdrant-client
|
||||
httpx-sse==0.4.0
|
||||
# via langchain-mistralai
|
||||
huggingface-hub==0.24.5
|
||||
# via fastembed
|
||||
# via tokenizers
|
||||
# via transformers
|
||||
humanfriendly==10.0
|
||||
# via coloredlogs
|
||||
hyperframe==6.0.1
|
||||
# via h2
|
||||
idna==3.7
|
||||
# via anyio
|
||||
# via httpx
|
||||
@ -218,6 +238,7 @@ langsmith==0.1.121
|
||||
# via langchain-core
|
||||
loguru==0.7.2
|
||||
# via burr
|
||||
# via fastembed
|
||||
lxml==5.3.0
|
||||
# via free-proxy
|
||||
markdown-it-py==3.0.0
|
||||
@ -236,8 +257,12 @@ minify-html==0.15.0
|
||||
# via scrapegraphai
|
||||
mistral-common==1.4.1
|
||||
# via scrapegraphai
|
||||
mmh3==4.1.0
|
||||
# via fastembed
|
||||
mpire==2.10.2
|
||||
# via semchunk
|
||||
mpmath==1.3.0
|
||||
# via sympy
|
||||
multidict==6.0.5
|
||||
# via aiohttp
|
||||
# via yarl
|
||||
@ -249,19 +274,27 @@ narwhals==1.3.0
|
||||
# via altair
|
||||
numpy==1.26.4
|
||||
# via contourpy
|
||||
# via fastembed
|
||||
# via langchain
|
||||
# via langchain-aws
|
||||
# via langchain-community
|
||||
# via matplotlib
|
||||
# via onnx
|
||||
# via onnxruntime
|
||||
# via opencv-python-headless
|
||||
# via pandas
|
||||
# via pyarrow
|
||||
# via pydeck
|
||||
# via qdrant-client
|
||||
# via sf-hamilton
|
||||
# via streamlit
|
||||
# via transformers
|
||||
ollama==0.3.2
|
||||
# via langchain-ollama
|
||||
onnx==1.17.0
|
||||
# via fastembed
|
||||
onnxruntime==1.19.2
|
||||
# via fastembed
|
||||
openai==1.40.3
|
||||
# via burr
|
||||
# via langchain-openai
|
||||
@ -275,6 +308,7 @@ packaging==24.1
|
||||
# via langchain-core
|
||||
# via marshmallow
|
||||
# via matplotlib
|
||||
# via onnxruntime
|
||||
# via pytest
|
||||
# via sphinx
|
||||
# via streamlit
|
||||
@ -284,6 +318,7 @@ pandas==2.2.2
|
||||
# via sf-hamilton
|
||||
# via streamlit
|
||||
pillow==10.4.0
|
||||
# via fastembed
|
||||
# via matplotlib
|
||||
# via mistral-common
|
||||
# via streamlit
|
||||
@ -294,6 +329,8 @@ playwright==1.45.1
|
||||
# via undetected-playwright
|
||||
pluggy==1.5.0
|
||||
# via pytest
|
||||
portalocker==2.10.1
|
||||
# via qdrant-client
|
||||
proto-plus==1.24.0
|
||||
# via google-ai-generativelanguage
|
||||
# via google-api-core
|
||||
@ -303,6 +340,9 @@ protobuf==4.25.4
|
||||
# via google-generativeai
|
||||
# via googleapis-common-protos
|
||||
# via grpcio-status
|
||||
# via grpcio-tools
|
||||
# via onnx
|
||||
# via onnxruntime
|
||||
# via proto-plus
|
||||
# via streamlit
|
||||
pyarrow==17.0.0
|
||||
@ -326,6 +366,7 @@ pydantic==2.8.2
|
||||
# via mistral-common
|
||||
# via openai
|
||||
# via pydantic-settings
|
||||
# via qdrant-client
|
||||
pydantic-core==2.20.1
|
||||
# via pydantic
|
||||
pydantic-settings==2.5.2
|
||||
@ -343,6 +384,8 @@ pylint==3.2.6
|
||||
pyparsing==3.1.2
|
||||
# via httplib2
|
||||
# via matplotlib
|
||||
pystemmer==2.2.0.1
|
||||
# via fastembed
|
||||
pytest==8.0.0
|
||||
# via pytest-mock
|
||||
pytest-mock==3.14.0
|
||||
@ -361,6 +404,8 @@ pyyaml==6.0.2
|
||||
# via langchain-community
|
||||
# via langchain-core
|
||||
# via transformers
|
||||
qdrant-client==1.11.3
|
||||
# via scrapegraphai
|
||||
referencing==0.35.1
|
||||
# via jsonschema
|
||||
# via jsonschema-specifications
|
||||
@ -369,6 +414,7 @@ regex==2024.7.24
|
||||
# via transformers
|
||||
requests==2.32.3
|
||||
# via burr
|
||||
# via fastembed
|
||||
# via free-proxy
|
||||
# via google-api-core
|
||||
# via huggingface-hub
|
||||
@ -395,6 +441,8 @@ semchunk==2.2.0
|
||||
# via scrapegraphai
|
||||
sentencepiece==0.2.0
|
||||
# via mistral-common
|
||||
setuptools==75.1.0
|
||||
# via grpcio-tools
|
||||
sf-hamilton==1.73.1
|
||||
# via burr
|
||||
six==1.16.0
|
||||
@ -406,6 +454,7 @@ sniffio==1.3.1
|
||||
# via httpx
|
||||
# via openai
|
||||
snowballstemmer==2.2.0
|
||||
# via fastembed
|
||||
# via sphinx
|
||||
soupsieve==2.5
|
||||
# via beautifulsoup4
|
||||
@ -434,6 +483,8 @@ starlette==0.37.2
|
||||
# via fastapi
|
||||
streamlit==1.37.1
|
||||
# via burr
|
||||
sympy==1.13.3
|
||||
# via onnxruntime
|
||||
tenacity==8.5.0
|
||||
# via langchain
|
||||
# via langchain-community
|
||||
@ -444,6 +495,7 @@ tiktoken==0.7.0
|
||||
# via mistral-common
|
||||
# via scrapegraphai
|
||||
tokenizers==0.19.1
|
||||
# via fastembed
|
||||
# via langchain-mistralai
|
||||
# via transformers
|
||||
toml==0.10.2
|
||||
@ -456,6 +508,7 @@ tomlkit==0.13.0
|
||||
tornado==6.4.1
|
||||
# via streamlit
|
||||
tqdm==4.66.5
|
||||
# via fastembed
|
||||
# via google-generativeai
|
||||
# via huggingface-hub
|
||||
# via mpire
|
||||
@ -495,6 +548,7 @@ uritemplate==4.1.1
|
||||
# via google-api-python-client
|
||||
urllib3==1.26.19
|
||||
# via botocore
|
||||
# via qdrant-client
|
||||
# via requests
|
||||
uvicorn==0.30.5
|
||||
# via burr
|
||||
|
||||
@ -41,6 +41,8 @@ certifi==2024.7.4
|
||||
# via requests
|
||||
charset-normalizer==3.3.2
|
||||
# via requests
|
||||
coloredlogs==15.0.1
|
||||
# via onnxruntime
|
||||
dataclasses-json==0.6.7
|
||||
# via langchain-community
|
||||
dill==0.3.8
|
||||
@ -49,9 +51,13 @@ distro==1.9.0
|
||||
# via openai
|
||||
exceptiongroup==1.2.2
|
||||
# via anyio
|
||||
fastembed==0.3.6
|
||||
# via scrapegraphai
|
||||
filelock==3.15.4
|
||||
# via huggingface-hub
|
||||
# via transformers
|
||||
flatbuffers==24.3.25
|
||||
# via onnxruntime
|
||||
free-proxy==1.1.1
|
||||
# via scrapegraphai
|
||||
frozenlist==1.4.1
|
||||
@ -87,10 +93,18 @@ greenlet==3.0.3
|
||||
grpcio==1.65.1
|
||||
# via google-api-core
|
||||
# via grpcio-status
|
||||
# via grpcio-tools
|
||||
# via qdrant-client
|
||||
grpcio-status==1.62.2
|
||||
# via google-api-core
|
||||
grpcio-tools==1.62.3
|
||||
# via qdrant-client
|
||||
h11==0.14.0
|
||||
# via httpcore
|
||||
h2==4.1.0
|
||||
# via httpx
|
||||
hpack==4.0.0
|
||||
# via h2
|
||||
html2text==2024.2.26
|
||||
# via scrapegraphai
|
||||
httpcore==1.0.5
|
||||
@ -103,11 +117,17 @@ httpx==0.27.0
|
||||
# via langsmith
|
||||
# via ollama
|
||||
# via openai
|
||||
# via qdrant-client
|
||||
httpx-sse==0.4.0
|
||||
# via langchain-mistralai
|
||||
huggingface-hub==0.24.1
|
||||
# via fastembed
|
||||
# via tokenizers
|
||||
# via transformers
|
||||
humanfriendly==10.0
|
||||
# via coloredlogs
|
||||
hyperframe==6.0.1
|
||||
# via h2
|
||||
idna==3.7
|
||||
# via anyio
|
||||
# via httpx
|
||||
@ -156,6 +176,8 @@ langsmith==0.1.121
|
||||
# via langchain
|
||||
# via langchain-community
|
||||
# via langchain-core
|
||||
loguru==0.7.2
|
||||
# via fastembed
|
||||
lxml==5.2.2
|
||||
# via free-proxy
|
||||
marshmallow==3.21.3
|
||||
@ -164,8 +186,12 @@ minify-html==0.15.0
|
||||
# via scrapegraphai
|
||||
mistral-common==1.4.1
|
||||
# via scrapegraphai
|
||||
mmh3==4.1.0
|
||||
# via fastembed
|
||||
mpire==2.10.2
|
||||
# via semchunk
|
||||
mpmath==1.3.0
|
||||
# via sympy
|
||||
multidict==6.0.5
|
||||
# via aiohttp
|
||||
# via yarl
|
||||
@ -174,14 +200,22 @@ multiprocess==0.70.16
|
||||
mypy-extensions==1.0.0
|
||||
# via typing-inspect
|
||||
numpy==1.26.4
|
||||
# via fastembed
|
||||
# via langchain
|
||||
# via langchain-aws
|
||||
# via langchain-community
|
||||
# via onnx
|
||||
# via onnxruntime
|
||||
# via opencv-python-headless
|
||||
# via pandas
|
||||
# via qdrant-client
|
||||
# via transformers
|
||||
ollama==0.3.2
|
||||
# via langchain-ollama
|
||||
onnx==1.17.0
|
||||
# via fastembed
|
||||
onnxruntime==1.19.2
|
||||
# via fastembed
|
||||
openai==1.41.0
|
||||
# via langchain-openai
|
||||
opencv-python-headless==4.10.0.84
|
||||
@ -192,14 +226,18 @@ packaging==24.1
|
||||
# via huggingface-hub
|
||||
# via langchain-core
|
||||
# via marshmallow
|
||||
# via onnxruntime
|
||||
# via transformers
|
||||
pandas==2.2.2
|
||||
# via scrapegraphai
|
||||
pillow==10.4.0
|
||||
# via fastembed
|
||||
# via mistral-common
|
||||
playwright==1.45.1
|
||||
# via scrapegraphai
|
||||
# via undetected-playwright
|
||||
portalocker==2.10.1
|
||||
# via qdrant-client
|
||||
proto-plus==1.24.0
|
||||
# via google-ai-generativelanguage
|
||||
# via google-api-core
|
||||
@ -209,6 +247,9 @@ protobuf==4.25.3
|
||||
# via google-generativeai
|
||||
# via googleapis-common-protos
|
||||
# via grpcio-status
|
||||
# via grpcio-tools
|
||||
# via onnx
|
||||
# via onnxruntime
|
||||
# via proto-plus
|
||||
pyasn1==0.6.0
|
||||
# via pyasn1-modules
|
||||
@ -226,6 +267,7 @@ pydantic==2.8.2
|
||||
# via mistral-common
|
||||
# via openai
|
||||
# via pydantic-settings
|
||||
# via qdrant-client
|
||||
pydantic-core==2.20.1
|
||||
# via pydantic
|
||||
pydantic-settings==2.5.2
|
||||
@ -236,6 +278,8 @@ pygments==2.18.0
|
||||
# via mpire
|
||||
pyparsing==3.1.2
|
||||
# via httplib2
|
||||
pystemmer==2.2.0.1
|
||||
# via fastembed
|
||||
python-dateutil==2.9.0.post0
|
||||
# via botocore
|
||||
# via pandas
|
||||
@ -250,6 +294,8 @@ pyyaml==6.0.1
|
||||
# via langchain-community
|
||||
# via langchain-core
|
||||
# via transformers
|
||||
qdrant-client==1.11.3
|
||||
# via scrapegraphai
|
||||
referencing==0.35.1
|
||||
# via jsonschema
|
||||
# via jsonschema-specifications
|
||||
@ -257,6 +303,7 @@ regex==2024.5.15
|
||||
# via tiktoken
|
||||
# via transformers
|
||||
requests==2.32.3
|
||||
# via fastembed
|
||||
# via free-proxy
|
||||
# via google-api-core
|
||||
# via huggingface-hub
|
||||
@ -279,17 +326,23 @@ semchunk==2.2.0
|
||||
# via scrapegraphai
|
||||
sentencepiece==0.2.0
|
||||
# via mistral-common
|
||||
setuptools==75.1.0
|
||||
# via grpcio-tools
|
||||
six==1.16.0
|
||||
# via python-dateutil
|
||||
sniffio==1.3.1
|
||||
# via anyio
|
||||
# via httpx
|
||||
# via openai
|
||||
snowballstemmer==2.2.0
|
||||
# via fastembed
|
||||
soupsieve==2.5
|
||||
# via beautifulsoup4
|
||||
sqlalchemy==2.0.31
|
||||
# via langchain
|
||||
# via langchain-community
|
||||
sympy==1.13.3
|
||||
# via onnxruntime
|
||||
tenacity==8.5.0
|
||||
# via langchain
|
||||
# via langchain-community
|
||||
@ -299,9 +352,11 @@ tiktoken==0.7.0
|
||||
# via mistral-common
|
||||
# via scrapegraphai
|
||||
tokenizers==0.19.1
|
||||
# via fastembed
|
||||
# via langchain-mistralai
|
||||
# via transformers
|
||||
tqdm==4.66.4
|
||||
# via fastembed
|
||||
# via google-generativeai
|
||||
# via huggingface-hub
|
||||
# via mpire
|
||||
@ -333,6 +388,7 @@ uritemplate==4.1.1
|
||||
# via google-api-python-client
|
||||
urllib3==1.26.19
|
||||
# via botocore
|
||||
# via qdrant-client
|
||||
# via requests
|
||||
yarl==1.9.4
|
||||
# via aiohttp
|
||||
|
||||
@ -146,6 +146,6 @@ class DepthSearchGraph(AbstractGraph):
|
||||
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
|
||||
self.final_state, self.execution_info = self.graph.execute(inputs)
|
||||
|
||||
docs = self.final_state.get("docs", "No docs")
|
||||
docs = self.final_state.get("answer", "No answer")
|
||||
|
||||
return docs
|
||||
return docs
|
||||
|
||||
@ -44,34 +44,25 @@ class DescriptionNode(BaseNode):
|
||||
def execute(self, state: dict) -> dict:
|
||||
self.logger.info(f"--- Executing {self.node_name} Node ---")
|
||||
|
||||
input_keys = self.get_input_keys(state)
|
||||
input_data = [state[key] for key in input_keys]
|
||||
docs = input_data[1]
|
||||
docs = [elem for elem in state.get("docs")]
|
||||
|
||||
chains_dict = {}
|
||||
|
||||
for i, chunk in enumerate(tqdm(docs, desc="Processing chunks", disable=not self.verbose)):
|
||||
prompt = PromptTemplate(
|
||||
template=DESCRIPTION_NODE_PROMPT,
|
||||
partial_variables={"context": chunk,
|
||||
"chunk_id": i + 1
|
||||
}
|
||||
partial_variables={"content": chunk.get("document")}
|
||||
)
|
||||
chain_name = f"chunk{i+1}"
|
||||
chains_dict[chain_name] = prompt | self.llm_model
|
||||
|
||||
async_runner = RunnableParallel(**chains_dict)
|
||||
batch_results = async_runner.invoke()
|
||||
batch_results = async_runner.invoke({})
|
||||
|
||||
temp_res = {}
|
||||
|
||||
for i, (summary, document) in enumerate(zip(batch_results, docs)):
|
||||
temp_res[summary] = {
|
||||
"id": i,
|
||||
"summary": summary,
|
||||
"document": document
|
||||
}
|
||||
for i in range(1, len(docs)+1):
|
||||
docs[i-1]["summary"] = batch_results.get(f"chunk{i}").content
|
||||
|
||||
state["descriptions"] = temp_res
|
||||
state.update({self.output[0]: docs})
|
||||
|
||||
return state
|
||||
|
||||
@ -52,9 +52,9 @@ class GenerateAnswerNodeKLevel(BaseNode):
|
||||
self.additional_info = node_config.get("additional_info")
|
||||
|
||||
def execute(self, state: dict) -> dict:
|
||||
input_keys = self.get_input_keys(state)
|
||||
input_data = [state[key] for key in input_keys]
|
||||
user_prompt = input_data[0]
|
||||
self.logger.info(f"--- Executing {self.node_name} Node ---")
|
||||
|
||||
user_prompt = state.get("user_prompt")
|
||||
|
||||
if self.node_config.get("schema", None) is not None:
|
||||
if isinstance(self.llm_model, (ChatOpenAI, ChatMistralAI)):
|
||||
@ -113,19 +113,18 @@ class GenerateAnswerNodeKLevel(BaseNode):
|
||||
else:
|
||||
answer_db = client.query(
|
||||
collection_name="vectorial_collection",
|
||||
query_text=state["question"]
|
||||
query_text=user_prompt
|
||||
)
|
||||
|
||||
## TODO: from the id get the data
|
||||
results_db = [elem for elem in state[answer_db]]
|
||||
|
||||
chains_dict = {}
|
||||
for i, chunk in enumerate(tqdm(results_db,
|
||||
elems =[state.get("docs")[elem.id-1] for elem in answer_db if elem.score>0.5]
|
||||
|
||||
for i, chunk in enumerate(tqdm(elems,
|
||||
desc="Processing chunks", disable=not self.verbose)):
|
||||
prompt = PromptTemplate(
|
||||
template=template_chunks_prompt,
|
||||
input_variables=["question"],
|
||||
partial_variables={"context": chunk,
|
||||
input_variables=["format_instructions"],
|
||||
partial_variables={"context": chunk.get("document"),
|
||||
"chunk_id": i + 1,
|
||||
}
|
||||
)
|
||||
@ -133,7 +132,7 @@ class GenerateAnswerNodeKLevel(BaseNode):
|
||||
chains_dict[chain_name] = prompt | self.llm_model
|
||||
|
||||
async_runner = RunnableParallel(**chains_dict)
|
||||
batch_results = async_runner.invoke({"question": user_prompt})
|
||||
batch_results = async_runner.invoke({"format_instructions": user_prompt})
|
||||
|
||||
merge_prompt = PromptTemplate(
|
||||
template=template_merge_prompt,
|
||||
|
||||
@ -40,8 +40,9 @@ class RAGNode(BaseNode):
|
||||
)
|
||||
|
||||
def execute(self, state: dict) -> dict:
|
||||
|
||||
if self.node_config.get("client_type") == "memory":
|
||||
self.logger.info(f"--- Executing {self.node_name} Node ---")
|
||||
|
||||
if self.node_config.get("client_type") in ["memory", None]:
|
||||
client = QdrantClient(":memory:")
|
||||
elif self.node_config.get("client_type") == "local_db":
|
||||
client = QdrantClient(path="path/to/db")
|
||||
@ -50,8 +51,8 @@ class RAGNode(BaseNode):
|
||||
else:
|
||||
raise ValueError("client_type provided not correct")
|
||||
|
||||
docs = [elem.get("summary") for elem in state.get("descriptions", {})]
|
||||
ids = [elem.get("id") for elem in state.get("descriptions", {})]
|
||||
docs = [elem.get("summary") for elem in state.get("docs")]
|
||||
ids = [i for i in range(1, len(state.get("docs"))+1)]
|
||||
|
||||
if state.get("embeddings"):
|
||||
import openai
|
||||
|
||||
@ -5,6 +5,6 @@ description node prompts
|
||||
DESCRIPTION_NODE_PROMPT = """
|
||||
You are a scraper and you have just scraped the
|
||||
following content from a website. \n
|
||||
Please provide a description summary of maximum of 10 words
|
||||
Please provide a description summary of maximum of 20 words
|
||||
Content of the website: {content}
|
||||
"""
|
||||
@ -2,6 +2,7 @@
|
||||
Generate answer node prompts
|
||||
"""
|
||||
|
||||
|
||||
TEMPLATE_CHUNKS_MD = """
|
||||
You are a website scraper and you have just scraped the
|
||||
following content from a website converted in markdown format.
|
||||
@ -32,6 +33,7 @@ following content from a website converted in markdown format.
|
||||
You are now asked to answer a user question about the content you have scraped.\n
|
||||
You have scraped many chunks since the website is big and now you are asked to merge them into a single answer without repetitions (if there are any).\n
|
||||
Make sure that if a maximum number of items is specified in the instructions that you get that maximum number and do not exceed it. \n
|
||||
The structure should be coherent. \n
|
||||
Make sure the output format is a valid JSON and does not contain errors. \n
|
||||
OUTPUT INSTRUCTIONS: {format_instructions}\n
|
||||
USER QUESTION: {question}\n
|
||||
|
||||
Loading…
Reference in New Issue
Block a user