CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

search_link_node prompt moved

Browse Source
This commit is contained in:
Matteo Vedovati 2024-08-11 11:34:49 +02:00
parent 86a4903e99
commit ef966525c3
3 changed files with 30 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrapegraphai/nodes/search_link_node.py
View File

@ -10,6 +10,7 @@ from langchain_core.output_parsers import JsonOutputParser
from langchain_core.runnables import RunnableParallel
from ..utils.logging import get_logger
from .base_node import BaseNode
from ..prompts import prompt_relevant_links
class SearchLinkNode(BaseNode):
@ -83,29 +84,6 @@ class SearchLinkNode(BaseNode):
except Exception as e:
# Fallback approach: Using the LLM to extract links
self.logger.error(f"Error extracting links: {e}. Falling back to LLM.")
prompt_relevant_links = """
You are a website scraper and you have just scraped the following content from a website.
Content: {content}
Assume relevance broadly, including any links that might be related or potentially useful
in relation to the task.
Sort it in order of importance, the first one should be the most important one, the last one
the least important
Please list only valid URLs and make sure to err on the side of inclusion if it's uncertain
whether the content at the link is directly relevant.
Output only a list of relevant links in the format:
[
"link1",
"link2",
"link3",
.
.
.
]
"""
merge_prompt = PromptTemplate(
template=prompt_relevant_links,

3
scrapegraphai/prompts/__init__.py
View File

@ -8,4 +8,5 @@ from .generate_answer_node_pdf_prompts import template_chunks_pdf, template_no_c
from .generate_answer_node_omni_prompts import template_chunks_omni, template_no_chunk_omni, template_merge_omni
from .merge_answer_node_prompts import template_combined
from .robots_node_prompts import template_robot
from .search_internet_node_prompts import search_internet_template
from .search_internet_node_prompts import search_internet_template
from .search_link_node_prompts import prompt_relevant_links

27
scrapegraphai/prompts/search_link_node_prompts.py Normal file
View File

@ -0,0 +1,27 @@
"""
Search link node prompts helper
"""
prompt_relevant_links = """
You are a website scraper and you have just scraped the following content from a website.
Content: {content}
Assume relevance broadly, including any links that might be related or potentially useful
in relation to the task.
Sort it in order of importance, the first one should be the most important one, the last one
the least important
Please list only valid URLs and make sure to err on the side of inclusion if it's uncertain
whether the content at the link is directly relevant.
Output only a list of relevant links in the format:
[
"link1",
"link2",
"link3",
.
.
.
]
"""