CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-23 21:00:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: template names

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-05-18 02:42:25 +02:00
parent 05e511e36f
commit b82f33aee7
8 changed files with 32 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapegraphai/helpers/generate_answer_node_csv_prompts.py
View File

@ -1,7 +1,7 @@
""" """
Generate answer csv schema Generate answer csv schema
""" """
template_chunks = """ template_chunks_csv = """
You are a scraper and you have just scraped the You are a scraper and you have just scraped the
following content from a csv. following content from a csv.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -12,7 +12,7 @@ Output instructions: {format_instructions}\n
Content of {chunk_id}: {context}. \n Content of {chunk_id}: {context}. \n
""" """
template_no_chunks = """ template_no_chunks_csv = """
You are a csv scraper and you have just scraped the You are a csv scraper and you have just scraped the
following content from a csv. following content from a csv.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -23,7 +23,7 @@ User question: {question}\n
csv content: {context}\n csv content: {context}\n
""" """
template_merge = """ template_merge_csv = """
You are a csv scraper and you have just scraped the You are a csv scraper and you have just scraped the
following content from a csv. following content from a csv.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n

6
scrapegraphai/helpers/generate_answer_node_omni_prompts.py
View File

@ -2,7 +2,7 @@
Generate answer node omni prompts helper Generate answer node omni prompts helper
""" """
template_chunks = """ template_chunks_omni = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -13,7 +13,7 @@ Output instructions: {format_instructions}\n
Content of {chunk_id}: {context}. \n Content of {chunk_id}: {context}. \n
""" """
template_no_chunks = """ template_no_chunk_omni = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -26,7 +26,7 @@ Website content: {context}\n
Image descriptions: {img_desc}\n Image descriptions: {img_desc}\n
""" """
template_merge = """ template_merge_omni = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n

6
scrapegraphai/helpers/generate_answer_node_pdf_prompts.py
View File

@ -1,7 +1,7 @@
""" """
Generate anwer node pdf prompt Generate anwer node pdf prompt
""" """
template_chunks = """ template_chunks_pdf = """
You are a scraper and you have just scraped the You are a scraper and you have just scraped the
following content from a PDF. following content from a PDF.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -12,7 +12,7 @@ template_chunks = """
Content of {chunk_id}: {context}. \n Content of {chunk_id}: {context}. \n
""" """
template_no_chunks = """ template_no_chunks_pdf = """
You are a PDF scraper and you have just scraped the You are a PDF scraper and you have just scraped the
following content from a PDF. following content from a PDF.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -23,7 +23,7 @@ template_no_chunks = """
PDF content: {context}\n PDF content: {context}\n
""" """
template_merge = """ template_merge_pdf = """
You are a PDF scraper and you have just scraped the You are a PDF scraper and you have just scraped the
following content from a PDF. following content from a PDF.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n

10
scrapegraphai/helpers/generate_answer_node_prompts.py
View File

@ -1,7 +1,7 @@
""" """
Generate answer node prompts Generate answer node prompts
""" """
template_chunks = """ template_chunks_gen_answ = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -12,7 +12,7 @@ Output instructions: {format_instructions}\n
Content of {chunk_id}: {context}. \n Content of {chunk_id}: {context}. \n
""" """
template_chunks_with_schema = """ template_chunks_with_schema_gen_answ = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -24,7 +24,7 @@ Output instructions: {format_instructions}\n
Content of {chunk_id}: {context}. \n Content of {chunk_id}: {context}. \n
""" """
template_no_chunks = """ template_no_chunks_gen_answ = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -35,7 +35,7 @@ User question: {question}\n
Website content: {context}\n Website content: {context}\n
""" """
template_no_chunks_with_schema = """ template_no_chunks_with_schema_gen_answ = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n
@ -48,7 +48,7 @@ Website content: {context}\n
""" """
template_merge = """ template_merge_gen_answ = """
You are a website scraper and you have just scraped the You are a website scraper and you have just scraped the
following content from a website. following content from a website.
You are now asked to answer a user question about the content you have scraped.\n You are now asked to answer a user question about the content you have scraped.\n

8
scrapegraphai/nodes/generate_answer_csv_node.py
View File

@ -10,7 +10,7 @@ from langchain.prompts import PromptTemplate
from langchain_core.output_parsers import JsonOutputParser from langchain_core.output_parsers import JsonOutputParser
from langchain_core.runnables import RunnableParallel from langchain_core.runnables import RunnableParallel
from ..helpers.generate_answer_node_csv_prompts import template_chunks, template_no_chunks, template_merge from ..helpers.generate_answer_node_csv_prompts import template_chunks_csv, template_no_chunks_csv, template_merge_csv
# Imports from the library # Imports from the library
from .base_node import BaseNode from .base_node import BaseNode
@ -94,14 +94,14 @@ class GenerateAnswerCSVNode(BaseNode):
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)): for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
if len(doc) == 1: if len(doc) == 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_no_chunks, template=template_no_chunks_csv,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"format_instructions": format_instructions}, "format_instructions": format_instructions},
) )
else: else:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_chunks, template=template_chunks_csv,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"chunk_id": i + 1, "chunk_id": i + 1,
@ -119,7 +119,7 @@ class GenerateAnswerCSVNode(BaseNode):
answer = map_chain.invoke({"question": user_prompt}) answer = map_chain.invoke({"question": user_prompt})
# Merge the answers from the chunks # Merge the answers from the chunks
merge_prompt = PromptTemplate( merge_prompt = PromptTemplate(
template=template_merge, template=template_merge_csv,
input_variables=["context", "question"], input_variables=["context", "question"],
partial_variables={"format_instructions": format_instructions}, partial_variables={"format_instructions": format_instructions},
) )

12
scrapegraphai/nodes/generate_answer_node.py
View File

@ -13,7 +13,7 @@ from langchain_core.runnables import RunnableParallel
# Imports from the library # Imports from the library
from .base_node import BaseNode from .base_node import BaseNode
from ..helpers import template_chunks, template_no_chunks, template_merge, template_chunks_with_schema, template_no_chunks_with_schema from ..helpers import template_chunks_gen_answ, template_no_chunks_gen_answ, template_merge_gen_answ, template_chunks_with_schema_gen_answ, template_chunks_with_schema_gen_answ
class GenerateAnswerNode(BaseNode): class GenerateAnswerNode(BaseNode):
""" """
@ -77,13 +77,13 @@ class GenerateAnswerNode(BaseNode):
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)): for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
if self.node_config["schema"] is None and len(doc) == 1: if self.node_config["schema"] is None and len(doc) == 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_no_chunks, template=template_no_chunks_gen_answ,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"format_instructions": format_instructions}) "format_instructions": format_instructions})
elif self.node_config["schema"] is not None and len(doc) == 1: elif self.node_config["schema"] is not None and len(doc) == 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_no_chunks_with_schema, template=template_chunks_with_schema_gen_answ,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"format_instructions": format_instructions, "format_instructions": format_instructions,
@ -91,14 +91,14 @@ class GenerateAnswerNode(BaseNode):
}) })
elif self.node_config["schema"] is None and len(doc) > 1: elif self.node_config["schema"] is None and len(doc) > 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_chunks, template=template_chunks_gen_answ,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"chunk_id": i + 1, "chunk_id": i + 1,
"format_instructions": format_instructions}) "format_instructions": format_instructions})
elif self.node_config["schema"] is not None and len(doc) > 1: elif self.node_config["schema"] is not None and len(doc) > 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_chunks_with_schema, template=template_chunks_with_schema_gen_answ,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"chunk_id": i + 1, "chunk_id": i + 1,
@ -116,7 +116,7 @@ class GenerateAnswerNode(BaseNode):
answer = map_chain.invoke({"question": user_prompt}) answer = map_chain.invoke({"question": user_prompt})
# Merge the answers from the chunks # Merge the answers from the chunks
merge_prompt = PromptTemplate( merge_prompt = PromptTemplate(
template=template_merge, template=template_merge_gen_answ,
input_variables=["context", "question"], input_variables=["context", "question"],
partial_variables={"format_instructions": format_instructions}, partial_variables={"format_instructions": format_instructions},
) )

9
scrapegraphai/nodes/generate_answer_omni_node.py
View File

@ -13,8 +13,7 @@ from langchain_core.runnables import RunnableParallel
# Imports from the library # Imports from the library
from .base_node import BaseNode from .base_node import BaseNode
from ..helpers.generate_answer_node_omni_prompts import template_no_chunk_omni, template_chunks_omni, template_merge_omni
from ..helpers.generate_answer_node_omni_prompts import template_chunks, template_no_chunks, template_merge
class GenerateAnswerOmniNode(BaseNode): class GenerateAnswerOmniNode(BaseNode):
""" """
@ -82,7 +81,7 @@ class GenerateAnswerOmniNode(BaseNode):
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)): for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
if len(doc) == 1: if len(doc) == 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_no_chunks, template=template_no_chunk_omni,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"format_instructions": format_instructions, "format_instructions": format_instructions,
@ -90,7 +89,7 @@ class GenerateAnswerOmniNode(BaseNode):
) )
else: else:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_chunks, template=template_chunks_omni,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"chunk_id": i + 1, "chunk_id": i + 1,
@ -108,7 +107,7 @@ class GenerateAnswerOmniNode(BaseNode):
answer = map_chain.invoke({"question": user_prompt}) answer = map_chain.invoke({"question": user_prompt})
# Merge the answers from the chunks # Merge the answers from the chunks
merge_prompt = PromptTemplate( merge_prompt = PromptTemplate(
template=template_merge, template=template_merge_omni,
input_variables=["context", "question"], input_variables=["context", "question"],
partial_variables={ partial_variables={
"format_instructions": format_instructions, "format_instructions": format_instructions,

8
scrapegraphai/nodes/generate_answer_pdf_node.py
View File

@ -12,7 +12,7 @@ from langchain_core.runnables import RunnableParallel
# Imports from the library # Imports from the library
from .base_node import BaseNode from .base_node import BaseNode
from ..helpers.generate_answer_node_pdf_prompts import template_chunks, template_no_chunks, template_merge from ..helpers.generate_answer_node_pdf_prompts import template_chunks_pdf, template_no_chunks_pdf, template_merge_pdf
class GenerateAnswerPDFNode(BaseNode): class GenerateAnswerPDFNode(BaseNode):
""" """
@ -93,14 +93,14 @@ class GenerateAnswerPDFNode(BaseNode):
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)): for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
if len(doc) == 1: if len(doc) == 1:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_no_chunks, template=template_no_chunks_pdf,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"format_instructions": format_instructions}, "format_instructions": format_instructions},
) )
else: else:
prompt = PromptTemplate( prompt = PromptTemplate(
template=template_chunks, template=template_chunks_pdf,
input_variables=["question"], input_variables=["question"],
partial_variables={"context": chunk.page_content, partial_variables={"context": chunk.page_content,
"chunk_id": i + 1, "chunk_id": i + 1,
@ -118,7 +118,7 @@ class GenerateAnswerPDFNode(BaseNode):
answer = map_chain.invoke({"question": user_prompt}) answer = map_chain.invoke({"question": user_prompt})
# Merge the answers from the chunks # Merge the answers from the chunks
merge_prompt = PromptTemplate( merge_prompt = PromptTemplate(
template=template_merge, template=template_merge_pdf,
input_variables=["context", "question"], input_variables=["context", "question"],
partial_variables={"format_instructions": format_instructions}, partial_variables={"format_instructions": format_instructions},
) )