CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: refactoring of mdscraper

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-10-11 08:32:41 +02:00
parent 528a9746fe
commit 3b7b701a89
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/openai/md_scraper_openai.py
View File

@ -37,7 +37,7 @@ graph_config = {
# ************************************************
md_scraper_graph = DocumentScraperGraph(
prompt="List me all the authors, title and genres of the books",
prompt="List me all the projects",
source=text, # Pass the content of the file, not the file object
config=graph_config
)

7
scrapegraphai/nodes/parse_node.py
View File

@ -85,10 +85,13 @@ class ParseNode(BaseNode):
else:
docs_transformed = docs_transformed[0]
link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
try:
link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
except Exception as e:
link_urls, img_urls = "", ""
chunk_size = self.chunk_size
chunk_size = min(chunk_size - 500, int(chunk_size * 0.75))
chunk_size = min(chunk_size - 500, int(chunk_size * 0.8))
if isinstance(docs_transformed, Document):
chunks = split_text_into_chunks(text=docs_transformed.page_content,