diff --git a/examples/openai/md_scraper_openai.py b/examples/openai/md_scraper_openai.py index 3a152243..118e7d59 100644 --- a/examples/openai/md_scraper_openai.py +++ b/examples/openai/md_scraper_openai.py @@ -37,7 +37,7 @@ graph_config = { # ************************************************ md_scraper_graph = DocumentScraperGraph( - prompt="List me all the authors, title and genres of the books", + prompt="List me all the projects", source=text, # Pass the content of the file, not the file object config=graph_config ) diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index fd2f3810..7c80373d 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -85,10 +85,13 @@ class ParseNode(BaseNode): else: docs_transformed = docs_transformed[0] - link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) + try: + link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) + except Exception as e: + link_urls, img_urls = "", "" chunk_size = self.chunk_size - chunk_size = min(chunk_size - 500, int(chunk_size * 0.75)) + chunk_size = min(chunk_size - 500, int(chunk_size * 0.8)) if isinstance(docs_transformed, Document): chunks = split_text_into_chunks(text=docs_transformed.page_content,