CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-04 21:00:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #555 from ScrapeGraphAI/547-different-prompt_tokens-size-between-azure-openai-and-openai

Browse Source 
fix(FetchNode): add OpenAI markdown optimization to Azure
This commit is contained in:
Marco Vinciguerra 2024-08-17 11:35:25 +02:00 committed by GitHub
commit 08782ff42d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapegraphai/nodes/fetch_node.py
View File

@ -4,7 +4,7 @@ FetchNode Module
import json
from typing import List, Optional
from langchain_openai import ChatOpenAI
from langchain_openai import ChatOpenAI, AzureChatOpenAI
import pandas as pd
import requests
from langchain_community.document_loaders import PyPDFLoader
@ -221,7 +221,7 @@ class FetchNode(BaseNode):
parsed_content = source
if isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator:
if (isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI)) and not self.script_creator or self.force and not self.script_creator:
parsed_content = convert_to_md(source)
else:
parsed_content = source
@ -258,7 +258,7 @@ class FetchNode(BaseNode):
if not self.cut:
parsed_content = cleanup_html(response, source)
if (isinstance(self.llm_model, ChatOpenAI)
if ((isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI))
and not self.script_creator) or (self.force and not self.script_creator):
parsed_content = convert_to_md(source, parsed_content)
@ -287,7 +287,7 @@ class FetchNode(BaseNode):
raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
parsed_content = document[0].page_content
if isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:
if (isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI)) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:
parsed_content = convert_to_md(document[0].page_content, parsed_content)
compressed_document = [