CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(FetchNode) add OpenAI optimization to GPT models on Azure

Browse Source 
closes #547
This commit is contained in:
Federico Aguzzi 2024-08-17 10:33:25 +02:00
parent a6fcc1ea58
commit 5eed0a7654
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapegraphai/nodes/fetch_node.py
View File

@ -4,7 +4,7 @@ FetchNode Module
import json
from typing import List, Optional
from langchain_openai import ChatOpenAI
from langchain_openai import ChatOpenAI, AzureChatOpenAI
import pandas as pd
import requests
from langchain_community.document_loaders import PyPDFLoader
@ -221,7 +221,7 @@ class FetchNode(BaseNode):
parsed_content = source
if isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator:
if (isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI)) and not self.script_creator or self.force and not self.script_creator:
parsed_content = convert_to_md(source)
else:
parsed_content = source
@ -258,7 +258,7 @@ class FetchNode(BaseNode):
if not self.cut:
parsed_content = cleanup_html(response, source)
if (isinstance(self.llm_model, ChatOpenAI)
if ((isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI))
and not self.script_creator) or (self.force and not self.script_creator):
parsed_content = convert_to_md(source, parsed_content)
@ -287,7 +287,7 @@ class FetchNode(BaseNode):
raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
parsed_content = document[0].page_content
if isinstance(self.llm_model, ChatOpenAI) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:
if (isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI)) and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:
parsed_content = convert_to_md(document[0].page_content, parsed_content)
compressed_document = [