CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: model count

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-08-16 17:38:55 +02:00
parent 6730797008
commit faef3186f7
4 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/local_models/smart_scraper_ollama.py
View File

@ -9,7 +9,7 @@ from scrapegraphai.utils import prettify_exec_info
graph_config = { graph_config = {
"llm": { "llm": {
"model": "ollama/llama3.1", "model": "ollama/mistral",
"temperature": 0, "temperature": 0,
"format": "json", # Ollama needs the format to be specified explicitly "format": "json", # Ollama needs the format to be specified explicitly
# "base_url": "http://localhost:11434", # set ollama URL arbitrarily # "base_url": "http://localhost:11434", # set ollama URL arbitrarily

2
scrapegraphai/graphs/abstract_graph.py
View File

@ -149,7 +149,7 @@ class AbstractGraph(ABC):
known_models = ["openai", "azure_openai", "google_genai", "ollama", "oneapi", "nvidia", "groq", "google_vertexai", "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", "fireworks"] known_models = ["openai", "azure_openai", "google_genai", "ollama", "oneapi", "nvidia", "groq", "google_vertexai", "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", "fireworks"]
if llm_params["model"] not in known_models: if llm_params["model"].split("/")[0] not in known_models:
raise ValueError(f"Model '{llm_params['model']}' is not supported") raise ValueError(f"Model '{llm_params['model']}' is not supported")
try: try:

1
scrapegraphai/helpers/models_tokens.py
View File

@ -62,6 +62,7 @@ models_tokens = {
"scrapegraph": 8192, "scrapegraph": 8192,
"llava": 4096, "llava": 4096,
"mixtral:8x22b-instruct": 65536, "mixtral:8x22b-instruct": 65536,
"mistral":8192,
"mistral-openorca": 32000, "mistral-openorca": 32000,
"nomic-embed-text": 8192, "nomic-embed-text": 8192,
"nous-hermes2:34b": 4096, "nous-hermes2:34b": 4096,

1
scrapegraphai/nodes/parse_node.py
View File

@ -80,6 +80,7 @@ class ParseNode(BaseNode):
docs_transformed = docs_transformed[0] docs_transformed = docs_transformed[0]
if isinstance(docs_transformed, Document): if isinstance(docs_transformed, Document):
chunks = chunk(text=docs_transformed.page_content, chunks = chunk(text=docs_transformed.page_content,
chunk_size=self.node_config.get("chunk_size", 4096)-250, chunk_size=self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda text: len(text.split()), token_counter=lambda text: len(text.split()),