CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: add chinese embedding model

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-17 14:17:44 +02:00
parent 93342b4708
commit 03ffebc52d
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapegraphai/graphs/abstract_graph.py
View File

@ -320,7 +320,7 @@ class AbstractGraph(ABC):
elif "azure" in embedder_params["model"]: elif "azure" in embedder_params["model"]:
return AzureOpenAIEmbeddings() return AzureOpenAIEmbeddings()
elif "ollama" in embedder_params["model"]: elif "ollama" in embedder_params["model"]:
embedder_params["model"] = embedder_params["model"].split("ollama/")[-1] embedder_params["model"] = "/".join(embedder_params["model"].split("/")[1:])
try: try:
models_tokens["ollama"][embedder_params["model"]] models_tokens["ollama"][embedder_params["model"]]
except KeyError as exc: except KeyError as exc:

3
scrapegraphai/helpers/models_tokens.py
View File

@ -59,7 +59,8 @@ models_tokens = {
"qwen:110b": 32000, "qwen:110b": 32000,
"stablelm-zephyr": 8192, "stablelm-zephyr": 8192,
"wizardlm2:8x22b": 65536, "wizardlm2:8x22b": 65536,
# embedding models # embedding models
"shaw/dmeta-embedding-zh": 8192,
"snowflake-arctic-embed": 8192, "snowflake-arctic-embed": 8192,
"mxbai-embed-large": 512 "mxbai-embed-large": 512
}, },