CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: removed duplicate from ollama dictionary

Browse Source 
Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com>
This commit is contained in:
Marco Vinciguerra 2024-06-15 19:51:20 +02:00
parent c44b701d2d
commit dcd216e345
1 changed files with 27 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
scrapegraphai/helpers/models_tokens.py
View File

@ -35,50 +35,33 @@ models_tokens = {
"gemini-1.5-pro-latest":128000, "gemini-1.5-pro-latest":128000,
"models/embedding-001": 2048 "models/embedding-001": 2048
}, },
"ollama": { "command-r": 12800,
"ollama": { "codellama": 16000,
"command-r": 12800, "dbrx": 32768,
"command-r-plus": 12800, "deepseek-coder:33b": 16000,
"codellama": 16000, "falcon": 2048,
"dbrx": 32768, "llama2": 4096,
"dbrx:instruct": 32768, "llama3": 8192,
"deepseek-coder:33b": 16000, "scrapegraph": 8192,
"dolphin-mixtral": 32000, "llava": 4096,
"llama2": 4096, "mixtral:8x22b-instruct": 65536,
"llama3": 8192, "mistral-openorca": 32000,
"llama3:70b-instruct": 8192, "nomic-embed-text": 8192,
"llava": 4096, "nous-hermes2:34b": 4096,
"llava:34b": 4096, "orca-mini": 2048,
"llava_next": 4096, "phi3:3.8b": 12800,
"mistral": 8192, "qwen:0.5b": 32000,
"falcon": 2048, "qwen:1.8b": 32000,
"codellama": 16000, "qwen:4b": 32000,
"dolphin-mixtral": 32000, "qwen:14b": 32000,
"mistral-openorca": 32000, "qwen:32b": 32000,
"stablelm-zephyr": 8192, "qwen:72b": 32000,
"command-r-plus": 12800, "qwen:110b": 32000,
"command-r": 12800, "stablelm-zephyr": 8192,
"mistral:7b-instruct": 32768, "wizardlm2:8x22b": 65536,
"mistral-openorca": 32000, # embedding models
"mixtral:8x22b-instruct": 65536, "snowflake-arctic-embed": 8192,
"nous-hermes2:34b": 4096, "mxbai-embed-large": 512
"orca-mini": 2048,
"phi3:3.8b": 12800,
"phi3:14b": 12800,
"qwen:0.5b": 32000,
"qwen:1.8b": 32000,
"qwen:4b": 32000,
"qwen:14b": 32000,
"qwen:32b": 32000,
"qwen:72b": 32000,
"qwen:110b": 32000,
"stablelm-zephyr": 8192,
"wizardlm2:8x22b": 65536,
# embedding models
"nomic-embed-text": 8192,
"snowflake-arctic-embed:335m": 8192,
"snowflake-arctic-embed:l": 8192,
"mxbai-embed-large": 512,
}, },
"oneapi": { "oneapi": {
"qwen-turbo": 16380 "qwen-turbo": 16380