CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: add grok integration

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-08-06 15:17:49 +02:00
parent 0571b6da55
commit fa651d4cd9
1 changed files with 38 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
scrapegraphai/helpers/models_tokens.py
View File

@ -44,41 +44,43 @@ models_tokens = {
"gemini-1.5-pro-latest": 128000, "gemini-1.5-pro-latest": 128000,
"models/embedding-001": 2048 "models/embedding-001": 2048
}, },
"ollama": { "command-r": 12800, "ollama": {
"codellama": 16000, "grok-1": 8192,
"dbrx": 32768, "command-r": 12800,
"deepseek-coder:33b": 16000, "codellama": 16000,
"falcon": 2048, "dbrx": 32768,
"llama2": 4096, "deepseek-coder:33b": 16000,
"llama3": 8192, "falcon": 2048,
"llama3:70b": 8192, "llama2": 4096,
"llama3.1":128000, "llama3": 8192,
"llama3.1:70b": 128000, "llama3:70b": 8192,
"lama3.1:405b": 128000, "llama3.1":128000,
"scrapegraph": 8192, "llama3.1:70b": 128000,
"llava": 4096, "lama3.1:405b": 128000,
"mixtral:8x22b-instruct": 65536, "scrapegraph": 8192,
"mistral-openorca": 32000, "llava": 4096,
"nomic-embed-text": 8192, "mixtral:8x22b-instruct": 65536,
"nous-hermes2:34b": 4096, "mistral-openorca": 32000,
"orca-mini": 2048, "nomic-embed-text": 8192,
"phi3:3.8b": 12800, "nous-hermes2:34b": 4096,
"qwen:0.5b": 32000, "orca-mini": 2048,
"qwen:1.8b": 32000, "phi3:3.8b": 12800,
"qwen:4b": 32000, "qwen:0.5b": 32000,
"qwen:14b": 32000, "qwen:1.8b": 32000,
"qwen:32b": 32000, "qwen:4b": 32000,
"qwen:72b": 32000, "qwen:14b": 32000,
"qwen:110b": 32000, "qwen:32b": 32000,
"stablelm-zephyr": 8192, "qwen:72b": 32000,
"wizardlm2:8x22b": 65536, "qwen:110b": 32000,
# embedding models "stablelm-zephyr": 8192,
"shaw/dmeta-embedding-zh-small-q4": 8192, "wizardlm2:8x22b": 65536,
"shaw/dmeta-embedding-zh-q4": 8192, # embedding models
"chevalblanc/acge_text_embedding": 8192, "shaw/dmeta-embedding-zh-small-q4": 8192,
"martcreation/dmeta-embedding-zh": 8192, "shaw/dmeta-embedding-zh-q4": 8192,
"snowflake-arctic-embed": 8192, "chevalblanc/acge_text_embedding": 8192,
"mxbai-embed-large": 512 "martcreation/dmeta-embedding-zh": 8192,
"snowflake-arctic-embed": 8192,
"mxbai-embed-large": 512
}, },
"oneapi": { "oneapi": {
"qwen-turbo": 6000 "qwen-turbo": 6000
@ -147,6 +149,7 @@ models_tokens = {
"mistralai/Mistral-7B-Instruct-v0.2": 32000 "mistralai/Mistral-7B-Instruct-v0.2": 32000
}, },
"hugging_face": { "hugging_face": {
"xai-org/grok-1": 8192,
"meta-llama/Meta-Llama-3-8B": 8192, "meta-llama/Meta-Llama-3-8B": 8192,
"meta-llama/Meta-Llama-3-8B-Instruct": 8192, "meta-llama/Meta-Llama-3-8B-Instruct": 8192,
"meta-llama/Meta-Llama-3-70B": 8192, "meta-llama/Meta-Llama-3-70B": 8192,