From 58289eccc523814a2898650c41410f9a35b4e4c2 Mon Sep 17 00:00:00 2001
From: Alok Saboo <arsaboo@gmx.com>
Date: Wed, 22 May 2024 15:07:27 -0400
Subject: [PATCH] feat: Add new models and update existing ones

---
 scrapegraphai/helpers/models_tokens.py | 39 +++++++++++++++++---------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py
index f1780049..d84e1094 100644
--- a/scrapegraphai/helpers/models_tokens.py
+++ b/scrapegraphai/helpers/models_tokens.py
@@ -33,25 +33,36 @@ models_tokens = {
     },
 
     "ollama": {
-        "llama2": 4096,
-        "llama3": 8192,
-        "llava": 4096,
-        "llava_next": 4096,
-        "mistral": 8192,
-        "codellama": 16000,
-        "dolphin-mixtral": 32000,
-        "mistral-openorca": 32000,
-        "stablelm-zephyr": 8192,
-        "command-r-plus": 12800,
         "command-r": 12800,
-        "mistral:7b-instruct": 32768,
-        "llama3:70b-instruct": 8192,
-        "mixtral:8x22b-instruct": 65536,
-        "wizardlm2:8x22b": 65536,
+        "command-r-plus": 12800,
+        "codellama": 16000,
         "dbrx": 32768,
         "dbrx:instruct": 32768,
+        "deepseek-coder:33b": 16000,
+        "dolphin-mixtral": 32000,
+        "llama2": 4096,
+        "llama3": 8192,
+        "llama3:70b-instruct": 8192,
+        "llava": 4096,
+        "llava:34b": 4096,
+        "llava_next": 4096,
+        "mistral": 8192,
+        "mistral:7b-instruct": 32768,
+        "mistral-openorca": 32000,
+        "mixtral:8x22b-instruct": 65536,
         "nous-hermes2:34b": 4096,
         "orca-mini": 2048,
+        "phi3:3.8b": 12800,
+        "phi3:14b": 12800,
+        "qwen:0.5b": 32000,
+        "qwen:1.8b": 32000,
+        "qwen:4b": 32000,
+        "qwen:14b": 32000,
+        "qwen:32b": 32000,
+        "qwen:72b": 32000,
+        "qwen:110b": 32000,
+        "stablelm-zephyr": 8192,
+        "wizardlm2:8x22b": 65536,
         # embedding models
         "nomic-embed-text": 8192,
         "snowflake-arctic-embed:335m": 8192,