From dcd216e3457bdbbbc7b8dc27783866b748e322fa Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 15 Jun 2024 19:51:20 +0200 Subject: [PATCH 1/3] fix: removed duplicate from ollama dictionary Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com> --- scrapegraphai/helpers/models_tokens.py | 71 ++++++++++---------------- 1 file changed, 27 insertions(+), 44 deletions(-) diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 510777fe..ee20e9b9 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -35,50 +35,33 @@ models_tokens = { "gemini-1.5-pro-latest":128000, "models/embedding-001": 2048 }, - - "ollama": { - "command-r": 12800, - "command-r-plus": 12800, - "codellama": 16000, - "dbrx": 32768, - "dbrx:instruct": 32768, - "deepseek-coder:33b": 16000, - "dolphin-mixtral": 32000, - "llama2": 4096, - "llama3": 8192, - "llama3:70b-instruct": 8192, - "llava": 4096, - "llava:34b": 4096, - "llava_next": 4096, - "mistral": 8192, - "falcon": 2048, - "codellama": 16000, - "dolphin-mixtral": 32000, - "mistral-openorca": 32000, - "stablelm-zephyr": 8192, - "command-r-plus": 12800, - "command-r": 12800, - "mistral:7b-instruct": 32768, - "mistral-openorca": 32000, - "mixtral:8x22b-instruct": 65536, - "nous-hermes2:34b": 4096, - "orca-mini": 2048, - "phi3:3.8b": 12800, - "phi3:14b": 12800, - "qwen:0.5b": 32000, - "qwen:1.8b": 32000, - "qwen:4b": 32000, - "qwen:14b": 32000, - "qwen:32b": 32000, - "qwen:72b": 32000, - "qwen:110b": 32000, - "stablelm-zephyr": 8192, - "wizardlm2:8x22b": 65536, - # embedding models - "nomic-embed-text": 8192, - "snowflake-arctic-embed:335m": 8192, - "snowflake-arctic-embed:l": 8192, - "mxbai-embed-large": 512, + "ollama": { "command-r": 12800, + "codellama": 16000, + "dbrx": 32768, + "deepseek-coder:33b": 16000, + "falcon": 2048, + "llama2": 4096, + "llama3": 8192, + "scrapegraph": 8192, + "llava": 4096, + "mixtral:8x22b-instruct": 65536, + "mistral-openorca": 32000, + "nomic-embed-text": 8192, + "nous-hermes2:34b": 4096, + "orca-mini": 2048, + "phi3:3.8b": 12800, + "qwen:0.5b": 32000, + "qwen:1.8b": 32000, + "qwen:4b": 32000, + "qwen:14b": 32000, + "qwen:32b": 32000, + "qwen:72b": 32000, + "qwen:110b": 32000, + "stablelm-zephyr": 8192, + "wizardlm2:8x22b": 65536, + # embedding models + "snowflake-arctic-embed": 8192, + "mxbai-embed-large": 512 }, "oneapi": { "qwen-turbo": 16380 From 44fbd71742a57a4b10f22ed33781bb67aa77e58d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 15 Jun 2024 17:52:28 +0000 Subject: [PATCH 2/3] ci(release): 1.6.1 [skip ci] ## [1.6.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.6.0...v1.6.1) (2024-06-15) ### Bug Fixes * removed duplicate from ollama dictionary ([dcd216e](https://github.com/VinciGit00/Scrapegraph-ai/commit/dcd216e3457bdbbbc7b8dc27783866b748e322fa)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab78743..63d3fbe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.6.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.6.0...v1.6.1) (2024-06-15) + + +### Bug Fixes + +* removed duplicate from ollama dictionary ([dcd216e](https://github.com/VinciGit00/Scrapegraph-ai/commit/dcd216e3457bdbbbc7b8dc27783866b748e322fa)) + ## [1.6.0](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.5.7...v1.6.0) (2024-06-09) diff --git a/pyproject.toml b/pyproject.toml index 73039f64..bfbdeba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "scrapegraphai" -version = "1.6.0" +version = "1.6.1" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." From d8d5cd267ae89764490a427f94afc9df3beae536 Mon Sep 17 00:00:00 2001 From: shubihu <36021958+shubihu@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:26:33 +0800 Subject: [PATCH 3/3] Update abstract_graph.py fix: Incorrect API Key Error with OpenAI Proxy --- scrapegraphai/graphs/abstract_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index b5f3a681..87a00f9b 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -298,7 +298,7 @@ class AbstractGraph(ABC): google_api_key=llm_config["api_key"], model="models/embedding-001" ) if isinstance(self.llm_model, OpenAI): - return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key) + return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key, base_url=self.llm_model.openai_api_base) elif isinstance(self.llm_model, DeepSeek): return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key) elif isinstance(self.llm_model, AzureOpenAIEmbeddings): @@ -407,4 +407,4 @@ class AbstractGraph(ABC): """ Abstract method to execute the graph and return the result. """ - pass \ No newline at end of file + pass