From 86f68770e920d800fb14d14ee34bf0d1a9cefd51 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 21 Sep 2024 08:25:22 +0200 Subject: [PATCH] fix: removed faiss --- README.md | 8 +++++++- pyproject.toml | 6 +++++- requirements-dev.lock | 4 ---- requirements.lock | 4 ---- scrapegraphai/nodes/generate_answer_node.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index bb1b8042..cf437203 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Additional dependecies can be added while installing the library: This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints. ```bash pip install scrapegraphai[other-language-models] - + ``` - Semantic Options: this group includes tools for advanced semantic processing, such as Graphviz. ```bash @@ -55,6 +55,12 @@ pip install scrapegraphai[other-language-models] pip install scrapegraphai[more-browser-options] ``` +- faiss Options: this group includes faiss integration + + ```bash + pip install scrapegraphai[faiss-cpu] + ``` + diff --git a/pyproject.toml b/pyproject.toml index 95839a28..10f52386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ dependencies = [ "langchain-aws>=0.1.3", "mistral-common>=1.4.0", "html2text>=2024.2.26", - "faiss-cpu>=1.8.0", "beautifulsoup4>=4.12.3", "pandas>=2.2.2", "python-dotenv>=1.0.1", @@ -100,6 +99,11 @@ screenshot_scraper = [ "pillow>=10.4.0", ] +# Group 5: Faiss CPU +faiss-cpu = [ + "faiss-cpu>=1.8.0", +] + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/requirements-dev.lock b/requirements-dev.lock index 34b10812..1d9d469a 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -80,8 +80,6 @@ docutils==0.19 exceptiongroup==1.2.2 # via anyio # via pytest -faiss-cpu==1.8.0.post1 - # via scrapegraphai fastapi==0.112.0 # via burr fastapi-pagination==0.12.26 @@ -251,7 +249,6 @@ narwhals==1.3.0 # via altair numpy==1.26.4 # via contourpy - # via faiss-cpu # via langchain # via langchain-aws # via langchain-community @@ -274,7 +271,6 @@ orjson==3.10.7 # via langsmith packaging==24.1 # via altair - # via faiss-cpu # via huggingface-hub # via langchain-core # via marshmallow diff --git a/requirements.lock b/requirements.lock index 24e52dee..84e25a0f 100644 --- a/requirements.lock +++ b/requirements.lock @@ -49,8 +49,6 @@ distro==1.9.0 # via openai exceptiongroup==1.2.2 # via anyio -faiss-cpu==1.8.0.post1 - # via scrapegraphai filelock==3.15.4 # via huggingface-hub # via transformers @@ -176,7 +174,6 @@ multiprocess==0.70.16 mypy-extensions==1.0.0 # via typing-inspect numpy==1.26.4 - # via faiss-cpu # via langchain # via langchain-aws # via langchain-community @@ -192,7 +189,6 @@ opencv-python-headless==4.10.0.84 orjson==3.10.6 # via langsmith packaging==24.1 - # via faiss-cpu # via huggingface-hub # via langchain-core # via marshmallow diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index 3d1c0481..44b8451f 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -91,7 +91,7 @@ class GenerateAnswerNode(BaseNode): if isinstance(self.llm_model, (ChatOpenAI, ChatMistralAI)): self.llm_model = self.llm_model.with_structured_output( - schema = self.node_config["schema"]) + schema = self.node_config["schema"]) output_parser = get_structured_output_parser(self.node_config["schema"]) format_instructions = "NA" else: