From d79036149a3197a385b73553f29df66d36480c38 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 6 Jun 2024 21:35:52 +0200 Subject: [PATCH] feat: add caching --- scrapegraphai/nodes/rag_node.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapegraphai/nodes/rag_node.py b/scrapegraphai/nodes/rag_node.py index bc239ebb..9c4dc164 100644 --- a/scrapegraphai/nodes/rag_node.py +++ b/scrapegraphai/nodes/rag_node.py @@ -99,14 +99,15 @@ class RAGNode(BaseNode): ) embeddings = self.embedder_model - if self.node_config.get("cache", False): - index = FAISS.from_documents(chunked_docs, embeddings) - folder_name = "cache" + folder_name = "cache" - if not os.path.exists(folder_name): - os.makedirs(folder_name) + if self.node_config.get("cache", False) and not os.path.exists(folder_name): + index = FAISS.from_documents(chunked_docs, embeddings) + os.makedirs(folder_name) index.save_local(folder_name) + if self.node_config.get("cache", False) and os.path.exists(folder_name): + index = FAISS.load_local(folder_path=folder_name, embeddings=embeddings) else: index = FAISS.from_documents(chunked_docs, embeddings)