CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add dynamic caching

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-04 14:27:46 +02:00
parent 7a13a6819f
commit 7ed2fe8ef0
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scrapegraphai/nodes/rag_node.py
View File

@ -99,18 +99,18 @@ class RAGNode(BaseNode):
) )
embeddings = self.embedder_model embeddings = self.embedder_model
#------ if self.node_config.get("cache", False):
index = FAISS.from_documents(chunked_docs, embeddings) index = FAISS.from_documents(chunked_docs, embeddings)
# Define the folder name folder_name = "cache"
folder_name = "cache"
# Check if the folder exists, if not, create it if not os.path.exists(folder_name):
if not os.path.exists(folder_name): os.makedirs(folder_name)
os.makedirs(folder_name)
# Save the index to the folder index.save_local(folder_name)
index.save_local(folder_name) else:
index = FAISS.from_documents(chunked_docs, embeddings)
retriever = index.as_retriever() retriever = index.as_retriever()
#------
redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings) redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)
# similarity_threshold could be set, now k=20 # similarity_threshold could be set, now k=20
@ -133,4 +133,4 @@ class RAGNode(BaseNode):
self.logger.info("--- (tokens compressed and vector stored) ---") self.logger.info("--- (tokens compressed and vector stored) ---")
state.update({self.output[0]: compressed_docs}) state.update({self.output[0]: compressed_docs})
return state return state