CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add dynamic caching

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-04 14:27:46 +02:00
parent 7a13a6819f
commit 7ed2fe8ef0
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scrapegraphai/nodes/rag_node.py
View File

@ -99,18 +99,18 @@ class RAGNode(BaseNode):
)
embeddings = self.embedder_model
#------
index = FAISS.from_documents(chunked_docs, embeddings)
# Define the folder name
folder_name = "cache"
# Check if the folder exists, if not, create it
if not os.path.exists(folder_name):
os.makedirs(folder_name)
# Save the index to the folder
index.save_local(folder_name)
if self.node_config.get("cache", False):
index = FAISS.from_documents(chunked_docs, embeddings)
folder_name = "cache"
if not os.path.exists(folder_name):
os.makedirs(folder_name)
index.save_local(folder_name)
else:
index = FAISS.from_documents(chunked_docs, embeddings)
retriever = index.as_retriever()
#------
redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)
# similarity_threshold could be set, now k=20
@ -133,4 +133,4 @@ class RAGNode(BaseNode):
self.logger.info("--- (tokens compressed and vector stored) ---")
state.update({self.output[0]: compressed_docs})
return state
return state