CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add caching
Some checks are pending
/ build (3.10) (push) Waiting to run
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-06 21:35:52 +02:00
parent 7ed2fe8ef0
commit d79036149a
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapegraphai/nodes/rag_node.py
View File

@ -99,14 +99,15 @@ class RAGNode(BaseNode):
) )
embeddings = self.embedder_model embeddings = self.embedder_model
if self.node_config.get("cache", False): folder_name = "cache"
index = FAISS.from_documents(chunked_docs, embeddings)
folder_name = "cache"
if not os.path.exists(folder_name): if self.node_config.get("cache", False) and not os.path.exists(folder_name):
os.makedirs(folder_name) index = FAISS.from_documents(chunked_docs, embeddings)
os.makedirs(folder_name)
index.save_local(folder_name) index.save_local(folder_name)
if self.node_config.get("cache", False) and os.path.exists(folder_name):
index = FAISS.load_local(folder_path=folder_name, embeddings=embeddings)
else: else:
index = FAISS.from_documents(chunked_docs, embeddings) index = FAISS.from_documents(chunked_docs, embeddings)