CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-04 21:00:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #584 from ScrapeGraphAI/revert-583-anthropic-refactoring
Some checks failed
Release / Build (push) Has been cancelled
Details
Release / Release (push) Has been cancelled
Details

Browse Source 
Revert "Anthropic refactoring"
This commit is contained in:
Marco Vinciguerra 2024-08-25 10:54:35 +02:00 committed by GitHub
commit 869bbd798e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 23 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/anthropic/csv_scraper_graph_multi_anthropic.py → examples/anthropic/csv_scraper_graph_multi_haiku.py
View File

@ -26,7 +26,7 @@ text = pd.read_csv(file_path)
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000},
}

2
examples/anthropic/csv_scraper_anthropic.py → examples/anthropic/csv_scraper_haiku.py
View File

@ -32,7 +32,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/custom_graph_anthropic.py → examples/anthropic/custom_graph_haiku.py
View File

@ -18,7 +18,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/json_scraper_anthropic.py → examples/anthropic/json_scraper_haiku.py
View File

@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file:
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/json_scraper_multi_anthropic.py → examples/anthropic/json_scraper_multi_haiku.py
View File

@ -11,7 +11,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/pdf_scraper_graph_anthropic.py → examples/anthropic/pdf_scraper_graph_haiku.py
View File

@ -14,7 +14,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/pdf_scraper_multi_anthropic.py → examples/anthropic/pdf_scraper_multi_haiku.py
View File

@ -11,7 +11,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/scrape_plain_text_anthropic.py → examples/anthropic/scrape_plain_text_haiku.py
View File

@ -28,7 +28,7 @@ with open(file_path, 'r', encoding="utf-8") as file:
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/script_generator_anthropic.py → examples/anthropic/script_generator_haiku.py
View File

@ -16,7 +16,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/script_multi_generator_anthropic.py → examples/anthropic/script_multi_generator_haiku.py
View File

@ -16,7 +16,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
"library": "beautifulsoup"

2
examples/anthropic/search_graph_anthropic.py → examples/anthropic/search_graph_haiku.py
View File

@ -15,7 +15,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

5
examples/anthropic/search_graph_schema_anthropic.py → examples/anthropic/search_graph_schema_haiku.py
View File

@ -27,9 +27,8 @@ class Dishes(BaseModel):
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"max_tokens": 4000
},
"model": "claude-3-haiku-20240307",
"max_tokens": 4000},
}
# ************************************************

7
examples/anthropic/search_link_graph_anthropic.py → examples/anthropic/search_link_graph_haiku.py
View File

@ -29,11 +29,8 @@ embedder_model_instance = AzureOpenAIEmbeddings(
# ************************************************
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"max_tokens": 4000
},
"llm": {"model_instance": llm_model_instance},
"embeddings": {"model_instance": embedder_model_instance}
}
# ************************************************

2
examples/anthropic/smart_scraper_anthropic.py → examples/anthropic/smart_scraper_haiku.py
View File

@ -19,7 +19,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/smart_scraper_multi_anthropic.py → examples/anthropic/smart_scraper_multi_haiku.py
View File

@ -17,7 +17,7 @@ load_dotenv()
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

2
examples/anthropic/smart_scraper_schema_anthropic.py → examples/anthropic/smart_scraper_schema_haiku.py
View File

@ -33,7 +33,7 @@ class Projects(BaseModel):
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000},
}

2
examples/anthropic/xml_scraper_graph_multi_anthropic.py → examples/anthropic/xml_scraper_graph_multi_haiku.py
View File

@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file:
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000},
}

2
examples/anthropic/xml_scraper_anthropic.py → examples/anthropic/xml_scraper_haiku.py
View File

@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file:
graph_config = {
"llm": {
"api_key": os.getenv("ANTHROPIC_API_KEY"),
"model": "anthropic/claude-3-haiku-20240307",
"model": "claude-3-haiku-20240307",
"max_tokens": 4000
},
}

7
scrapegraphai/graphs/abstract_graph.py
View File

@ -142,7 +142,7 @@ class AbstractGraph(ABC):
known_models = {"chatgpt","gpt","openai", "azure_openai", "google_genai",
"ollama", "oneapi", "nvidia", "groq", "google_vertexai",
"bedrock", "mistralai", "hugging_face", "deepseek", "ernie",
"fireworks", "anthropic"}
"fireworks", "claude-3-"}
if llm_params["model"].split("/")[0] not in known_models and llm_params["model"].split("-")[0] not in known_models:
raise ValueError(f"Model '{llm_params['model']}' is not supported")
@ -172,9 +172,8 @@ class AbstractGraph(ABC):
token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"]
return handle_model(model_name, "ollama", token_key)
elif "anthropic" in llm_params["model"]:
model_name = llm_params["model"].split("anthropic/")[-1]
return handle_model(model_name, "anthropic", model_name)
elif "claude-3-" in llm_params["model"]:
return handle_model(llm_params["model"], "anthropic", "claude3")
elif llm_params["model"].startswith("mistral"):
model_name = llm_params["model"].split("/")[-1]