From e3fbf01c5b48a28878a541df628e9703f249db7d Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra <88108002+VinciGit00@users.noreply.github.com> Date: Sun, 25 Aug 2024 10:54:19 +0200 Subject: [PATCH] Revert "Anthropic refactoring" --- ...multi_anthropic.py => csv_scraper_graph_multi_haiku.py} | 2 +- .../{csv_scraper_anthropic.py => csv_scraper_haiku.py} | 2 +- .../{custom_graph_anthropic.py => custom_graph_haiku.py} | 2 +- .../{json_scraper_anthropic.py => json_scraper_haiku.py} | 2 +- ...aper_multi_anthropic.py => json_scraper_multi_haiku.py} | 2 +- ...raper_graph_anthropic.py => pdf_scraper_graph_haiku.py} | 2 +- ...raper_multi_anthropic.py => pdf_scraper_multi_haiku.py} | 2 +- ..._plain_text_anthropic.py => scrape_plain_text_haiku.py} | 2 +- ...pt_generator_anthropic.py => script_generator_haiku.py} | 2 +- ...erator_anthropic.py => script_multi_generator_haiku.py} | 2 +- .../{search_graph_anthropic.py => search_graph_haiku.py} | 2 +- ...ph_schema_anthropic.py => search_graph_schema_haiku.py} | 5 ++--- ..._link_graph_anthropic.py => search_link_graph_haiku.py} | 7 ++----- .../{smart_scraper_anthropic.py => smart_scraper_haiku.py} | 2 +- ...per_multi_anthropic.py => smart_scraper_multi_haiku.py} | 2 +- ...r_schema_anthropic.py => smart_scraper_schema_haiku.py} | 2 +- ...multi_anthropic.py => xml_scraper_graph_multi_haiku.py} | 2 +- .../{xml_scraper_anthropic.py => xml_scraper_haiku.py} | 2 +- scrapegraphai/graphs/abstract_graph.py | 7 +++---- 19 files changed, 23 insertions(+), 28 deletions(-) rename examples/anthropic/{csv_scraper_graph_multi_anthropic.py => csv_scraper_graph_multi_haiku.py} (96%) rename examples/anthropic/{csv_scraper_anthropic.py => csv_scraper_haiku.py} (96%) rename examples/anthropic/{custom_graph_anthropic.py => custom_graph_haiku.py} (97%) rename examples/anthropic/{json_scraper_anthropic.py => json_scraper_haiku.py} (96%) rename examples/anthropic/{json_scraper_multi_anthropic.py => json_scraper_multi_haiku.py} (93%) rename examples/anthropic/{pdf_scraper_graph_anthropic.py => pdf_scraper_graph_haiku.py} (96%) rename examples/anthropic/{pdf_scraper_multi_anthropic.py => pdf_scraper_multi_haiku.py} (99%) rename examples/anthropic/{scrape_plain_text_anthropic.py => scrape_plain_text_haiku.py} (96%) rename examples/anthropic/{script_generator_anthropic.py => script_generator_haiku.py} (95%) rename examples/anthropic/{script_multi_generator_anthropic.py => script_multi_generator_haiku.py} (96%) rename examples/anthropic/{search_graph_anthropic.py => search_graph_haiku.py} (95%) rename examples/anthropic/{search_graph_schema_anthropic.py => search_graph_schema_haiku.py} (92%) rename examples/anthropic/{search_link_graph_anthropic.py => search_link_graph_haiku.py} (91%) rename examples/anthropic/{smart_scraper_anthropic.py => smart_scraper_haiku.py} (96%) rename examples/anthropic/{smart_scraper_multi_anthropic.py => smart_scraper_multi_haiku.py} (96%) rename examples/anthropic/{smart_scraper_schema_anthropic.py => smart_scraper_schema_haiku.py} (96%) rename examples/anthropic/{xml_scraper_graph_multi_anthropic.py => xml_scraper_graph_multi_haiku.py} (96%) rename examples/anthropic/{xml_scraper_anthropic.py => xml_scraper_haiku.py} (96%) diff --git a/examples/anthropic/csv_scraper_graph_multi_anthropic.py b/examples/anthropic/csv_scraper_graph_multi_haiku.py similarity index 96% rename from examples/anthropic/csv_scraper_graph_multi_anthropic.py rename to examples/anthropic/csv_scraper_graph_multi_haiku.py index fcc297ab..b833af01 100644 --- a/examples/anthropic/csv_scraper_graph_multi_anthropic.py +++ b/examples/anthropic/csv_scraper_graph_multi_haiku.py @@ -26,7 +26,7 @@ text = pd.read_csv(file_path) graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/csv_scraper_anthropic.py b/examples/anthropic/csv_scraper_haiku.py similarity index 96% rename from examples/anthropic/csv_scraper_anthropic.py rename to examples/anthropic/csv_scraper_haiku.py index 01a26a2b..2e0ebe81 100644 --- a/examples/anthropic/csv_scraper_anthropic.py +++ b/examples/anthropic/csv_scraper_haiku.py @@ -32,7 +32,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/custom_graph_anthropic.py b/examples/anthropic/custom_graph_haiku.py similarity index 97% rename from examples/anthropic/custom_graph_anthropic.py rename to examples/anthropic/custom_graph_haiku.py index 6d787484..cea14361 100644 --- a/examples/anthropic/custom_graph_anthropic.py +++ b/examples/anthropic/custom_graph_haiku.py @@ -18,7 +18,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/json_scraper_anthropic.py b/examples/anthropic/json_scraper_haiku.py similarity index 96% rename from examples/anthropic/json_scraper_anthropic.py rename to examples/anthropic/json_scraper_haiku.py index 05ee7fdf..2610b658 100644 --- a/examples/anthropic/json_scraper_anthropic.py +++ b/examples/anthropic/json_scraper_haiku.py @@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/json_scraper_multi_anthropic.py b/examples/anthropic/json_scraper_multi_haiku.py similarity index 93% rename from examples/anthropic/json_scraper_multi_anthropic.py rename to examples/anthropic/json_scraper_multi_haiku.py index c07fc54f..0327673b 100644 --- a/examples/anthropic/json_scraper_multi_anthropic.py +++ b/examples/anthropic/json_scraper_multi_haiku.py @@ -11,7 +11,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/pdf_scraper_graph_anthropic.py b/examples/anthropic/pdf_scraper_graph_haiku.py similarity index 96% rename from examples/anthropic/pdf_scraper_graph_anthropic.py rename to examples/anthropic/pdf_scraper_graph_haiku.py index 3e4191a6..61be06b4 100644 --- a/examples/anthropic/pdf_scraper_graph_anthropic.py +++ b/examples/anthropic/pdf_scraper_graph_haiku.py @@ -14,7 +14,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/pdf_scraper_multi_anthropic.py b/examples/anthropic/pdf_scraper_multi_haiku.py similarity index 99% rename from examples/anthropic/pdf_scraper_multi_anthropic.py rename to examples/anthropic/pdf_scraper_multi_haiku.py index 0c842787..974dd2f8 100644 --- a/examples/anthropic/pdf_scraper_multi_anthropic.py +++ b/examples/anthropic/pdf_scraper_multi_haiku.py @@ -11,7 +11,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/scrape_plain_text_anthropic.py b/examples/anthropic/scrape_plain_text_haiku.py similarity index 96% rename from examples/anthropic/scrape_plain_text_anthropic.py rename to examples/anthropic/scrape_plain_text_haiku.py index 7ebf84da..d3f36638 100644 --- a/examples/anthropic/scrape_plain_text_anthropic.py +++ b/examples/anthropic/scrape_plain_text_haiku.py @@ -28,7 +28,7 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/script_generator_anthropic.py b/examples/anthropic/script_generator_haiku.py similarity index 95% rename from examples/anthropic/script_generator_anthropic.py rename to examples/anthropic/script_generator_haiku.py index 160987cc..889ce0b5 100644 --- a/examples/anthropic/script_generator_anthropic.py +++ b/examples/anthropic/script_generator_haiku.py @@ -16,7 +16,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/script_multi_generator_anthropic.py b/examples/anthropic/script_multi_generator_haiku.py similarity index 96% rename from examples/anthropic/script_multi_generator_anthropic.py rename to examples/anthropic/script_multi_generator_haiku.py index c4b3f09b..f7c69010 100644 --- a/examples/anthropic/script_multi_generator_anthropic.py +++ b/examples/anthropic/script_multi_generator_haiku.py @@ -16,7 +16,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, "library": "beautifulsoup" diff --git a/examples/anthropic/search_graph_anthropic.py b/examples/anthropic/search_graph_haiku.py similarity index 95% rename from examples/anthropic/search_graph_anthropic.py rename to examples/anthropic/search_graph_haiku.py index 4ae0e6b3..f90d7598 100644 --- a/examples/anthropic/search_graph_anthropic.py +++ b/examples/anthropic/search_graph_haiku.py @@ -15,7 +15,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/search_graph_schema_anthropic.py b/examples/anthropic/search_graph_schema_haiku.py similarity index 92% rename from examples/anthropic/search_graph_schema_anthropic.py rename to examples/anthropic/search_graph_schema_haiku.py index 58e1ca0f..c9e7a875 100644 --- a/examples/anthropic/search_graph_schema_anthropic.py +++ b/examples/anthropic/search_graph_schema_haiku.py @@ -27,9 +27,8 @@ class Dishes(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "claude-3-haiku-20240307", + "max_tokens": 4000}, } # ************************************************ diff --git a/examples/anthropic/search_link_graph_anthropic.py b/examples/anthropic/search_link_graph_haiku.py similarity index 91% rename from examples/anthropic/search_link_graph_anthropic.py rename to examples/anthropic/search_link_graph_haiku.py index 4d671817..ccfbc1d2 100644 --- a/examples/anthropic/search_link_graph_anthropic.py +++ b/examples/anthropic/search_link_graph_haiku.py @@ -29,11 +29,8 @@ embedder_model_instance = AzureOpenAIEmbeddings( # ************************************************ graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "llm": {"model_instance": llm_model_instance}, + "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/anthropic/smart_scraper_anthropic.py b/examples/anthropic/smart_scraper_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_anthropic.py rename to examples/anthropic/smart_scraper_haiku.py index 612363c7..f0bb2a57 100644 --- a/examples/anthropic/smart_scraper_anthropic.py +++ b/examples/anthropic/smart_scraper_haiku.py @@ -19,7 +19,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/smart_scraper_multi_anthropic.py b/examples/anthropic/smart_scraper_multi_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_multi_anthropic.py rename to examples/anthropic/smart_scraper_multi_haiku.py index 6e2af361..eb2001d4 100644 --- a/examples/anthropic/smart_scraper_multi_anthropic.py +++ b/examples/anthropic/smart_scraper_multi_haiku.py @@ -17,7 +17,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/smart_scraper_schema_anthropic.py b/examples/anthropic/smart_scraper_schema_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_schema_anthropic.py rename to examples/anthropic/smart_scraper_schema_haiku.py index 0a444923..83cedd2a 100644 --- a/examples/anthropic/smart_scraper_schema_anthropic.py +++ b/examples/anthropic/smart_scraper_schema_haiku.py @@ -33,7 +33,7 @@ class Projects(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/xml_scraper_graph_multi_anthropic.py b/examples/anthropic/xml_scraper_graph_multi_haiku.py similarity index 96% rename from examples/anthropic/xml_scraper_graph_multi_anthropic.py rename to examples/anthropic/xml_scraper_graph_multi_haiku.py index 31b350c2..6b79f709 100644 --- a/examples/anthropic/xml_scraper_graph_multi_anthropic.py +++ b/examples/anthropic/xml_scraper_graph_multi_haiku.py @@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/xml_scraper_anthropic.py b/examples/anthropic/xml_scraper_haiku.py similarity index 96% rename from examples/anthropic/xml_scraper_anthropic.py rename to examples/anthropic/xml_scraper_haiku.py index cd60f0d6..dd64f571 100644 --- a/examples/anthropic/xml_scraper_anthropic.py +++ b/examples/anthropic/xml_scraper_haiku.py @@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index f80e430d..555e8211 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -142,7 +142,7 @@ class AbstractGraph(ABC): known_models = {"chatgpt","gpt","openai", "azure_openai", "google_genai", "ollama", "oneapi", "nvidia", "groq", "google_vertexai", "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", - "fireworks", "anthropic"} + "fireworks", "claude-3-"} if llm_params["model"].split("/")[0] not in known_models and llm_params["model"].split("-")[0] not in known_models: raise ValueError(f"Model '{llm_params['model']}' is not supported") @@ -172,9 +172,8 @@ class AbstractGraph(ABC): token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"] return handle_model(model_name, "ollama", token_key) - elif "anthropic" in llm_params["model"]: - model_name = llm_params["model"].split("anthropic/")[-1] - return handle_model(model_name, "anthropic", model_name) + elif "claude-3-" in llm_params["model"]: + return handle_model(llm_params["model"], "anthropic", "claude3") elif llm_params["model"].startswith("mistral"): model_name = llm_params["model"].split("/")[-1]