From d5a8250f6dbd650b1495a5a73ff6a5eca328db66 Mon Sep 17 00:00:00 2001 From: mycoffeezzz Date: Thu, 24 Oct 2024 20:11:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E9=9A=8F=E6=9C=BA=E7=94=9F=E6=88=90?= =?UTF-8?q?=E6=96=B0ua=EF=BC=8C=E4=B8=8Etokens=E5=AF=B9=E5=BA=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 4 +- .gitignore | 4 +- chatgpt/ChatService.py | 147 ++++++++++++++++++++++++--------------- chatgpt/authorization.py | 18 +++++ chatgpt/globals.py | 59 ++++++++++++++-- requirements.txt | 3 +- utils/Client.py | 9 +-- 7 files changed, 173 insertions(+), 71 deletions(-) diff --git a/.dockerignore b/.dockerignore index 1706cd7..e728f62 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,6 @@ /.idea/ /docs/ /tmp/ -/data/ \ No newline at end of file +/data/ +/.venv/ +/.vscode/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index d8f1ffe..a8b81d9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ /.git/ /.idea/ /tmp/ -/data/ \ No newline at end of file +/data/ +/.venv/ +/.vscode/ \ No newline at end of file diff --git a/chatgpt/ChatService.py b/chatgpt/ChatService.py index e2e829f..5dd8988 100644 --- a/chatgpt/ChatService.py +++ b/chatgpt/ChatService.py @@ -8,27 +8,46 @@ from starlette.concurrency import run_in_threadpool from api.files import get_image_size, get_file_extension, determine_file_use_case from api.models import model_proxy -from chatgpt.authorization import get_req_token, verify_token +from chatgpt.authorization import get_req_token, verify_token, get_ua from chatgpt.chatFormat import api_messages_to_chat, stream_response, format_not_stream_response, head_process_response from chatgpt.chatLimit import check_is_limit, handle_request_limit from chatgpt.proofofWork import get_config, get_dpl, get_answer_token, get_requirements_token from utils.Client import Client from utils.Logger import logger -from utils.config import proxy_url_list, chatgpt_base_url_list, ark0se_token_url_list, history_disabled, pow_difficulty, \ - conversation_only, enable_limit, upload_by_url, check_model, auth_key, user_agents_list, turnstile_solver_url +from utils.config import ( + proxy_url_list, + chatgpt_base_url_list, + ark0se_token_url_list, + history_disabled, + pow_difficulty, + conversation_only, + enable_limit, + upload_by_url, + check_model, + auth_key, + user_agents_list, + turnstile_solver_url, +) class ChatService: def __init__(self, origin_token=None): - self.user_agent = random.choice(user_agents_list) if user_agents_list else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" + # self.user_agent = random.choice(user_agents_list) if user_agents_list else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" self.req_token = get_req_token(origin_token) + self.ua = get_ua(self.req_token) + self.user_agent = self.ua.get( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", + ) self.chat_token = "gAAAAAB" self.s = None self.ws = None async def set_dynamic_data(self, data): if self.req_token: + logger.info(f"Request impersonate: {self.ua.get('impersonate')}") + logger.info(f"Request ua:{self.user_agent}") logger.info(f"Request token: {self.req_token}") req_len = len(self.req_token.split(",")) if req_len == 1: @@ -64,7 +83,7 @@ class ChatService: self.host_url = random.choice(chatgpt_base_url_list) if chatgpt_base_url_list else "https://chatgpt.com" self.ark0se_token_url = random.choice(ark0se_token_url_list) if ark0se_token_url_list else None - self.s = Client(proxy=self.proxy_url) + self.s = Client(proxy=self.proxy_url, impersonate=self.ua.get("impersonate", "safari15_3")) self.oai_device_id = str(uuid.uuid4()) self.persona = None @@ -85,13 +104,13 @@ class ChatService: 'Origin': self.host_url, 'Priority': 'u=1, i', 'Referer': f'{self.host_url}/', - 'Sec-Ch-Ua': '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"', - 'Sec-Ch-Ua-Mobile': '?0', - 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Ch-Ua': self.ua.get("Sec-Ch-Ua", '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"'), + 'Sec-Ch-Ua-Mobile': self.ua.get("Sec-Ch-Ua-Mobile", "?0"), + 'Sec-Ch-Ua-Platform': self.ua.get("Sec-Ch-Ua-Platform", '"Windows"'), 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', - 'User-Agent': self.user_agent + 'User-Agent': self.user_agent, } if self.access_token: self.base_url = self.host_url + "/backend-api" @@ -155,12 +174,15 @@ class ChatService: models = r.json().get('models') if not any(self.req_model in model.get("slug", "") for model in models): logger.error(f"Model {self.req_model} not support.") - raise HTTPException(status_code=404, detail={ - "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": None, - "code": "model_not_found" - }) + raise HTTPException( + status_code=404, + detail={ + "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + ) else: raise HTTPException(status_code=404, detail="Failed to get models") else: @@ -168,12 +190,15 @@ class ChatService: if self.persona != "chatgpt-paid": if self.req_model == "gpt-4": logger.error(f"Model {self.resp_model} not support for {self.persona}") - raise HTTPException(status_code=404, detail={ - "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": None, - "code": "model_not_found" - }) + raise HTTPException( + status_code=404, + detail={ + "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + ) turnstile = resp.get('turnstile', {}) turnstile_required = turnstile.get('required') @@ -181,7 +206,9 @@ class ChatService: turnstile_dx = turnstile.get("dx") try: if turnstile_solver_url: - res = await self.s.post(turnstile_solver_url, json={"url": "https://chatgpt.com", "p": p, "dx": turnstile_dx}) + res = await self.s.post( + turnstile_solver_url, json={"url": "https://chatgpt.com", "p": p, "dx": turnstile_dx} + ) self.turnstile_token = res.json().get("t") except Exception as e: logger.info(f"Turnstile ignored: {e}") @@ -197,12 +224,10 @@ class ChatService: if not self.ark0se_token_url: raise HTTPException(status_code=403, detail="Ark0se service required") ark0se_dx = ark0se.get("dx") - ark0se_client = Client() + ark0se_client = Client(impersonate=self.ua.get("impersonate", "safari15_3")) try: r2 = await ark0se_client.post( - url=self.ark0se_token_url, - json={"blob": ark0se_dx, "method": ark0se_method}, - timeout=15 + url=self.ark0se_token_url, json={"blob": ark0se_dx, "method": ark0se_method}, timeout=15 ) r2esp = r2.json() logger.info(f"ark0se_token: {r2esp}") @@ -220,11 +245,11 @@ class ChatService: if proofofwork_required: proofofwork_diff = proofofwork.get("difficulty") if proofofwork_diff <= pow_difficulty: - raise HTTPException(status_code=403, - detail=f"Proof of work difficulty too high: {proofofwork_diff}") + raise HTTPException(status_code=403, detail=f"Proof of work difficulty too high: {proofofwork_diff}") proofofwork_seed = proofofwork.get("seed") - self.proof_token, solved = await run_in_threadpool(get_answer_token, proofofwork_seed, - proofofwork_diff, config) + self.proof_token, solved = await run_in_threadpool( + get_answer_token, proofofwork_seed, proofofwork_diff, config + ) if not solved: raise HTTPException(status_code=403, detail="Failed to solve proof of work") @@ -254,11 +279,13 @@ class ChatService: logger.error(f"Failed to format messages: {str(e)}") raise HTTPException(status_code=400, detail="Failed to format messages.") self.chat_headers = self.base_headers.copy() - self.chat_headers.update({ - 'Accept': 'text/event-stream', - 'Openai-Sentinel-Chat-Requirements-Token': self.chat_token, - 'Openai-Sentinel-Proof-Token': self.proof_token, - }) + self.chat_headers.update( + { + 'Accept': 'text/event-stream', + 'Openai-Sentinel-Chat-Requirements-Token': self.chat_token, + 'Openai-Sentinel-Proof-Token': self.proof_token, + } + ) if self.ark0se_token: self.chat_headers['Openai-Sentinel-Ark' + 'ose-Token'] = self.ark0se_token @@ -294,7 +321,7 @@ class ChatService: "suggestions": [], "timezone_offset_min": -480, "variant_purpose": "comparison_implicit", - "websocket_request_id": f"{uuid.uuid4()}" + "websocket_request_id": f"{uuid.uuid4()}", } if self.conversation_id: self.chat_request['conversation_id'] = self.conversation_id @@ -304,8 +331,7 @@ class ChatService: try: url = f'{self.base_url}/conversation' stream = self.data.get("stream", False) - r = await self.s.post_stream(url, headers=self.chat_headers, json=self.chat_request, timeout=10, - stream=True) + r = await self.s.post_stream(url, headers=self.chat_headers, json=self.chat_request, timeout=10, stream=True) if r.status_code != 200: rtext = await r.atext() if "application/json" == r.headers.get("Content-Type", ""): @@ -327,13 +353,19 @@ class ChatService: if "text/event-stream" in content_type: res, start = await head_process_response(r.aiter_lines()) if not start: - raise HTTPException(status_code=403, detail="Our systems have detected unusual activity coming from your system. Please try again later.") + raise HTTPException( + status_code=403, + detail="Our systems have detected unusual activity coming from your system. Please try again later.", + ) if stream: return stream_response(self, res, self.resp_model, self.max_tokens) else: return await format_not_stream_response( - stream_response(self, res, self.resp_model, self.max_tokens), self.prompt_tokens, - self.max_tokens, self.resp_model) + stream_response(self, res, self.resp_model, self.max_tokens), + self.prompt_tokens, + self.max_tokens, + self.resp_model, + ) elif "application/json" in content_type: rtext = await r.atext() resp = json.loads(rtext) @@ -376,12 +408,12 @@ class ChatService: url = f'{self.base_url}/files' headers = self.base_headers.copy() try: - r = await self.s.post(url, headers=headers, json={ - "file_name": file_name, - "file_size": file_size, - "timezone_offset_min": -480, - "use_case": use_case - }, timeout=5) + r = await self.s.post( + url, + headers=headers, + json={"file_name": file_name, "file_size": file_size, "timezone_offset_min": -480, "use_case": use_case}, + timeout=5, + ) if r.status_code == 200: res = r.json() file_id = res.get('file_id') @@ -395,12 +427,14 @@ class ChatService: async def upload(self, upload_url, file_content, mime_type): headers = self.base_headers.copy() - headers.update({ - 'Accept': 'application/json, text/plain, */*', - 'Content-Type': mime_type, - 'X-Ms-Blob-Type': 'BlockBlob', - 'X-Ms-Version': '2020-04-08' - }) + headers.update( + { + 'Accept': 'application/json, text/plain, */*', + 'Content-Type': mime_type, + 'X-Ms-Blob-Type': 'BlockBlob', + 'X-Ms-Version': '2020-04-08', + } + ) headers.pop('Authorization', None) try: r = await self.s.put(upload_url, headers=headers, data=file_content) @@ -438,7 +472,7 @@ class ChatService: "mime_type": mime_type, "width": width, "height": height, - "use_case": use_case + "use_case": use_case, } logger.info(f"File_meta: {file_meta}") return file_meta @@ -468,10 +502,7 @@ class ChatService: async def get_response_file_url(self, conversation_id, message_id, sandbox_path): try: url = f"{self.base_url}/conversation/{conversation_id}/interpreter/download" - params = { - "message_id": message_id, - "sandbox_path": sandbox_path - } + params = {"message_id": message_id, "sandbox_path": sandbox_path} headers = self.base_headers.copy() r = await self.s.get(url, headers=headers, params=params, timeout=10) if r.status_code == 200: diff --git a/chatgpt/authorization.py b/chatgpt/authorization.py index d357145..be2f65d 100644 --- a/chatgpt/authorization.py +++ b/chatgpt/authorization.py @@ -1,6 +1,8 @@ import asyncio +import random from fastapi import HTTPException +import ua_generator from chatgpt.refreshToken import rt2ac from utils.Logger import logger @@ -23,6 +25,22 @@ def get_req_token(req_token): return req_token +def get_ua(req_token): + user_agent = globals.user_agent_map.get(req_token, "") + # token为空，免登录用户，则随机生成ua + if not user_agent: + ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos')) + return { + "User-Agent": ua.text, + "Sec-Ch-Ua-Platform": ua.platform, + "Sec-Ch-Ua": ua.ch.brands, + "Sec-Ch-Ua-Mobile": ua.ch.mobile, + "impersonate": random.choice(globals.impersonate_list), + } + else: + return user_agent + + async def verify_token(req_token): if not req_token: if authorization_list: diff --git a/chatgpt/globals.py b/chatgpt/globals.py index 82e1cdb..1568ad6 100644 --- a/chatgpt/globals.py +++ b/chatgpt/globals.py @@ -1,6 +1,9 @@ import json import os +import ua_generator +import random + from utils.Logger import logger DATA_FOLDER = "data" @@ -8,13 +11,29 @@ TOKENS_FILE = os.path.join(DATA_FOLDER, "token.txt") REFRESH_MAP_FILE = os.path.join(DATA_FOLDER, "refresh_map.json") ERROR_TOKENS_FILE = os.path.join(DATA_FOLDER, "error_token.txt") WSS_MAP_FILE = os.path.join(DATA_FOLDER, "wss_map.json") +USER_AGENTS_FILE = os.path.join(DATA_FOLDER, "user_agents.json") count = 0 token_list = [] error_token_list = [] refresh_map = {} wss_map = {} - +user_agent_map = {} +impersonate_list = [ + "chrome99", + "chrome100", + "chrome101", + "chrome104", + "chrome107", + "chrome110", + "chrome116", + "chrome119", + "chrome120", + "chrome123", + "chrome124", + "edge99", + "edge101", +] if not os.path.exists(DATA_FOLDER): os.makedirs(DATA_FOLDER) @@ -30,8 +49,8 @@ if os.path.exists(WSS_MAP_FILE): wss_map = json.load(file) else: wss_map = {} - - + + if os.path.exists(TOKENS_FILE): with open(TOKENS_FILE, "r", encoding="utf-8") as f: for line in f: @@ -50,5 +69,37 @@ else: with open(ERROR_TOKENS_FILE, "w", encoding="utf-8") as f: pass +if os.path.exists(USER_AGENTS_FILE): + with open(USER_AGENTS_FILE, "r", encoding="utf-8") as f: + user_agent_map = json.load(f) + # token数量变化时，更新ua + if len(user_agent_map.keys()) != len(token_list): + new_tokens = list(set(token_list) - user_agent_map.keys()) + for token in new_tokens: + ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos')) + ua_dict = { + "User-Agent": ua.text, + "Sec-Ch-Ua-Platform": ua.platform, + "Sec-Ch-Ua": ua.ch.brands, + "Sec-Ch-Ua-Mobile": ua.ch.mobile, + "impersonate": random.choice(impersonate_list), + } + user_agent_map[token] = ua_dict + with open(USER_AGENTS_FILE, "w", encoding="utf-8") as f: + f.write(json.dumps(user_agent_map, indent=4)) +else: + for token in token_list: + ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos')) + ua_dict = { + "User-Agent": ua.text, + "Sec-Ch-Ua-Platform": ua.platform, + "Sec-Ch-Ua": ua.ch.brands, + "Sec-Ch-Ua-Mobile": ua.ch.mobile, + "impersonate": random.choice(impersonate_list), + } + user_agent_map[token] = ua_dict + with open(USER_AGENTS_FILE, "w", encoding="utf-8") as f: + f.write(json.dumps(user_agent_map, indent=4)) + if token_list: - logger.info(f"Token list count: {len(token_list)}, Error token list count: {len(error_token_list)}") \ No newline at end of file + logger.info(f"Token list count: {len(token_list)}, Error token list count: {len(error_token_list)}") diff --git a/requirements.txt b/requirements.txt index 8da543b..426ed63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ websockets pillow pybase64 jinja2 -APScheduler \ No newline at end of file +APScheduler +ua-generator \ No newline at end of file diff --git a/utils/Client.py b/utils/Client.py index 03ff2b8..ed0e539 100644 --- a/utils/Client.py +++ b/utils/Client.py @@ -4,15 +4,12 @@ from curl_cffi.requests import AsyncSession class Client: - def __init__(self, proxy=None, timeout=15, verify=True): - self.proxies = { - "http": proxy, - "https": proxy, - } + def __init__(self, proxy=None, timeout=15, verify=True, impersonate='safari15_3'): + self.proxies = {"http": proxy, "https": proxy} self.timeout = timeout self.verify = verify - self.impersonate = random.choice(['safari15_3']) + self.impersonate = impersonate # impersonate=self.impersonate # self.ja3 = ""