feat:随机生成新ua，与tokens对应

.dockerignore

@@ -4,4 +4,6 @@
 /.idea/
 /docs/
 /tmp/
-/data/
+/data/
+/.venv/
+/.vscode/

.gitignore

@@ -3,4 +3,6 @@
 /.git/
 /.idea/
 /tmp/
-/data/
+/data/
+/.venv/
+/.vscode/

chatgpt/ChatService.py

@@ -8,27 +8,46 @@ from starlette.concurrency import run_in_threadpool
 
 from api.files import get_image_size, get_file_extension, determine_file_use_case
 from api.models import model_proxy
-from chatgpt.authorization import get_req_token, verify_token
+from chatgpt.authorization import get_req_token, verify_token, get_ua
 from chatgpt.chatFormat import api_messages_to_chat, stream_response, format_not_stream_response, head_process_response
 from chatgpt.chatLimit import check_is_limit, handle_request_limit
 from chatgpt.proofofWork import get_config, get_dpl, get_answer_token, get_requirements_token
 
 from utils.Client import Client
 from utils.Logger import logger
-from utils.config import proxy_url_list, chatgpt_base_url_list, ark0se_token_url_list, history_disabled, pow_difficulty, \
-    conversation_only, enable_limit, upload_by_url, check_model, auth_key, user_agents_list, turnstile_solver_url
+from utils.config import (
+    proxy_url_list,
+    chatgpt_base_url_list,
+    ark0se_token_url_list,
+    history_disabled,
+    pow_difficulty,
+    conversation_only,
+    enable_limit,
+    upload_by_url,
+    check_model,
+    auth_key,
+    user_agents_list,
+    turnstile_solver_url,
+)
 
 
 class ChatService:
     def __init__(self, origin_token=None):
-        self.user_agent = random.choice(user_agents_list) if user_agents_list else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
+        # self.user_agent = random.choice(user_agents_list) if user_agents_list else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
         self.req_token = get_req_token(origin_token)
+        self.ua = get_ua(self.req_token)
+        self.user_agent = self.ua.get(
+            "User-Agent",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+        )
         self.chat_token = "gAAAAAB"
         self.s = None
         self.ws = None
 
     async def set_dynamic_data(self, data):
         if self.req_token:
+            logger.info(f"Request impersonate: {self.ua.get('impersonate')}")
+            logger.info(f"Request ua:{self.user_agent}")
             logger.info(f"Request token: {self.req_token}")
             req_len = len(self.req_token.split(","))
             if req_len == 1:
@@ -64,7 +83,7 @@ class ChatService:
         self.host_url = random.choice(chatgpt_base_url_list) if chatgpt_base_url_list else "https://chatgpt.com"
         self.ark0se_token_url = random.choice(ark0se_token_url_list) if ark0se_token_url_list else None
 
-        self.s = Client(proxy=self.proxy_url)
+        self.s = Client(proxy=self.proxy_url, impersonate=self.ua.get("impersonate", "safari15_3"))
 
         self.oai_device_id = str(uuid.uuid4())
         self.persona = None
@@ -85,13 +104,13 @@ class ChatService:
             'Origin': self.host_url,
             'Priority': 'u=1, i',
             'Referer': f'{self.host_url}/',
-            'Sec-Ch-Ua': '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
-            'Sec-Ch-Ua-Mobile': '?0',
-            'Sec-Ch-Ua-Platform': '"Windows"',
+            'Sec-Ch-Ua': self.ua.get("Sec-Ch-Ua", '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"'),
+            'Sec-Ch-Ua-Mobile': self.ua.get("Sec-Ch-Ua-Mobile", "?0"),
+            'Sec-Ch-Ua-Platform': self.ua.get("Sec-Ch-Ua-Platform", '"Windows"'),
             'Sec-Fetch-Dest': 'empty',
             'Sec-Fetch-Mode': 'cors',
             'Sec-Fetch-Site': 'same-origin',
-            'User-Agent': self.user_agent
+            'User-Agent': self.user_agent,
         }
         if self.access_token:
             self.base_url = self.host_url + "/backend-api"
@@ -155,12 +174,15 @@ class ChatService:
                         models = r.json().get('models')
                         if not any(self.req_model in model.get("slug", "") for model in models):
                             logger.error(f"Model {self.req_model} not support.")
-                            raise HTTPException(status_code=404, detail={
-                                "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.",
-                                "type": "invalid_request_error",
-                                "param": None,
-                                "code": "model_not_found"
-                            })
+                            raise HTTPException(
+                                status_code=404,
+                                detail={
+                                    "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.",
+                                    "type": "invalid_request_error",
+                                    "param": None,
+                                    "code": "model_not_found",
+                                },
+                            )
                     else:
                         raise HTTPException(status_code=404, detail="Failed to get models")
                 else:
@@ -168,12 +190,15 @@ class ChatService:
                     if self.persona != "chatgpt-paid":
                         if self.req_model == "gpt-4":
                             logger.error(f"Model {self.resp_model} not support for {self.persona}")
-                            raise HTTPException(status_code=404, detail={
-                                "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.",
-                                "type": "invalid_request_error",
-                                "param": None,
-                                "code": "model_not_found"
-                            })
+                            raise HTTPException(
+                                status_code=404,
+                                detail={
+                                    "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.",
+                                    "type": "invalid_request_error",
+                                    "param": None,
+                                    "code": "model_not_found",
+                                },
+                            )
 
                 turnstile = resp.get('turnstile', {})
                 turnstile_required = turnstile.get('required')
@@ -181,7 +206,9 @@ class ChatService:
                     turnstile_dx = turnstile.get("dx")
                     try:
                         if turnstile_solver_url:
-                            res = await self.s.post(turnstile_solver_url, json={"url": "https://chatgpt.com", "p": p, "dx": turnstile_dx})
+                            res = await self.s.post(
+                                turnstile_solver_url, json={"url": "https://chatgpt.com", "p": p, "dx": turnstile_dx}
+                            )
                             self.turnstile_token = res.json().get("t")
                     except Exception as e:
                         logger.info(f"Turnstile ignored: {e}")
@@ -197,12 +224,10 @@ class ChatService:
                     if not self.ark0se_token_url:
                         raise HTTPException(status_code=403, detail="Ark0se service required")
                     ark0se_dx = ark0se.get("dx")
-                    ark0se_client = Client()
+                    ark0se_client = Client(impersonate=self.ua.get("impersonate", "safari15_3"))
                     try:
                         r2 = await ark0se_client.post(
-                            url=self.ark0se_token_url,
-                            json={"blob": ark0se_dx, "method": ark0se_method},
-                            timeout=15
+                            url=self.ark0se_token_url, json={"blob": ark0se_dx, "method": ark0se_method}, timeout=15
                         )
                         r2esp = r2.json()
                         logger.info(f"ark0se_token: {r2esp}")
@@ -220,11 +245,11 @@ class ChatService:
                 if proofofwork_required:
                     proofofwork_diff = proofofwork.get("difficulty")
                     if proofofwork_diff <= pow_difficulty:
-                        raise HTTPException(status_code=403,
-                                            detail=f"Proof of work difficulty too high: {proofofwork_diff}")
+                        raise HTTPException(status_code=403, detail=f"Proof of work difficulty too high: {proofofwork_diff}")
                     proofofwork_seed = proofofwork.get("seed")
-                    self.proof_token, solved = await run_in_threadpool(get_answer_token, proofofwork_seed,
-                                                                       proofofwork_diff, config)
+                    self.proof_token, solved = await run_in_threadpool(
+                        get_answer_token, proofofwork_seed, proofofwork_diff, config
+                    )
                     if not solved:
                         raise HTTPException(status_code=403, detail="Failed to solve proof of work")
 
@@ -254,11 +279,13 @@ class ChatService:
             logger.error(f"Failed to format messages: {str(e)}")
             raise HTTPException(status_code=400, detail="Failed to format messages.")
         self.chat_headers = self.base_headers.copy()
-        self.chat_headers.update({
-            'Accept': 'text/event-stream',
-            'Openai-Sentinel-Chat-Requirements-Token': self.chat_token,
-            'Openai-Sentinel-Proof-Token': self.proof_token,
-        })
+        self.chat_headers.update(
+            {
+                'Accept': 'text/event-stream',
+                'Openai-Sentinel-Chat-Requirements-Token': self.chat_token,
+                'Openai-Sentinel-Proof-Token': self.proof_token,
+            }
+        )
         if self.ark0se_token:
             self.chat_headers['Openai-Sentinel-Ark' + 'ose-Token'] = self.ark0se_token
 
@@ -294,7 +321,7 @@ class ChatService:
             "suggestions": [],
             "timezone_offset_min": -480,
             "variant_purpose": "comparison_implicit",
-            "websocket_request_id": f"{uuid.uuid4()}"
+            "websocket_request_id": f"{uuid.uuid4()}",
         }
         if self.conversation_id:
             self.chat_request['conversation_id'] = self.conversation_id
@@ -304,8 +331,7 @@ class ChatService:
         try:
             url = f'{self.base_url}/conversation'
             stream = self.data.get("stream", False)
-            r = await self.s.post_stream(url, headers=self.chat_headers, json=self.chat_request, timeout=10,
-                                         stream=True)
+            r = await self.s.post_stream(url, headers=self.chat_headers, json=self.chat_request, timeout=10, stream=True)
             if r.status_code != 200:
                 rtext = await r.atext()
                 if "application/json" == r.headers.get("Content-Type", ""):
@@ -327,13 +353,19 @@ class ChatService:
             if "text/event-stream" in content_type:
                 res, start = await head_process_response(r.aiter_lines())
                 if not start:
-                    raise HTTPException(status_code=403, detail="Our systems have detected unusual activity coming from your system. Please try again later.")
+                    raise HTTPException(
+                        status_code=403,
+                        detail="Our systems have detected unusual activity coming from your system. Please try again later.",
+                    )
                 if stream:
                     return stream_response(self, res, self.resp_model, self.max_tokens)
                 else:
                     return await format_not_stream_response(
-                        stream_response(self, res, self.resp_model, self.max_tokens), self.prompt_tokens,
-                        self.max_tokens, self.resp_model)
+                        stream_response(self, res, self.resp_model, self.max_tokens),
+                        self.prompt_tokens,
+                        self.max_tokens,
+                        self.resp_model,
+                    )
             elif "application/json" in content_type:
                 rtext = await r.atext()
                 resp = json.loads(rtext)
@@ -376,12 +408,12 @@ class ChatService:
         url = f'{self.base_url}/files'
         headers = self.base_headers.copy()
         try:
-            r = await self.s.post(url, headers=headers, json={
-                "file_name": file_name,
-                "file_size": file_size,
-                "timezone_offset_min": -480,
-                "use_case": use_case
-            }, timeout=5)
+            r = await self.s.post(
+                url,
+                headers=headers,
+                json={"file_name": file_name, "file_size": file_size, "timezone_offset_min": -480, "use_case": use_case},
+                timeout=5,
+            )
             if r.status_code == 200:
                 res = r.json()
                 file_id = res.get('file_id')
@@ -395,12 +427,14 @@ class ChatService:
 
     async def upload(self, upload_url, file_content, mime_type):
         headers = self.base_headers.copy()
-        headers.update({
-            'Accept': 'application/json, text/plain, */*',
-            'Content-Type': mime_type,
-            'X-Ms-Blob-Type': 'BlockBlob',
-            'X-Ms-Version': '2020-04-08'
-        })
+        headers.update(
+            {
+                'Accept': 'application/json, text/plain, */*',
+                'Content-Type': mime_type,
+                'X-Ms-Blob-Type': 'BlockBlob',
+                'X-Ms-Version': '2020-04-08',
+            }
+        )
         headers.pop('Authorization', None)
         try:
             r = await self.s.put(upload_url, headers=headers, data=file_content)
@@ -438,7 +472,7 @@ class ChatService:
                         "mime_type": mime_type,
                         "width": width,
                         "height": height,
-                        "use_case": use_case
+                        "use_case": use_case,
                     }
                     logger.info(f"File_meta: {file_meta}")
                     return file_meta
@@ -468,10 +502,7 @@ class ChatService:
     async def get_response_file_url(self, conversation_id, message_id, sandbox_path):
         try:
             url = f"{self.base_url}/conversation/{conversation_id}/interpreter/download"
-            params = {
-                "message_id": message_id,
-                "sandbox_path": sandbox_path
-            }
+            params = {"message_id": message_id, "sandbox_path": sandbox_path}
             headers = self.base_headers.copy()
             r = await self.s.get(url, headers=headers, params=params, timeout=10)
             if r.status_code == 200:

chatgpt/authorization.py

@@ -1,6 +1,8 @@
 import asyncio
+import random
 
 from fastapi import HTTPException
+import ua_generator
 
 from chatgpt.refreshToken import rt2ac
 from utils.Logger import logger
@@ -23,6 +25,22 @@ def get_req_token(req_token):
         return req_token
 
 
+def get_ua(req_token):
+    user_agent = globals.user_agent_map.get(req_token, "")
+    # token为空，免登录用户，则随机生成ua
+    if not user_agent:
+        ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos'))
+        return {
+            "User-Agent": ua.text,
+            "Sec-Ch-Ua-Platform": ua.platform,
+            "Sec-Ch-Ua": ua.ch.brands,
+            "Sec-Ch-Ua-Mobile": ua.ch.mobile,
+            "impersonate": random.choice(globals.impersonate_list),
+        }
+    else:
+        return user_agent
+
+
 async def verify_token(req_token):
     if not req_token:
         if authorization_list:

chatgpt/globals.py

@@ -1,6 +1,9 @@
 import json
 import os
 
+import ua_generator
+import random
+
 from utils.Logger import logger
 
 DATA_FOLDER = "data"
@@ -8,13 +11,29 @@ TOKENS_FILE = os.path.join(DATA_FOLDER, "token.txt")
 REFRESH_MAP_FILE = os.path.join(DATA_FOLDER, "refresh_map.json")
 ERROR_TOKENS_FILE = os.path.join(DATA_FOLDER, "error_token.txt")
 WSS_MAP_FILE = os.path.join(DATA_FOLDER, "wss_map.json")
+USER_AGENTS_FILE = os.path.join(DATA_FOLDER, "user_agents.json")
 
 count = 0
 token_list = []
 error_token_list = []
 refresh_map = {}
 wss_map = {}
-
+user_agent_map = {}
+impersonate_list = [
+    "chrome99",
+    "chrome100",
+    "chrome101",
+    "chrome104",
+    "chrome107",
+    "chrome110",
+    "chrome116",
+    "chrome119",
+    "chrome120",
+    "chrome123",
+    "chrome124",
+    "edge99",
+    "edge101",
+]
 
 if not os.path.exists(DATA_FOLDER):
     os.makedirs(DATA_FOLDER)
@@ -30,8 +49,8 @@ if os.path.exists(WSS_MAP_FILE):
         wss_map = json.load(file)
 else:
     wss_map = {}
-    
-    
+
+
 if os.path.exists(TOKENS_FILE):
     with open(TOKENS_FILE, "r", encoding="utf-8") as f:
         for line in f:
@@ -50,5 +69,37 @@ else:
     with open(ERROR_TOKENS_FILE, "w", encoding="utf-8") as f:
         pass
 
+if os.path.exists(USER_AGENTS_FILE):
+    with open(USER_AGENTS_FILE, "r", encoding="utf-8") as f:
+        user_agent_map = json.load(f)
+    # token数量变化时，更新ua
+    if len(user_agent_map.keys()) != len(token_list):
+        new_tokens = list(set(token_list) - user_agent_map.keys())
+        for token in new_tokens:
+            ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos'))
+            ua_dict = {
+                "User-Agent": ua.text,
+                "Sec-Ch-Ua-Platform": ua.platform,
+                "Sec-Ch-Ua": ua.ch.brands,
+                "Sec-Ch-Ua-Mobile": ua.ch.mobile,
+                "impersonate": random.choice(impersonate_list),
+            }
+            user_agent_map[token] = ua_dict
+        with open(USER_AGENTS_FILE, "w", encoding="utf-8") as f:
+            f.write(json.dumps(user_agent_map, indent=4))
+else:
+    for token in token_list:
+        ua = ua_generator.generate(device='desktop', browser=('chrome', 'edge'), platform=('windows', 'macos'))
+        ua_dict = {
+            "User-Agent": ua.text,
+            "Sec-Ch-Ua-Platform": ua.platform,
+            "Sec-Ch-Ua": ua.ch.brands,
+            "Sec-Ch-Ua-Mobile": ua.ch.mobile,
+            "impersonate": random.choice(impersonate_list),
+        }
+        user_agent_map[token] = ua_dict
+    with open(USER_AGENTS_FILE, "w", encoding="utf-8") as f:
+        f.write(json.dumps(user_agent_map, indent=4))
+
 if token_list:
-    logger.info(f"Token list count: {len(token_list)}, Error token list count: {len(error_token_list)}")
+    logger.info(f"Token list count: {len(token_list)}, Error token list count: {len(error_token_list)}")

requirements.txt

@@ -8,4 +8,5 @@ websockets
 pillow
 pybase64
 jinja2
-APScheduler
+APScheduler
+ua-generator

utils/Client.py

@@ -4,15 +4,12 @@ from curl_cffi.requests import AsyncSession
 
 
 class Client:
-    def __init__(self, proxy=None, timeout=15, verify=True):
-        self.proxies = {
-            "http": proxy,
-            "https": proxy,
-        }
+    def __init__(self, proxy=None, timeout=15, verify=True, impersonate='safari15_3'):
+        self.proxies = {"http": proxy, "https": proxy}
         self.timeout = timeout
         self.verify = verify
 
-        self.impersonate = random.choice(['safari15_3'])
+        self.impersonate = impersonate
         # impersonate=self.impersonate
 
         # self.ja3 = ""