diff options
Diffstat (limited to 'g4f/Provider')
| -rw-r--r-- | g4f/Provider/Liaobots.py | 79 | ||||
| -rw-r--r-- | g4f/Provider/needs_auth/Gemini.py | 36 | ||||
| -rw-r--r-- | g4f/Provider/needs_auth/OpenaiChat.py | 12 | ||||
| -rw-r--r-- | g4f/Provider/openai/har_file.py | 30 | ||||
| -rw-r--r-- | g4f/Provider/openai/proofofwork.py | 48 |
5 files changed, 134 insertions, 71 deletions
diff --git a/g4f/Provider/Liaobots.py b/g4f/Provider/Liaobots.py index deb7899c..75ecf300 100644 --- a/g4f/Provider/Liaobots.py +++ b/g4f/Provider/Liaobots.py @@ -10,6 +10,15 @@ from .helper import get_connector from ..requests import raise_for_status models = { + "gpt-4o": { + "context": "8K", + "id": "gpt-4o-free", + "maxLength": 31200, + "model": "ChatGPT", + "name": "GPT-4o-free", + "provider": "OpenAI", + "tokenLimit": 7800, + }, "gpt-3.5-turbo": { "id": "gpt-3.5-turbo", "name": "GPT-3.5-Turbo", @@ -95,7 +104,7 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin): model_aliases = { "claude-v2": "claude-2" } - _auth_code = None + _auth_code = "" _cookie_jar = None @classmethod @@ -120,7 +129,13 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin): cookie_jar=cls._cookie_jar, connector=get_connector(connector, proxy, True) ) as session: - cls._auth_code = auth if isinstance(auth, str) else cls._auth_code + data = { + "conversationId": str(uuid.uuid4()), + "model": models[cls.get_model(model)], + "messages": messages, + "key": "", + "prompt": kwargs.get("system_message", "You are a helpful assistant."), + } if not cls._auth_code: async with session.post( "https://liaobots.work/recaptcha/api/login", @@ -128,31 +143,49 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin): verify_ssl=False ) as response: await raise_for_status(response) + try: async with session.post( "https://liaobots.work/api/user", - json={"authcode": ""}, + json={"authcode": cls._auth_code}, verify_ssl=False ) as response: await raise_for_status(response) cls._auth_code = (await response.json(content_type=None))["authCode"] + if not cls._auth_code: + raise RuntimeError("Empty auth code") cls._cookie_jar = session.cookie_jar - - data = { - "conversationId": str(uuid.uuid4()), - "model": models[cls.get_model(model)], - "messages": messages, - "key": "", - "prompt": kwargs.get("system_message", "You are a helpful assistant."), - } - async with session.post( - "https://liaobots.work/api/chat", - json=data, - headers={"x-auth-code": cls._auth_code}, - verify_ssl=False - ) as response: - await raise_for_status(response) - async for chunk in response.content.iter_any(): - if b"<html coupert-item=" in chunk: - raise RuntimeError("Invalid session") - if chunk: - yield chunk.decode(errors="ignore") + async with session.post( + "https://liaobots.work/api/chat", + json=data, + headers={"x-auth-code": cls._auth_code}, + verify_ssl=False + ) as response: + await raise_for_status(response) + async for chunk in response.content.iter_any(): + if b"<html coupert-item=" in chunk: + raise RuntimeError("Invalid session") + if chunk: + yield chunk.decode(errors="ignore") + except: + async with session.post( + "https://liaobots.work/api/user", + json={"authcode": "pTIQr4FTnVRfr"}, + verify_ssl=False + ) as response: + await raise_for_status(response) + cls._auth_code = (await response.json(content_type=None))["authCode"] + if not cls._auth_code: + raise RuntimeError("Empty auth code") + cls._cookie_jar = session.cookie_jar + async with session.post( + "https://liaobots.work/api/chat", + json=data, + headers={"x-auth-code": cls._auth_code}, + verify_ssl=False + ) as response: + await raise_for_status(response) + async for chunk in response.content.iter_any(): + if b"<html coupert-item=" in chunk: + raise RuntimeError("Invalid session") + if chunk: + yield chunk.decode(errors="ignore") diff --git a/g4f/Provider/needs_auth/Gemini.py b/g4f/Provider/needs_auth/Gemini.py index e468f64a..f9b1c4a5 100644 --- a/g4f/Provider/needs_auth/Gemini.py +++ b/g4f/Provider/needs_auth/Gemini.py @@ -4,6 +4,7 @@ import os import json import random import re +import base64 from aiohttp import ClientSession, BaseConnector @@ -22,7 +23,7 @@ from ..base_provider import AsyncGeneratorProvider from ..helper import format_prompt, get_cookies from ...requests.raise_for_status import raise_for_status from ...errors import MissingAuthError, MissingRequirementsError -from ...image import to_bytes, to_data_uri, ImageResponse +from ...image import to_bytes, ImageResponse, ImageDataResponse from ...webdriver import get_browser, get_driver_cookies REQUEST_HEADERS = { @@ -122,6 +123,7 @@ class Gemini(AsyncGeneratorProvider): connector: BaseConnector = None, image: ImageType = None, image_name: str = None, + response_format: str = None, **kwargs ) -> AsyncResult: prompt = format_prompt(messages) @@ -192,22 +194,22 @@ class Gemini(AsyncGeneratorProvider): if image_prompt: images = [image[0][3][3] for image in response_part[4][0][12][7][0]] resolved_images = [] - preview = [] - for image in images: - async with client.get(image, allow_redirects=False) as fetch: - image = fetch.headers["location"] - async with client.get(image, allow_redirects=False) as fetch: - image = fetch.headers["location"] - resolved_images.append(image) - preview.append(image.replace('=s512', '=s200')) - # preview_url = image.replace('=s512', '=s200') - # async with client.get(preview_url) as fetch: - # preview_data = to_data_uri(await fetch.content.read()) - # async with client.get(image) as fetch: - # data = to_data_uri(await fetch.content.read()) - # preview.append(preview_data) - # resolved_images.append(data) - yield ImageResponse(resolved_images, image_prompt, {"orginal_links": images, "preview": preview}) + if response_format == "b64_json": + for image in images: + async with client.get(image) as response: + data = base64.b64encode(await response.content.read()).decode() + resolved_images.append(data) + yield ImageDataResponse(resolved_images, image_prompt) + else: + preview = [] + for image in images: + async with client.get(image, allow_redirects=False) as fetch: + image = fetch.headers["location"] + async with client.get(image, allow_redirects=False) as fetch: + image = fetch.headers["location"] + resolved_images.append(image) + preview.append(image.replace('=s512', '=s200')) + yield ImageResponse(resolved_images, image_prompt, {"orginal_links": images, "preview": preview}) def build_request( prompt: str, diff --git a/g4f/Provider/needs_auth/OpenaiChat.py b/g4f/Provider/needs_auth/OpenaiChat.py index b4b8bb02..28d0558b 100644 --- a/g4f/Provider/needs_auth/OpenaiChat.py +++ b/g4f/Provider/needs_auth/OpenaiChat.py @@ -61,7 +61,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): supports_system_message = True default_model = None default_vision_model = "gpt-4o" - models = ["gpt-3.5-turbo", "gpt-4", "gpt-4-gizmo", "gpt-4o"] + models = ["gpt-3.5-turbo", "gpt-4", "gpt-4-gizmo", "gpt-4o", "auto"] model_aliases = { "text-davinci-002-render-sha": "gpt-3.5-turbo", "": "gpt-3.5-turbo", @@ -394,10 +394,11 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): print(f"{e.__class__.__name__}: {e}") arkose_token = None + proofTokens = None if cls.default_model is None: error = None try: - arkose_token, api_key, cookies, headers = await getArkoseAndAccessToken(proxy) + arkose_token, api_key, cookies, headers, proofTokens = await getArkoseAndAccessToken(proxy) cls._create_request_args(cookies, headers) cls._set_api_key(api_key) except NoValidHarFileError as e: @@ -413,17 +414,17 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): if cls._api_key is None else f"{cls.url}/backend-api/sentinel/chat-requirements", json={"conversation_mode_kind": "primary_assistant"}, + #json={"p": generate_proof_token(True, user_agent=cls._headers["user-agent"], proofTokens=proofTokens)}, headers=cls._headers ) as response: cls._update_request_args(session) await raise_for_status(response) data = await response.json() - blob = data["arkose"]["dx"] - need_arkose = data["arkose"]["required"] + need_arkose = data.get("arkose", {}).get("required") chat_token = data["token"] proofofwork = "" if "proofofwork" in data: - proofofwork = generate_proof_token(**data["proofofwork"], user_agent=cls._headers["user-agent"]) + proofofwork = generate_proof_token(**data["proofofwork"], user_agent=cls._headers["user-agent"], proofTokens=proofTokens) if need_arkose and arkose_token is None: arkose_token, api_key, cookies, headers = await getArkoseAndAccessToken(proxy) @@ -435,7 +436,6 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): if debug.logging: print( 'Arkose:', False if not need_arkose else arkose_token[:12]+"...", - 'Turnstile:', data["turnstile"]["required"], 'Proofofwork:', False if proofofwork is None else proofofwork[:12]+"...", ) diff --git a/g4f/Provider/openai/har_file.py b/g4f/Provider/openai/har_file.py index 220c20bf..eefe305f 100644 --- a/g4f/Provider/openai/har_file.py +++ b/g4f/Provider/openai/har_file.py @@ -12,6 +12,7 @@ from copy import deepcopy from .crypt import decrypt, encrypt from ...requests import StreamSession +from ... import debug class NoValidHarFileError(Exception): ... @@ -31,6 +32,7 @@ chatArk: arkReq = None accessToken: str = None cookies: dict = None headers: dict = None +proofTokens: list = [] def readHAR(): dirPath = "./" @@ -54,6 +56,15 @@ def readHAR(): # Error: not a HAR file! continue for v in harFile['log']['entries']: + v_headers = get_headers(v) + try: + if "openai-sentinel-proof-token" in v_headers: + proofTokens.append(json.loads(base64.b64decode( + v_headers["openai-sentinel-proof-token"].split("gAAAAAB", 1)[-1].encode() + ).decode())) + except Exception as e: + if debug.logging: + print(f"Read proof token: {e}") if arkPreURL in v['request']['url']: chatArks.append(parseHAREntry(v)) elif v['request']['url'] == sessionUrl: @@ -61,13 +72,13 @@ def readHAR(): accessToken = json.loads(v["response"]["content"]["text"]).get("accessToken") except KeyError: continue - cookies = {c['name']: c['value'] for c in v['request']['cookies']} - headers = get_headers(v) + cookies = {c['name']: c['value'] for c in v['request']['cookies'] if c['name'] != "oai-did"} + headers = v_headers if not accessToken: raise NoValidHarFileError("No accessToken found in .har files") if not chatArks: - return None, accessToken, cookies, headers - return chatArks.pop(), accessToken, cookies, headers + return None, accessToken, cookies, headers, proofTokens + return chatArks.pop(), accessToken, cookies, headers, proofTokens def get_headers(entry) -> dict: return {h['name'].lower(): h['value'] for h in entry['request']['headers'] if h['name'].lower() not in ['content-length', 'cookie'] and not h['name'].startswith(':')} @@ -101,7 +112,8 @@ def genArkReq(chatArk: arkReq) -> arkReq: async def sendRequest(tmpArk: arkReq, proxy: str = None): async with StreamSession(headers=tmpArk.arkHeader, cookies=tmpArk.arkCookies, proxies={"https": proxy}) as session: async with session.post(tmpArk.arkURL, data=tmpArk.arkBody) as response: - arkose = (await response.json()).get("token") + data = await response.json() + arkose = data.get("token") if "sup=1|rid=" not in arkose: return RuntimeError("No valid arkose token generated") return arkose @@ -131,10 +143,10 @@ def getN() -> str: return base64.b64encode(timestamp.encode()).decode() async def getArkoseAndAccessToken(proxy: str) -> tuple[str, str, dict, dict]: - global chatArk, accessToken, cookies, headers + global chatArk, accessToken, cookies, headers, proofTokens if chatArk is None or accessToken is None: - chatArk, accessToken, cookies, headers = readHAR() + chatArk, accessToken, cookies, headers, proofTokens = readHAR() if chatArk is None: - return None, accessToken, cookies, headers + return None, accessToken, cookies, headers, proofTokens newReq = genArkReq(chatArk) - return await sendRequest(newReq, proxy), accessToken, cookies, headers + return await sendRequest(newReq, proxy), accessToken, cookies, headers, proofTokens diff --git a/g4f/Provider/openai/proofofwork.py b/g4f/Provider/openai/proofofwork.py index 51d96bc4..cbce153f 100644 --- a/g4f/Provider/openai/proofofwork.py +++ b/g4f/Provider/openai/proofofwork.py @@ -2,35 +2,51 @@ import random import hashlib import json import base64 -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone -def generate_proof_token(required: bool, seed: str, difficulty: str, user_agent: str): +proof_token_cache: dict = {} + +def generate_proof_token(required: bool, seed: str = None, difficulty: str = None, user_agent: str = None, proofTokens: list = None): if not required: return - - cores = [8, 12, 16, 24] - screens = [3000, 4000, 6000] - - core = random.choice(cores) - screen = random.choice(screens) + if seed is not None and seed in proof_token_cache: + return proof_token_cache[seed] # Get current UTC time now_utc = datetime.now(timezone.utc) parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT') - config = [core + screen, parse_time, None, 0, user_agent, "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js","dpl=53d243de46ff04dadd88d293f088c2dd728f126f","en","en-US",442,"plugins−[object PluginArray]","","alert"] - - diff_len = len(difficulty) // 2 - + if proofTokens: + config = random.choice(proofTokens) + else: + screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4]) + config = [ + screen, parse_time, + None, 0, user_agent, + "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js", + "dpl=1440a687921de39ff5ee56b92807faaadce73f13","en","en-US", + None, + "plugins−[object PluginArray]", + random.choice(["_reactListeningcfilawjnerp", "_reactListening9ne2dfo1i47", "_reactListening410nzwhan2a"]), + random.choice(["alert", "ontransitionend", "onprogress"]) + ] + + config[1] = parse_time + config[4] = user_agent + config[7] = random.randint(101, 2100) + + diff_len = None if difficulty is None else len(difficulty) for i in range(100000): config[3] = i json_data = json.dumps(config) base = base64.b64encode(json_data.encode()).decode() - hash_value = hashlib.sha3_512((seed + base).encode()).digest() + hash_value = hashlib.sha3_512((seed or "" + base).encode()).digest() - if hash_value.hex()[:diff_len] <= difficulty: - result = "gAAAAAB" + base - return result + if difficulty is None or hash_value.hex()[:diff_len] <= difficulty: + if seed is None: + return "gAAAAAC" + base + proof_token_cache[seed] = "gAAAAAB" + base + return proof_token_cache[seed] fallback_base = base64.b64encode(f'"{seed}"'.encode()).decode() return "gAAAAABwQ8Lk5FbGpA2NcR9dShT6gYjU7VxZ4D" + fallback_base |