mirror of https://github.com/xtekky/gpt4free
Add local models to gui, Fix You Provider, add AsyncClient
Heiner Lohaus
1 month ago
parent
674ba8f2c5
commit
b35dfcd1b0
@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..locals.models import get_models
|
||||
try:
|
||||
from ..locals.provider import LocalProvider
|
||||
has_requirements = True
|
||||
except ModuleNotFoundError:
|
||||
has_requirements = False
|
||||
|
||||
from ..typing import Messages, CreateResult
|
||||
from ..providers.base_provider import AbstractProvider, ProviderModelMixin
|
||||
from ..errors import MissingRequirementsError
|
||||
|
||||
class Local(AbstractProvider, ProviderModelMixin):
|
||||
working = True
|
||||
supports_message_history = True
|
||||
supports_system_message = True
|
||||
supports_stream = True
|
||||
|
||||
@classmethod
|
||||
def get_models(cls):
|
||||
if not cls.models:
|
||||
cls.models = list(get_models())
|
||||
cls.default_model = cls.models[0]
|
||||
return cls.models
|
||||
|
||||
@classmethod
|
||||
def create_completion(
|
||||
cls,
|
||||
model: str,
|
||||
messages: Messages,
|
||||
stream: bool,
|
||||
**kwargs
|
||||
) -> CreateResult:
|
||||
if not has_requirements:
|
||||
raise MissingRequirementsError('Install "gpt4all" package | pip install -U g4f[local]')
|
||||
return LocalProvider.create_completion(
|
||||
cls.get_model(model),
|
||||
messages,
|
||||
stream,
|
||||
**kwargs
|
||||
)
|
||||
@ -0,0 +1,70 @@
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
|
||||
from ...requests import StreamSession, raise_for_status
|
||||
|
||||
class NoValidHarFileError(Exception):
|
||||
...
|
||||
|
||||
class arkReq:
|
||||
def __init__(self, arkURL, arkHeaders, arkBody, arkCookies, userAgent):
|
||||
self.arkURL = arkURL
|
||||
self.arkHeaders = arkHeaders
|
||||
self.arkBody = arkBody
|
||||
self.arkCookies = arkCookies
|
||||
self.userAgent = userAgent
|
||||
|
||||
arkPreURL = "https://telemetry.stytch.com/submit"
|
||||
chatArks: list = None
|
||||
|
||||
def readHAR():
|
||||
dirPath = "./"
|
||||
harPath = []
|
||||
chatArks = []
|
||||
for root, dirs, files in os.walk(dirPath):
|
||||
for file in files:
|
||||
if file.endswith(".har"):
|
||||
harPath.append(os.path.join(root, file))
|
||||
if harPath:
|
||||
break
|
||||
if not harPath:
|
||||
raise NoValidHarFileError("No .har file found")
|
||||
for path in harPath:
|
||||
with open(path, 'rb') as file:
|
||||
try:
|
||||
harFile = json.load(file)
|
||||
except json.JSONDecodeError:
|
||||
# Error: not a HAR file!
|
||||
continue
|
||||
for v in harFile['log']['entries']:
|
||||
if arkPreURL in v['request']['url']:
|
||||
chatArks.append(parseHAREntry(v))
|
||||
if not chatArks:
|
||||
raise NoValidHarFileError("No telemetry in .har files found")
|
||||
return chatArks
|
||||
|
||||
def parseHAREntry(entry) -> arkReq:
|
||||
tmpArk = arkReq(
|
||||
arkURL=entry['request']['url'],
|
||||
arkHeaders={h['name'].lower(): h['value'] for h in entry['request']['headers'] if h['name'].lower() not in ['content-length', 'cookie'] and not h['name'].startswith(':')},
|
||||
arkBody=entry['request']['postData']['text'],
|
||||
arkCookies={c['name']: c['value'] for c in entry['request']['cookies']},
|
||||
userAgent=""
|
||||
)
|
||||
tmpArk.userAgent = tmpArk.arkHeaders.get('user-agent', '')
|
||||
return tmpArk
|
||||
|
||||
async def sendRequest(tmpArk: arkReq, proxy: str = None):
|
||||
async with StreamSession(headers=tmpArk.arkHeaders, cookies=tmpArk.arkCookies, proxies={"all": proxy}) as session:
|
||||
async with session.post(tmpArk.arkURL, data=tmpArk.arkBody) as response:
|
||||
await raise_for_status(response)
|
||||
return await response.text()
|
||||
|
||||
async def get_dfp_telemetry_id(proxy: str = None):
|
||||
return str(uuid.uuid4())
|
||||
global chatArks
|
||||
if chatArks is None:
|
||||
chatArks = readHAR()
|
||||
return await sendRequest(random.choice(chatArks), proxy)
|
||||
@ -0,0 +1,3 @@
|
||||
from .stubs import ChatCompletion, ChatCompletionChunk, ImagesResponse
|
||||
from .client import Client
|
||||
from .async_client import AsyncClient
|
||||
@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Union
|
||||
|
||||
from .. import debug, version
|
||||
from ..errors import ProviderNotFoundError, ModelNotFoundError, ProviderNotWorkingError, StreamNotSupportedError
|
||||
from ..models import Model, ModelUtils
|
||||
from ..Provider import ProviderUtils
|
||||
from ..providers.types import BaseRetryProvider, ProviderType
|
||||
from ..providers.retry_provider import IterProvider
|
||||
|
||||
def convert_to_provider(provider: str) -> ProviderType:
|
||||
if " " in provider:
|
||||
provider_list = [ProviderUtils.convert[p] for p in provider.split() if p in ProviderUtils.convert]
|
||||
if not provider_list:
|
||||
raise ProviderNotFoundError(f'Providers not found: {provider}')
|
||||
provider = IterProvider(provider_list)
|
||||
elif provider in ProviderUtils.convert:
|
||||
provider = ProviderUtils.convert[provider]
|
||||
elif provider:
|
||||
raise ProviderNotFoundError(f'Provider not found: {provider}')
|
||||
return provider
|
||||
|
||||
def get_model_and_provider(model : Union[Model, str],
|
||||
provider : Union[ProviderType, str, None],
|
||||
stream : bool,
|
||||
ignored : list[str] = None,
|
||||
ignore_working: bool = False,
|
||||
ignore_stream: bool = False) -> tuple[str, ProviderType]:
|
||||
"""
|
||||
Retrieves the model and provider based on input parameters.
|
||||
|
||||
Args:
|
||||
model (Union[Model, str]): The model to use, either as an object or a string identifier.
|
||||
provider (Union[ProviderType, str, None]): The provider to use, either as an object, a string identifier, or None.
|
||||
stream (bool): Indicates if the operation should be performed as a stream.
|
||||
ignored (list[str], optional): List of provider names to be ignored.
|
||||
ignore_working (bool, optional): If True, ignores the working status of the provider.
|
||||
ignore_stream (bool, optional): If True, ignores the streaming capability of the provider.
|
||||
|
||||
Returns:
|
||||
tuple[str, ProviderType]: A tuple containing the model name and the provider type.
|
||||
|
||||
Raises:
|
||||
ProviderNotFoundError: If the provider is not found.
|
||||
ModelNotFoundError: If the model is not found.
|
||||
ProviderNotWorkingError: If the provider is not working.
|
||||
StreamNotSupportedError: If streaming is not supported by the provider.
|
||||
"""
|
||||
if debug.version_check:
|
||||
debug.version_check = False
|
||||
version.utils.check_version()
|
||||
|
||||
if isinstance(provider, str):
|
||||
provider = convert_to_provider(provider)
|
||||
|
||||
if isinstance(model, str):
|
||||
if model in ModelUtils.convert:
|
||||
model = ModelUtils.convert[model]
|
||||
|
||||
if not provider:
|
||||
if isinstance(model, str):
|
||||
raise ModelNotFoundError(f'Model not found: {model}')
|
||||
provider = model.best_provider
|
||||
|
||||
if not provider:
|
||||
raise ProviderNotFoundError(f'No provider found for model: {model}')
|
||||
|
||||
if isinstance(model, Model):
|
||||
model = model.name
|
||||
|
||||
if not ignore_working and not provider.working:
|
||||
raise ProviderNotWorkingError(f'{provider.__name__} is not working')
|
||||
|
||||
if not ignore_working and isinstance(provider, BaseRetryProvider):
|
||||
provider.providers = [p for p in provider.providers if p.working]
|
||||
|
||||
if ignored and isinstance(provider, BaseRetryProvider):
|
||||
provider.providers = [p for p in provider.providers if p.__name__ not in ignored]
|
||||
|
||||
if not ignore_stream and not provider.supports_stream and stream:
|
||||
raise StreamNotSupportedError(f'{provider.__name__} does not support "stream" argument')
|
||||
|
||||
if debug.logging:
|
||||
if model:
|
||||
print(f'Using {provider.__name__} provider and {model} model')
|
||||
else:
|
||||
print(f'Using {provider.__name__} provider')
|
||||
|
||||
debug.last_provider = provider
|
||||
debug.last_model = model
|
||||
|
||||
return model, provider
|
||||
|
||||
def get_last_provider(as_dict: bool = False) -> Union[ProviderType, dict[str, str]]:
|
||||
"""
|
||||
Retrieves the last used provider.
|
||||
|
||||
Args:
|
||||
as_dict (bool, optional): If True, returns the provider information as a dictionary.
|
||||
|
||||
Returns:
|
||||
Union[ProviderType, dict[str, str]]: The last used provider, either as an object or a dictionary.
|
||||
"""
|
||||
last = debug.last_provider
|
||||
if isinstance(last, BaseRetryProvider):
|
||||
last = last.last_provider
|
||||
if last and as_dict:
|
||||
return {
|
||||
"name": last.__name__,
|
||||
"url": last.url,
|
||||
"model": debug.last_model,
|
||||
}
|
||||
return last
|
||||
@ -1,42 +0,0 @@
|
||||
import os
|
||||
|
||||
from gpt4all import GPT4All
|
||||
from ._models import models
|
||||
|
||||
class LocalProvider:
|
||||
@staticmethod
|
||||
def create_completion(model, messages, stream, **kwargs):
|
||||
if model not in models:
|
||||
raise ValueError(f"Model '{model}' not found / not yet implemented")
|
||||
|
||||
model = models[model]
|
||||
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models/')
|
||||
full_model_path = os.path.join(model_dir, model['path'])
|
||||
|
||||
if not os.path.isfile(full_model_path):
|
||||
print(f"Model file '{full_model_path}' not found.")
|
||||
download = input(f'Do you want to download {model["path"]} ? [y/n]')
|
||||
|
||||
if download in ['y', 'Y']:
|
||||
GPT4All.download_model(model['path'], model_dir)
|
||||
else:
|
||||
raise ValueError(f"Model '{model['path']}' not found.")
|
||||
|
||||
model = GPT4All(model_name=model['path'],
|
||||
#n_threads=8,
|
||||
verbose=False,
|
||||
allow_download=False,
|
||||
model_path=model_dir)
|
||||
|
||||
system_template = next((message['content'] for message in messages if message['role'] == 'system'),
|
||||
'A chat between a curious user and an artificial intelligence assistant.')
|
||||
|
||||
prompt_template = 'USER: {0}\nASSISTANT: '
|
||||
conversation = '\n'.join(f"{msg['role'].upper()}: {msg['content']}" for msg in messages) + "\nASSISTANT: "
|
||||
|
||||
with model.chat_session(system_template, prompt_template):
|
||||
if stream:
|
||||
for token in model.generate(conversation, streaming=True):
|
||||
yield token
|
||||
else:
|
||||
yield model.generate(conversation)
|
||||
@ -1,86 +0,0 @@
|
||||
models = {
|
||||
"mistral-7b": {
|
||||
"path": "mistral-7b-openorca.gguf2.Q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
|
||||
"system": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
|
||||
},
|
||||
"mistral-7b-instruct": {
|
||||
"path": "mistral-7b-instruct-v0.1.Q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "[INST] %1 [/INST]",
|
||||
"system": None
|
||||
},
|
||||
"gpt4all-falcon": {
|
||||
"path": "gpt4all-falcon-newbpe-q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "### Instruction:\n%1\n### Response:\n",
|
||||
"system": None
|
||||
},
|
||||
"orca-2": {
|
||||
"path": "orca-2-13b.Q4_0.gguf",
|
||||
"ram": "16",
|
||||
"prompt": None,
|
||||
"system": None
|
||||
},
|
||||
"wizardlm-13b": {
|
||||
"path": "wizardlm-13b-v1.2.Q4_0.gguf",
|
||||
"ram": "16",
|
||||
"prompt": None,
|
||||
"system": None
|
||||
},
|
||||
"nous-hermes-llama2": {
|
||||
"path": "nous-hermes-llama2-13b.Q4_0.gguf",
|
||||
"ram": "16",
|
||||
"prompt": "### Instruction:\n%1\n### Response:\n",
|
||||
"system": None
|
||||
},
|
||||
"gpt4all-13b-snoozy": {
|
||||
"path": "gpt4all-13b-snoozy-q4_0.gguf",
|
||||
"ram": "16",
|
||||
"prompt": None,
|
||||
"system": None
|
||||
},
|
||||
"mpt-7b-chat": {
|
||||
"path": "mpt-7b-chat-newbpe-q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
|
||||
"system": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
|
||||
},
|
||||
"orca-mini-3b": {
|
||||
"path": "orca-mini-3b-gguf2-q4_0.gguf",
|
||||
"ram": "4",
|
||||
"prompt": "### User:\n%1\n### Response:\n",
|
||||
"system": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
|
||||
},
|
||||
"replit-code-3b": {
|
||||
"path": "replit-code-v1_5-3b-newbpe-q4_0.gguf",
|
||||
"ram": "4",
|
||||
"prompt": "%1",
|
||||
"system": None
|
||||
},
|
||||
"starcoder": {
|
||||
"path": "starcoder-newbpe-q4_0.gguf",
|
||||
"ram": "4",
|
||||
"prompt": "%1",
|
||||
"system": None
|
||||
},
|
||||
"rift-coder-7b": {
|
||||
"path": "rift-coder-v0-7b-q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "%1",
|
||||
"system": None
|
||||
},
|
||||
"all-MiniLM-L6-v2": {
|
||||
"path": "all-MiniLM-L6-v2-f16.gguf",
|
||||
"ram": "1",
|
||||
"prompt": None,
|
||||
"system": None
|
||||
},
|
||||
"mistral-7b-german": {
|
||||
"path": "em_german_mistral_v01.Q4_0.gguf",
|
||||
"ram": "8",
|
||||
"prompt": "USER: %1 ASSISTANT: ",
|
||||
"system": "Du bist ein hilfreicher Assistent. "
|
||||
}
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
.
|
||||
@ -0,0 +1,50 @@
|
||||
|
||||
import os
|
||||
import requests
|
||||
import json
|
||||
|
||||
from ..requests.raise_for_status import raise_for_status
|
||||
|
||||
def load_models():
|
||||
response = requests.get("https://gpt4all.io/models/models3.json")
|
||||
raise_for_status(response)
|
||||
return format_models(response.json())
|
||||
|
||||
def get_model_name(filename: str) -> str:
|
||||
name = filename.split(".", 1)[0]
|
||||
for replace in ["-v1_5", "-v1", "-q4_0", "_v01", "-v0", "-f16", "-gguf2", "-newbpe"]:
|
||||
name = name.replace(replace, "")
|
||||
return name
|
||||
|
||||
def format_models(models: list) -> dict:
|
||||
return {get_model_name(model["filename"]): {
|
||||
"path": model["filename"],
|
||||
"ram": model["ramrequired"],
|
||||
"prompt": model["promptTemplate"] if "promptTemplate" in model else None,
|
||||
"system": model["systemPrompt"] if "systemPrompt" in model else None,
|
||||
} for model in models}
|
||||
|
||||
def read_models(file_path: str):
|
||||
with open(file_path, "rb") as f:
|
||||
return json.load(f)
|
||||
|
||||
def save_models(file_path: str, data):
|
||||
with open(file_path, 'w') as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
||||
def get_model_dir() -> str:
|
||||
local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_dir = os.path.dirname(os.path.dirname(local_dir))
|
||||
model_dir = os.path.join(project_dir, "models")
|
||||
if os.path.exists(model_dir):
|
||||
return model_dir
|
||||
|
||||
def get_models() -> dict[str, dict]:
|
||||
model_dir = get_model_dir()
|
||||
file_path = os.path.join(model_dir, "models.json")
|
||||
if os.path.isfile(file_path):
|
||||
return read_models(file_path)
|
||||
else:
|
||||
models = load_models()
|
||||
save_models(file_path, models)
|
||||
return models
|
||||
@ -0,0 +1,72 @@
|
||||
import os
|
||||
|
||||
from gpt4all import GPT4All
|
||||
from .models import get_models
|
||||
from ..typing import Messages
|
||||
|
||||
MODEL_LIST: dict[str, dict] = None
|
||||
|
||||
def find_model_dir(model_file: str) -> str:
|
||||
local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_dir = os.path.dirname(os.path.dirname(local_dir))
|
||||
|
||||
new_model_dir = os.path.join(project_dir, "models")
|
||||
new_model_file = os.path.join(new_model_dir, model_file)
|
||||
if os.path.isfile(new_model_file):
|
||||
return new_model_dir
|
||||
|
||||
old_model_dir = os.path.join(local_dir, "models")
|
||||
old_model_file = os.path.join(old_model_dir, model_file)
|
||||
if os.path.isfile(old_model_file):
|
||||
return old_model_dir
|
||||
|
||||
working_dir = "./"
|
||||
for root, dirs, files in os.walk(working_dir):
|
||||
if model_file in files:
|
||||
return root
|
||||
|
||||
return new_model_dir
|
||||
|
||||
class LocalProvider:
|
||||
@staticmethod
|
||||
def create_completion(model: str, messages: Messages, stream: bool = False, **kwargs):
|
||||
global MODEL_LIST
|
||||
if MODEL_LIST is None:
|
||||
MODEL_LIST = get_models()
|
||||
if model not in MODEL_LIST:
|
||||
raise ValueError(f'Model "{model}" not found / not yet implemented')
|
||||
|
||||
model = MODEL_LIST[model]
|
||||
model_file = model["path"]
|
||||
model_dir = find_model_dir(model_file)
|
||||
if not os.path.isfile(os.path.join(model_dir, model_file)):
|
||||
print(f'Model file "models/{model_file}" not found.')
|
||||
download = input(f"Do you want to download {model_file}? [y/n]: ")
|
||||
if download in ["y", "Y"]:
|
||||
GPT4All.download_model(model_file, model_dir)
|
||||
else:
|
||||
raise ValueError(f'Model "{model_file}" not found.')
|
||||
|
||||
model = GPT4All(model_name=model_file,
|
||||
#n_threads=8,
|
||||
verbose=False,
|
||||
allow_download=False,
|
||||
model_path=model_dir)
|
||||
|
||||
system_message = "\n".join(message["content"] for message in messages if message["role"] == "system")
|
||||
if system_message:
|
||||
system_message = "A chat between a curious user and an artificial intelligence assistant."
|
||||
|
||||
prompt_template = "USER: {0}\nASSISTANT: "
|
||||
conversation = "\n" . join(
|
||||
f"{message['role'].upper()}: {message['content']}"
|
||||
for message in messages
|
||||
if message["role"] != "system"
|
||||
) + "\nASSISTANT: "
|
||||
|
||||
with model.chat_session(system_message, prompt_template):
|
||||
if stream:
|
||||
for token in model.generate(conversation, streaming=True):
|
||||
yield token
|
||||
else:
|
||||
yield model.generate(conversation)
|
||||
Loading…
Reference in New Issue