import React, { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import Sidebar from "@/components/SettingsSidebar"; import { isMobile } from "react-device-detect"; import System from "@/models/system"; import showToast from "@/utils/toast"; import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png"; import OpenAiLogo from "@/media/llmprovider/openai.png"; import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png"; import AzureOpenAiLogo from "@/media/llmprovider/azure.png"; import AnthropicLogo from "@/media/llmprovider/anthropic.png"; import GeminiLogo from "@/media/llmprovider/gemini.png"; import OllamaLogo from "@/media/llmprovider/ollama.png"; import NovitaLogo from "@/media/llmprovider/novita.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; import FireworksAILogo from "@/media/llmprovider/fireworksai.jpeg"; import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; import GroqLogo from "@/media/llmprovider/groq.png"; import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png"; import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png"; import CohereLogo from "@/media/llmprovider/cohere.png"; import LiteLLMLogo from "@/media/llmprovider/litellm.png"; import AWSBedrockLogo from "@/media/llmprovider/bedrock.png"; import DeepSeekLogo from "@/media/llmprovider/deepseek.png"; import APIPieLogo from "@/media/llmprovider/apipie.png"; import XAILogo from "@/media/llmprovider/xai.png"; import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions"; import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions"; import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions"; import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; import FireworksAiOptions from "@/components/LLMSelection/FireworksAiOptions"; import MistralOptions from "@/components/LLMSelection/MistralOptions"; import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions"; import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions"; import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions"; import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions"; import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions"; import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions"; import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions"; import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions"; import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions"; import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions"; import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions"; import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; import CTAButton from "@/components/lib/CTAButton"; export const AVAILABLE_LLM_PROVIDERS = [ { name: "OpenAI", value: "openai", logo: OpenAiLogo, options: (settings) => , description: "The standard option for most non-commercial use.", requiredConfig: ["OpenAiKey"], }, { name: "Azure OpenAI", value: "azure", logo: AzureOpenAiLogo, options: (settings) => , description: "The enterprise option of OpenAI hosted on Azure services.", requiredConfig: ["AzureOpenAiEndpoint"], }, { name: "Anthropic", value: "anthropic", logo: AnthropicLogo, options: (settings) => , description: "A friendly AI Assistant hosted by Anthropic.", requiredConfig: ["AnthropicApiKey"], }, { name: "Gemini", value: "gemini", logo: GeminiLogo, options: (settings) => , description: "Google's largest and most capable AI model", requiredConfig: ["GeminiLLMApiKey"], }, { name: "NVIDIA NIM", value: "nvidia-nim", logo: NvidiaNimLogo, options: (settings) => , description: "Run full parameter LLMs directly on your NVIDIA RTX GPU using NVIDIA NIM.", requiredConfig: ["NvidiaNimLLMBasePath"], }, { name: "HuggingFace", value: "huggingface", logo: HuggingFaceLogo, options: (settings) => , description: "Access 150,000+ open-source LLMs and the world's AI community", requiredConfig: [ "HuggingFaceLLMEndpoint", "HuggingFaceLLMAccessToken", "HuggingFaceLLMTokenLimit", ], }, { name: "Ollama", value: "ollama", logo: OllamaLogo, options: (settings) => , description: "Run LLMs locally on your own machine.", requiredConfig: ["OllamaLLMBasePath"], }, { name: "Novita AI", value: "novita", logo: NovitaLogo, options: (settings) => , description: "Reliable, Scalable, and Cost-Effective for LLMs from Novita AI", requiredConfig: ["NovitaLLMApiKey"], }, { name: "LM Studio", value: "lmstudio", logo: LMStudioLogo, options: (settings) => , description: "Discover, download, and run thousands of cutting edge LLMs in a few clicks.", requiredConfig: ["LMStudioBasePath"], }, { name: "Local AI", value: "localai", logo: LocalAiLogo, options: (settings) => , description: "Run LLMs locally on your own machine.", requiredConfig: ["LocalAiApiKey", "LocalAiBasePath", "LocalAiTokenLimit"], }, { name: "Together AI", value: "togetherai", logo: TogetherAILogo, options: (settings) => , description: "Run open source models from Together AI.", requiredConfig: ["TogetherAiApiKey"], }, { name: "Fireworks AI", value: "fireworksai", logo: FireworksAILogo, options: (settings) => , description: "The fastest and most efficient inference engine to build production-ready, compound AI systems.", requiredConfig: ["FireworksAiLLMApiKey"], }, { name: "Mistral", value: "mistral", logo: MistralLogo, options: (settings) => , description: "Run open source models from Mistral AI.", requiredConfig: ["MistralApiKey"], }, { name: "Perplexity AI", value: "perplexity", logo: PerplexityLogo, options: (settings) => , description: "Run powerful and internet-connected models hosted by Perplexity AI.", requiredConfig: ["PerplexityApiKey"], }, { name: "OpenRouter", value: "openrouter", logo: OpenRouterLogo, options: (settings) => , description: "A unified interface for LLMs.", requiredConfig: ["OpenRouterApiKey"], }, { name: "Groq", value: "groq", logo: GroqLogo, options: (settings) => , description: "The fastest LLM inferencing available for real-time AI applications.", requiredConfig: ["GroqApiKey"], }, { name: "KoboldCPP", value: "koboldcpp", logo: KoboldCPPLogo, options: (settings) => , description: "Run local LLMs using koboldcpp.", requiredConfig: [ "KoboldCPPModelPref", "KoboldCPPBasePath", "KoboldCPPTokenLimit", ], }, { name: "Oobabooga Web UI", value: "textgenwebui", logo: TextGenWebUILogo, options: (settings) => , description: "Run local LLMs using Oobabooga's Text Generation Web UI.", requiredConfig: ["TextGenWebUIBasePath", "TextGenWebUITokenLimit"], }, { name: "Cohere", value: "cohere", logo: CohereLogo, options: (settings) => , description: "Run Cohere's powerful Command models.", requiredConfig: ["CohereApiKey"], }, { name: "LiteLLM", value: "litellm", logo: LiteLLMLogo, options: (settings) => , description: "Run LiteLLM's OpenAI compatible proxy for various LLMs.", requiredConfig: ["LiteLLMBasePath"], }, { name: "DeepSeek", value: "deepseek", logo: DeepSeekLogo, options: (settings) => , description: "Run DeepSeek's powerful LLMs.", requiredConfig: ["DeepSeekApiKey"], }, { name: "AWS Bedrock", value: "bedrock", logo: AWSBedrockLogo, options: (settings) => , description: "Run powerful foundation models privately with AWS Bedrock.", requiredConfig: [ "AwsBedrockLLMAccessKeyId", "AwsBedrockLLMAccessKey", "AwsBedrockLLMRegion", "AwsBedrockLLMModel", ], }, { name: "APIpie", value: "apipie", logo: APIPieLogo, options: (settings) => , description: "A unified API of AI services from leading providers", requiredConfig: ["ApipieLLMApiKey", "ApipieLLMModelPref"], }, { name: "Generic OpenAI", value: "generic-openai", logo: GenericOpenAiLogo, options: (settings) => , description: "Connect to any OpenAi-compatible service via a custom configuration", requiredConfig: [ "GenericOpenAiBasePath", "GenericOpenAiModelPref", "GenericOpenAiTokenLimit", "GenericOpenAiKey", ], }, { name: "xAI", value: "xai", logo: XAILogo, options: (settings) => , description: "Run xAI's powerful LLMs like Grok-2 and more.", requiredConfig: ["XAIApiKey", "XAIModelPref"], }, ]; export default function GeneralLLMPreference() { const [saving, setSaving] = useState(false); const [hasChanges, setHasChanges] = useState(false); const [settings, setSettings] = useState(null); const [loading, setLoading] = useState(true); const [searchQuery, setSearchQuery] = useState(""); const [filteredLLMs, setFilteredLLMs] = useState([]); const [selectedLLM, setSelectedLLM] = useState(null); const [searchMenuOpen, setSearchMenuOpen] = useState(false); const searchInputRef = useRef(null); const { t } = useTranslation(); const handleSubmit = async (e) => { e.preventDefault(); const form = e.target; const data = { LLMProvider: selectedLLM }; const formData = new FormData(form); for (var [key, value] of formData.entries()) data[key] = value; const { error } = await System.updateSystem(data); setSaving(true); if (error) { showToast(`Failed to save LLM settings: ${error}`, "error"); } else { showToast("LLM preferences saved successfully.", "success"); } setSaving(false); setHasChanges(!!error); }; const updateLLMChoice = (selection) => { setSearchQuery(""); setSelectedLLM(selection); setSearchMenuOpen(false); setHasChanges(true); }; const handleXButton = () => { if (searchQuery.length > 0) { setSearchQuery(""); if (searchInputRef.current) searchInputRef.current.value = ""; } else { setSearchMenuOpen(!searchMenuOpen); } }; useEffect(() => { async function fetchKeys() { const _settings = await System.keys(); setSettings(_settings); setSelectedLLM(_settings?.LLMProvider); setLoading(false); } fetchKeys(); }, []); useEffect(() => { const filtered = AVAILABLE_LLM_PROVIDERS.filter((llm) => llm.name.toLowerCase().includes(searchQuery.toLowerCase()) ); setFilteredLLMs(filtered); }, [searchQuery, selectedLLM]); const selectedLLMObject = AVAILABLE_LLM_PROVIDERS.find( (llm) => llm.value === selectedLLM ); return (
{loading ? (
) : (

{t("llm.title")}

{t("llm.description")}

{hasChanges && ( handleSubmit()} className="mt-3 mr-0 -mb-14 z-10" > {saving ? "Saving..." : "Save changes"} )}
{t("llm.provider")}
{searchMenuOpen && (
setSearchMenuOpen(false)} /> )} {searchMenuOpen ? (
setSearchQuery(e.target.value)} ref={searchInputRef} onKeyDown={(e) => { if (e.key === "Enter") e.preventDefault(); }} />
{filteredLLMs.map((llm) => { return ( updateLLMChoice(llm.value)} /> ); })}
) : ( )}
setHasChanges(true)} className="mt-4 flex flex-col gap-y-1" > {selectedLLM && AVAILABLE_LLM_PROVIDERS.find( (llm) => llm.value === selectedLLM )?.options?.(settings)}
)}
); }