import React, { useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import Sidebar from "@/components/SettingsSidebar";
import { isMobile } from "react-device-detect";
import System from "@/models/system";
import showToast from "@/utils/toast";
import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import OpenAiLogo from "@/media/llmprovider/openai.png";
import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
import GeminiLogo from "@/media/llmprovider/gemini.png";
import OllamaLogo from "@/media/llmprovider/ollama.png";
import NovitaLogo from "@/media/llmprovider/novita.png";
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
import LocalAiLogo from "@/media/llmprovider/localai.png";
import TogetherAILogo from "@/media/llmprovider/togetherai.png";
import FireworksAILogo from "@/media/llmprovider/fireworksai.jpeg";
import MistralLogo from "@/media/llmprovider/mistral.jpeg";
import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
import PerplexityLogo from "@/media/llmprovider/perplexity.png";
import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import GroqLogo from "@/media/llmprovider/groq.png";
import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
import CohereLogo from "@/media/llmprovider/cohere.png";
import LiteLLMLogo from "@/media/llmprovider/litellm.png";
import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
import DeepSeekLogo from "@/media/llmprovider/deepseek.png";
import APIPieLogo from "@/media/llmprovider/apipie.png";
import XAILogo from "@/media/llmprovider/xai.png";
import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png";
import PreLoader from "@/components/Preloader";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions";
import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions";
import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions";
import FireworksAiOptions from "@/components/LLMSelection/FireworksAiOptions";
import MistralOptions from "@/components/LLMSelection/MistralOptions";
import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions";
import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions";
import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions";
import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions";
import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions";
import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions";
import LLMItem from "@/components/LLMSelection/LLMItem";
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
import CTAButton from "@/components/lib/CTAButton";
export const AVAILABLE_LLM_PROVIDERS = [
{
name: "OpenAI",
value: "openai",
logo: OpenAiLogo,
options: (settings) => ,
description: "The standard option for most non-commercial use.",
requiredConfig: ["OpenAiKey"],
},
{
name: "Azure OpenAI",
value: "azure",
logo: AzureOpenAiLogo,
options: (settings) => ,
description: "The enterprise option of OpenAI hosted on Azure services.",
requiredConfig: ["AzureOpenAiEndpoint"],
},
{
name: "Anthropic",
value: "anthropic",
logo: AnthropicLogo,
options: (settings) => ,
description: "A friendly AI Assistant hosted by Anthropic.",
requiredConfig: ["AnthropicApiKey"],
},
{
name: "Gemini",
value: "gemini",
logo: GeminiLogo,
options: (settings) => ,
description: "Google's largest and most capable AI model",
requiredConfig: ["GeminiLLMApiKey"],
},
{
name: "NVIDIA NIM",
value: "nvidia-nim",
logo: NvidiaNimLogo,
options: (settings) => ,
description:
"Run full parameter LLMs directly on your NVIDIA RTX GPU using NVIDIA NIM.",
requiredConfig: ["NvidiaNimLLMBasePath"],
},
{
name: "HuggingFace",
value: "huggingface",
logo: HuggingFaceLogo,
options: (settings) => ,
description:
"Access 150,000+ open-source LLMs and the world's AI community",
requiredConfig: [
"HuggingFaceLLMEndpoint",
"HuggingFaceLLMAccessToken",
"HuggingFaceLLMTokenLimit",
],
},
{
name: "Ollama",
value: "ollama",
logo: OllamaLogo,
options: (settings) => ,
description: "Run LLMs locally on your own machine.",
requiredConfig: ["OllamaLLMBasePath"],
},
{
name: "Novita AI",
value: "novita",
logo: NovitaLogo,
options: (settings) => ,
description:
"Reliable, Scalable, and Cost-Effective for LLMs from Novita AI",
requiredConfig: ["NovitaLLMApiKey"],
},
{
name: "LM Studio",
value: "lmstudio",
logo: LMStudioLogo,
options: (settings) => ,
description:
"Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
requiredConfig: ["LMStudioBasePath"],
},
{
name: "Local AI",
value: "localai",
logo: LocalAiLogo,
options: (settings) => ,
description: "Run LLMs locally on your own machine.",
requiredConfig: ["LocalAiApiKey", "LocalAiBasePath", "LocalAiTokenLimit"],
},
{
name: "Together AI",
value: "togetherai",
logo: TogetherAILogo,
options: (settings) => ,
description: "Run open source models from Together AI.",
requiredConfig: ["TogetherAiApiKey"],
},
{
name: "Fireworks AI",
value: "fireworksai",
logo: FireworksAILogo,
options: (settings) => ,
description:
"The fastest and most efficient inference engine to build production-ready, compound AI systems.",
requiredConfig: ["FireworksAiLLMApiKey"],
},
{
name: "Mistral",
value: "mistral",
logo: MistralLogo,
options: (settings) => ,
description: "Run open source models from Mistral AI.",
requiredConfig: ["MistralApiKey"],
},
{
name: "Perplexity AI",
value: "perplexity",
logo: PerplexityLogo,
options: (settings) => ,
description:
"Run powerful and internet-connected models hosted by Perplexity AI.",
requiredConfig: ["PerplexityApiKey"],
},
{
name: "OpenRouter",
value: "openrouter",
logo: OpenRouterLogo,
options: (settings) => ,
description: "A unified interface for LLMs.",
requiredConfig: ["OpenRouterApiKey"],
},
{
name: "Groq",
value: "groq",
logo: GroqLogo,
options: (settings) => ,
description:
"The fastest LLM inferencing available for real-time AI applications.",
requiredConfig: ["GroqApiKey"],
},
{
name: "KoboldCPP",
value: "koboldcpp",
logo: KoboldCPPLogo,
options: (settings) => ,
description: "Run local LLMs using koboldcpp.",
requiredConfig: [
"KoboldCPPModelPref",
"KoboldCPPBasePath",
"KoboldCPPTokenLimit",
],
},
{
name: "Oobabooga Web UI",
value: "textgenwebui",
logo: TextGenWebUILogo,
options: (settings) => ,
description: "Run local LLMs using Oobabooga's Text Generation Web UI.",
requiredConfig: ["TextGenWebUIBasePath", "TextGenWebUITokenLimit"],
},
{
name: "Cohere",
value: "cohere",
logo: CohereLogo,
options: (settings) => ,
description: "Run Cohere's powerful Command models.",
requiredConfig: ["CohereApiKey"],
},
{
name: "LiteLLM",
value: "litellm",
logo: LiteLLMLogo,
options: (settings) => ,
description: "Run LiteLLM's OpenAI compatible proxy for various LLMs.",
requiredConfig: ["LiteLLMBasePath"],
},
{
name: "DeepSeek",
value: "deepseek",
logo: DeepSeekLogo,
options: (settings) => ,
description: "Run DeepSeek's powerful LLMs.",
requiredConfig: ["DeepSeekApiKey"],
},
{
name: "AWS Bedrock",
value: "bedrock",
logo: AWSBedrockLogo,
options: (settings) => ,
description: "Run powerful foundation models privately with AWS Bedrock.",
requiredConfig: [
"AwsBedrockLLMAccessKeyId",
"AwsBedrockLLMAccessKey",
"AwsBedrockLLMRegion",
"AwsBedrockLLMModel",
],
},
{
name: "APIpie",
value: "apipie",
logo: APIPieLogo,
options: (settings) => ,
description: "A unified API of AI services from leading providers",
requiredConfig: ["ApipieLLMApiKey", "ApipieLLMModelPref"],
},
{
name: "Generic OpenAI",
value: "generic-openai",
logo: GenericOpenAiLogo,
options: (settings) => ,
description:
"Connect to any OpenAi-compatible service via a custom configuration",
requiredConfig: [
"GenericOpenAiBasePath",
"GenericOpenAiModelPref",
"GenericOpenAiTokenLimit",
"GenericOpenAiKey",
],
},
{
name: "xAI",
value: "xai",
logo: XAILogo,
options: (settings) => ,
description: "Run xAI's powerful LLMs like Grok-2 and more.",
requiredConfig: ["XAIApiKey", "XAIModelPref"],
},
];
export default function GeneralLLMPreference() {
const [saving, setSaving] = useState(false);
const [hasChanges, setHasChanges] = useState(false);
const [settings, setSettings] = useState(null);
const [loading, setLoading] = useState(true);
const [searchQuery, setSearchQuery] = useState("");
const [filteredLLMs, setFilteredLLMs] = useState([]);
const [selectedLLM, setSelectedLLM] = useState(null);
const [searchMenuOpen, setSearchMenuOpen] = useState(false);
const searchInputRef = useRef(null);
const { t } = useTranslation();
const handleSubmit = async (e) => {
e.preventDefault();
const form = e.target;
const data = { LLMProvider: selectedLLM };
const formData = new FormData(form);
for (var [key, value] of formData.entries()) data[key] = value;
const { error } = await System.updateSystem(data);
setSaving(true);
if (error) {
showToast(`Failed to save LLM settings: ${error}`, "error");
} else {
showToast("LLM preferences saved successfully.", "success");
}
setSaving(false);
setHasChanges(!!error);
};
const updateLLMChoice = (selection) => {
setSearchQuery("");
setSelectedLLM(selection);
setSearchMenuOpen(false);
setHasChanges(true);
};
const handleXButton = () => {
if (searchQuery.length > 0) {
setSearchQuery("");
if (searchInputRef.current) searchInputRef.current.value = "";
} else {
setSearchMenuOpen(!searchMenuOpen);
}
};
useEffect(() => {
async function fetchKeys() {
const _settings = await System.keys();
setSettings(_settings);
setSelectedLLM(_settings?.LLMProvider);
setLoading(false);
}
fetchKeys();
}, []);
useEffect(() => {
const filtered = AVAILABLE_LLM_PROVIDERS.filter((llm) =>
llm.name.toLowerCase().includes(searchQuery.toLowerCase())
);
setFilteredLLMs(filtered);
}, [searchQuery, selectedLLM]);
const selectedLLMObject = AVAILABLE_LLM_PROVIDERS.find(
(llm) => llm.value === selectedLLM
);
return (