diff --git a/assets/icons/model-logo-gemini.svg b/assets/icons/model-logo-gemini.svg new file mode 100644 index 00000000..ff9f5b1b --- /dev/null +++ b/assets/icons/model-logo-gemini.svg @@ -0,0 +1 @@ +Gemini \ No newline at end of file diff --git a/assets/icons/model-logo-openai.svg b/assets/icons/model-logo-openai.svg new file mode 100644 index 00000000..31cb9463 --- /dev/null +++ b/assets/icons/model-logo-openai.svg @@ -0,0 +1 @@ +OpenAI \ No newline at end of file diff --git a/components/ModelSelector.vue b/components/ModelSelector.vue index b975fd7f..5c326835 100644 --- a/components/ModelSelector.vue +++ b/components/ModelSelector.vue @@ -176,6 +176,8 @@ const userConfig = await getUserConfig() const ollamaBaseUrl = userConfig.llm.backends.ollama.baseUrl.toRef() const lmStudioBaseUrl = userConfig.llm.backends.lmStudio.baseUrl.toRef() const commonModel = userConfig.llm.model.toRef() +const geminiModel = userConfig.llm.backends.gemini.model.toRef() +const openaiModel = userConfig.llm.backends.openai.model.toRef() const translationModel = userConfig.translation.model.toRef() const endpointType = userConfig.llm.endpointType.toRef() const translationEndpointType = userConfig.translation.endpointType.toRef() @@ -205,6 +207,8 @@ const modelListUpdating = computed(() => { const modelOptions = computed(() => { const ollamaModels = modelList.value.filter((model) => model.backend === 'ollama') const lmStudioModels = modelList.value.filter((model) => model.backend === 'lm-studio') + const geminiModels = modelList.value.filter((model) => model.backend === 'gemini') + const openaiModels = modelList.value.filter((model) => model.backend === 'openai') const webllmModels = modelList.value.filter((model) => model.backend === 'web-llm') const makeModelOptions = (model: typeof modelList.value[number]) => ({ type: 'option' as const, id: `${model.backend}#${model.model}`, label: model.name, model: { backend: model.backend, id: model.model } }) @@ -226,6 +230,18 @@ const modelOptions = computed(() => { ...lmStudioModels.map((model) => makeModelOptions(model)), ) } + if (geminiModels.length) { + options.push( + makeHeader(`Gemini Models (${geminiModels.length})`), + ...geminiModels.map((model) => makeModelOptions(model)), + ) + } + if (openaiModels.length) { + options.push( + makeHeader(`OpenAI Models (${openaiModels.length})`), + ...openaiModels.map((model) => makeModelOptions(model)), + ) + } return options } }) @@ -244,6 +260,12 @@ const selectedModel = computed({ if (props.modelType === 'chat') { commonModel.value = modelInfo.model.id endpointType.value = modelInfo.model.backend as LLMEndpointType + if (modelInfo.model.backend === 'gemini') { + geminiModel.value = modelInfo.model.id + } + else if (modelInfo.model.backend === 'openai') { + openaiModel.value = modelInfo.model.id + } } else { translationModel.value = modelInfo.model.id diff --git a/entrypoints/content/components/GmailTools/GmailComposeCard.vue b/entrypoints/content/components/GmailTools/GmailComposeCard.vue index e5ad6f12..d1f331ec 100644 --- a/entrypoints/content/components/GmailTools/GmailComposeCard.vue +++ b/entrypoints/content/components/GmailTools/GmailComposeCard.vue @@ -380,7 +380,13 @@ async function checkLLMBackendStatus() { } else if (status === 'backend-unavailable') { toast(t('errors.model_request_error'), { duration: 2000 }) - endpointType === 'ollama' ? showSettings({ scrollTarget: 'ollama-server-address-section' }) : showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + endpointType === 'ollama' + ? showSettings({ scrollTarget: 'ollama-server-address-section' }) + : endpointType === 'lm-studio' + ? showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + : endpointType === 'gemini' + ? showSettings({ scrollTarget: 'gemini-api-config-section' }) + : showSettings({ scrollTarget: 'openai-api-config-section' }) emit('close') return false } diff --git a/entrypoints/content/components/GmailTools/GmailReplyCard.vue b/entrypoints/content/components/GmailTools/GmailReplyCard.vue index d09d8641..d0e4eb90 100644 --- a/entrypoints/content/components/GmailTools/GmailReplyCard.vue +++ b/entrypoints/content/components/GmailTools/GmailReplyCard.vue @@ -299,7 +299,13 @@ async function checkLLMBackendStatus() { } else if (status === 'backend-unavailable') { toast(t('errors.model_request_error'), { duration: 2000 }) - endpointType === 'ollama' ? showSettings({ scrollTarget: 'ollama-server-address-section' }) : showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + endpointType === 'ollama' + ? showSettings({ scrollTarget: 'ollama-server-address-section' }) + : endpointType === 'lm-studio' + ? showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + : endpointType === 'gemini' + ? showSettings({ scrollTarget: 'gemini-api-config-section' }) + : showSettings({ scrollTarget: 'openai-api-config-section' }) emit('close') return false } diff --git a/entrypoints/content/components/WritingTools/SuggestionCard.vue b/entrypoints/content/components/WritingTools/SuggestionCard.vue index 18a9893b..380bc2ed 100644 --- a/entrypoints/content/components/WritingTools/SuggestionCard.vue +++ b/entrypoints/content/components/WritingTools/SuggestionCard.vue @@ -163,7 +163,13 @@ async function checkLLMBackendStatus() { } else if (status === 'backend-unavailable') { toast(t('errors.model_request_error'), { duration: 2000 }) - endpointType === 'ollama' ? showSettings({ scrollTarget: 'ollama-server-address-section' }) : showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + endpointType === 'ollama' + ? showSettings({ scrollTarget: 'ollama-server-address-section' }) + : endpointType === 'lm-studio' + ? showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + : endpointType === 'gemini' + ? showSettings({ scrollTarget: 'gemini-api-config-section' }) + : showSettings({ scrollTarget: 'openai-api-config-section' }) emit('close') return false } diff --git a/entrypoints/content/composables/useTranslator.ts b/entrypoints/content/composables/useTranslator.ts index 8a6bac90..130c4a20 100644 --- a/entrypoints/content/composables/useTranslator.ts +++ b/entrypoints/content/composables/useTranslator.ts @@ -73,7 +73,13 @@ async function _useTranslator() { const { status, endpointType } = await llmBackendStatusStore.checkCurrentBackendStatus() if (status === 'backend-unavailable') { toast('Failed to connect to Ollama server, please check your Ollama connection', { duration: 2000 }) - endpointType === 'ollama' ? showSettings({ scrollTarget: `ollama-server-address-section` }) : showSettings({ scrollTarget: `lm-studio-server-address-section` }) + endpointType === 'ollama' + ? showSettings({ scrollTarget: 'ollama-server-address-section' }) + : endpointType === 'lm-studio' + ? showSettings({ scrollTarget: 'lm-studio-server-address-section' }) + : endpointType === 'gemini' + ? showSettings({ scrollTarget: 'gemini-api-config-section' }) + : showSettings({ scrollTarget: 'openai-api-config-section' }) return } else if (status === 'no-model') { diff --git a/entrypoints/main-world-injected/llm-api.ts b/entrypoints/main-world-injected/llm-api.ts index 4787bbc2..c540bef7 100644 --- a/entrypoints/main-world-injected/llm-api.ts +++ b/entrypoints/main-world-injected/llm-api.ts @@ -43,7 +43,7 @@ export class LLMResponses { async create(params: ResponseCreateParamsStreaming): Promise async create(params: ResponseCreateParamsBase): Promise { const readyStatus = await checkBackendModel(params.model) - if (!readyStatus.backend) throw new Error('ollama is not connected') + if (!readyStatus.backend) throw new Error('backend is not connected') if (!readyStatus.model) throw new Error('model is not ready') if (params.stream) { return this.createStreamingResponse(params as ResponseCreateParamsStreaming) diff --git a/entrypoints/main-world-injected/utils.ts b/entrypoints/main-world-injected/utils.ts index b424cc3e..5a9d4991 100644 --- a/entrypoints/main-world-injected/utils.ts +++ b/entrypoints/main-world-injected/utils.ts @@ -21,10 +21,13 @@ export async function getBrowserAIConfig() { export async function checkBackendModel(model?: string) { const status = await m2cRpc.checkBackendModelReady(model) if (!status.backend || !status.model) { + const modelUnavailableMessage = model + ? `Model [${model}] is not available in current provider settings.` + : 'Model is not available in current provider settings.' await m2cRpc.emit('toast', { - message: !status.backend ? 'This page relies on the AI backend provided by Nativemind. Please ensure the backend is running.' : `Model [${model}] is not available. Please download the model from ollama.com.`, + message: !status.backend ? 'This page relies on the AI backend provided by Nativemind. Please ensure the backend is running.' : modelUnavailableMessage, type: 'error', - isHTML: true, + isHTML: false, duration: 5000, }) } diff --git a/entrypoints/settings/components/DebugSettings/index.vue b/entrypoints/settings/components/DebugSettings/index.vue index eff73c55..2a1d0e68 100644 --- a/entrypoints/settings/components/DebugSettings/index.vue +++ b/entrypoints/settings/components/DebugSettings/index.vue @@ -630,6 +630,8 @@ const articles = ref<{ type: 'html' | 'pdf', url: string, title: string, content const modelProviderOptions = [ { id: 'ollama' as const, label: 'Ollama' }, { id: 'lm-studio' as const, label: 'LM Studio' }, + { id: 'gemini' as const, label: 'Gemini API' }, + { id: 'openai' as const, label: 'OpenAI API' }, { id: 'web-llm' as const, label: 'Web LLM' }, ] diff --git a/entrypoints/settings/components/GeneralSettings/Blocks/GeminiConfiguration.vue b/entrypoints/settings/components/GeneralSettings/Blocks/GeminiConfiguration.vue new file mode 100644 index 00000000..593aa105 --- /dev/null +++ b/entrypoints/settings/components/GeneralSettings/Blocks/GeminiConfiguration.vue @@ -0,0 +1,200 @@ + + + diff --git a/entrypoints/settings/components/GeneralSettings/Blocks/OpenAIConfiguration.vue b/entrypoints/settings/components/GeneralSettings/Blocks/OpenAIConfiguration.vue new file mode 100644 index 00000000..d98f86e2 --- /dev/null +++ b/entrypoints/settings/components/GeneralSettings/Blocks/OpenAIConfiguration.vue @@ -0,0 +1,200 @@ + + + diff --git a/entrypoints/settings/components/GeneralSettings/index.vue b/entrypoints/settings/components/GeneralSettings/index.vue index 2beaa682..24fe1c22 100644 --- a/entrypoints/settings/components/GeneralSettings/index.vue +++ b/entrypoints/settings/components/GeneralSettings/index.vue @@ -28,6 +28,8 @@
+ +
{ const ollamaModels = modelList.value.filter((model) => model.backend === 'ollama') const lmStudioModels = modelList.value.filter((model) => model.backend === 'lm-studio') const webllmModels = modelList.value.filter((model) => model.backend === 'web-llm') + const geminiModels = modelList.value.filter((model) => model.backend === 'gemini') + const openaiModels = modelList.value.filter((model) => model.backend === 'openai') const makeModelOptions = (model: typeof modelList.value[number]) => ({ type: 'option' as const, id: `${model.backend}#${model.model}`, label: model.name, model: { backend: model.backend, id: model.model } }) const makeHeader = (label: string) => ({ type: 'header' as const, id: `header-${label}`, label, selectable: false }) @@ -114,6 +117,18 @@ const modelOptions = computed(() => { ...lmStudioModels.map((model) => makeModelOptions(model)), ) } + if (geminiModels.length) { + options.push( + makeHeader(`Gemini Models (${geminiModels.length})`), + ...geminiModels.map((model) => makeModelOptions(model)), + ) + } + if (openaiModels.length) { + options.push( + makeHeader(`OpenAI Models (${openaiModels.length})`), + ...openaiModels.map((model) => makeModelOptions(model)), + ) + } return options } }) diff --git a/entrypoints/sidepanel/components/Chat/index.vue b/entrypoints/sidepanel/components/Chat/index.vue index 397e53ab..a58f005b 100644 --- a/entrypoints/sidepanel/components/Chat/index.vue +++ b/entrypoints/sidepanel/components/Chat/index.vue @@ -160,6 +160,7 @@ import { ActionEvent, Chat, initChatSideEffects, + initTabChatSync, } from '../../utils/chat/index' import AttachmentSelector from '../AttachmentSelector.vue' import CameraButton from './CameraButton.vue' @@ -197,6 +198,7 @@ const chat = await Chat.getInstance() const contextAttachmentStorage = chat.contextAttachmentStorage initChatSideEffects() +initTabChatSync() // Track the final assistant/agent message for each reply block (between user turns) FOR triggering retry action const assistantActionMessageIds = computed(() => { diff --git a/entrypoints/sidepanel/components/Onboarding/BackendSelectionTutorialCard.vue b/entrypoints/sidepanel/components/Onboarding/BackendSelectionTutorialCard.vue index b4d97a6d..a00fb8e3 100644 --- a/entrypoints/sidepanel/components/Onboarding/BackendSelectionTutorialCard.vue +++ b/entrypoints/sidepanel/components/Onboarding/BackendSelectionTutorialCard.vue @@ -153,12 +153,18 @@ import { useLLMBackendStatusStore } from '@/utils/pinia-store/store' const log = logger.child('BackendSelectionTutorialCard') +const props = withDefaults(defineProps<{ + initialEndpointType?: 'ollama' | 'lm-studio' +}>(), { + initialEndpointType: 'ollama', +}) + const emit = defineEmits<{ (event: 'installed', backend: 'ollama' | 'lm-studio'): void - (event: 'settings'): void + (event: 'settings', backend: 'ollama' | 'lm-studio'): void }>() const llmBackendStatusStore = useLLMBackendStatusStore() -const selectedEndpointType = ref<'ollama' | 'lm-studio'>('ollama') +const selectedEndpointType = ref<'ollama' | 'lm-studio'>(props.initialEndpointType) const { t } = useI18n() const selectedEndpointName = computed(() => { @@ -204,6 +210,6 @@ const reScanOllama = async () => { } const onClickOpenSettings = () => { - emit('settings') + emit('settings', selectedEndpointType.value) } diff --git a/entrypoints/sidepanel/components/Onboarding/index.vue b/entrypoints/sidepanel/components/Onboarding/index.vue index 32cf3016..986486c1 100644 --- a/entrypoints/sidepanel/components/Onboarding/index.vue +++ b/entrypoints/sidepanel/components/Onboarding/index.vue @@ -9,7 +9,7 @@
@@ -33,6 +33,7 @@ class="bg-bg-primary rounded-lg overflow-hidden grow flex flex-col justify-between font" > @@ -83,9 +84,9 @@ const llmBackendStatusStore = useLLMBackendStatusStore() const endpointType = userConfig.llm.endpointType.toRef() const onboardingVersion = userConfig.ui.onboarding.version.toRef() const panel = ref<'tutorial' | 'model-downloader'>('tutorial') -const downloadEndpointType = ref<'ollama' | 'lm-studio'>('ollama') +const downloadEndpointType = ref<'ollama' | 'lm-studio'>(endpointType.value === 'lm-studio' ? 'lm-studio' : 'ollama') const isShow = computed(() => { - return onboardingVersion.value !== TARGET_ONBOARDING_VERSION + return false }) const onBackendInstalled = async (backend: 'ollama' | 'lm-studio') => { @@ -100,16 +101,28 @@ const onBackendInstalled = async (backend: 'ollama' | 'lm-studio') => { } } -const onOpenSettings = async () => { - endpointType.value = 'ollama' +const onOpenSettings = async (backend: 'ollama' | 'lm-studio') => { + endpointType.value = backend + downloadEndpointType.value = backend await close() showSettings() } const onModelDownloaderFinished = async () => { - endpointType.value = 'ollama' - await llmBackendStatusStore.updateOllamaConnectionStatus() - await llmBackendStatusStore.updateOllamaModelList() + const backend = downloadEndpointType.value + endpointType.value = backend + if (backend === 'ollama') { + await llmBackendStatusStore.updateOllamaConnectionStatus() + await llmBackendStatusStore.updateOllamaModelList() + } + else { + await llmBackendStatusStore.updateLMStudioConnectionStatus() + await llmBackendStatusStore.updateLMStudioModelList() + } + await close() +} + +const onCloseOnboarding = async () => { await close() } @@ -138,10 +151,22 @@ const close = async () => { onMounted(async () => { if (isShow.value) { - const ollamaSuccess = await llmBackendStatusStore.updateOllamaConnectionStatus() - if (ollamaSuccess) return onBackendInstalled('ollama') - const lmStudioSuccess = await llmBackendStatusStore.updateLMStudioConnectionStatus() - if (lmStudioSuccess) return onBackendInstalled('lm-studio') + if (endpointType.value !== 'ollama' && endpointType.value !== 'lm-studio') return + + const preferredBackend = endpointType.value + const fallbackBackend = preferredBackend === 'ollama' ? 'lm-studio' : 'ollama' + const tryBackend = async (backend: 'ollama' | 'lm-studio') => { + const success = backend === 'ollama' + ? await llmBackendStatusStore.updateOllamaConnectionStatus() + : await llmBackendStatusStore.updateLMStudioConnectionStatus() + if (success) { + await onBackendInstalled(backend) + return true + } + return false + } + if (await tryBackend(preferredBackend)) return + if (await tryBackend(fallbackBackend)) return } }) diff --git a/entrypoints/sidepanel/utils/agent/index.ts b/entrypoints/sidepanel/utils/agent/index.ts index 13481326..cc4e67d8 100644 --- a/entrypoints/sidepanel/utils/agent/index.ts +++ b/entrypoints/sidepanel/utils/agent/index.ts @@ -8,7 +8,7 @@ import { AssistantMessageV1 } from '@/types/chat' import { PromiseOr } from '@/types/common' import { Base64ImageData, ImageDataWithId } from '@/types/image' import { TagBuilderJSON } from '@/types/prompt' -import { AbortError, AiSDKError, AppError, ErrorCode, fromError, LMStudioLoadModelError, ModelNotFoundError, ModelRequestError, ParseFunctionCallError, UnknownError } from '@/utils/error' +import { AbortError, AiSDKError, AppError, ErrorCode, fromError, LMStudioLoadModelError, ModelNotFoundError, ModelRequestError, ModelRequestTimeoutError, ParseFunctionCallError, UnknownError } from '@/utils/error' import { useGlobalI18n } from '@/utils/i18n' import { generateRandomId } from '@/utils/id' import { InferredParams } from '@/utils/llm/tools/prompt-based/helpers' @@ -81,6 +81,26 @@ interface AgentOptions { } type AgentStatus = 'idle' | 'running' | 'error' +const MAX_ERROR_DETAILS_LENGTH = 1200 + +function sanitizeErrorDetails(message?: string) { + if (!message) return '' + const normalized = message.replace(/\s+/g, ' ').trim() + if (!normalized) return '' + return normalized + .replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [REDACTED]') + .replace(/sk-[A-Za-z0-9_-]+/g, 'sk-[REDACTED]') + .replace(/AIza[0-9A-Za-z_-]{20,}/g, 'AIza[REDACTED]') +} + +function appendErrorDetails(baseMessage: string, details?: string) { + const sanitized = sanitizeErrorDetails(details) + if (!sanitized) return baseMessage + const trimmed = sanitized.length > MAX_ERROR_DETAILS_LENGTH + ? `${sanitized.slice(0, MAX_ERROR_DETAILS_LENGTH)}...` + : sanitized + return `${baseMessage}\n\n\`\`\`\n${trimmed}\n\`\`\`` +} export class Agent { abortControllers: AbortController[] = [] @@ -328,10 +348,13 @@ export class Agent { currentLoopAssistantRawMessage.content += chunk.textDelta agentMessage.content += chunk.textDelta } - else if (chunk.type === 'reasoning') { - reasoningStart = reasoningStart || Date.now() - agentMessage.reasoningTime = reasoningStart ? Date.now() - reasoningStart : undefined - agentMessage.reasoning = (agentMessage.reasoning || '') + chunk.textDelta + else if ((chunk as { type: string }).type === 'reasoning' || (chunk as { type: string }).type === 'reasoning-delta') { + const reasoningChunk = chunk as { textDelta?: string } + if (reasoningChunk.textDelta) { + reasoningStart = reasoningStart || Date.now() + agentMessage.reasoningTime = reasoningStart ? Date.now() - reasoningStart : undefined + agentMessage.reasoning = (agentMessage.reasoning || '') + reasoningChunk.textDelta + } } else if (chunk.type === 'tool-call') { this.log.debug('Tool call received', chunk) @@ -418,7 +441,8 @@ export class Agent { const { t } = await useGlobalI18n() const errorMsg = await agentMessageManager.convertToAssistantMessage() errorMsg.isError = true - errorMsg.content = t('errors.model_not_found', { endpointType: error.endpointType === 'ollama' ? 'Ollama' : 'LM Studio' }) + const endpointTypeName = error.endpointType === 'ollama' ? 'Ollama' : error.endpointType === 'lm-studio' ? 'LM Studio' : error.endpointType === 'gemini' ? 'Gemini' : 'OpenAI' + errorMsg.content = appendErrorDetails(t('errors.model_not_found', { endpointType: endpointTypeName }), error.message) // unresolvable error, break the loop return false } @@ -426,7 +450,15 @@ export class Agent { const { t } = await useGlobalI18n() const errorMsg = await agentMessageManager.convertToAssistantMessage() errorMsg.isError = true - errorMsg.content = t('errors.model_request_error', { endpointType: error.endpointType === 'ollama' ? 'Ollama' : 'LM Studio' }) + const endpointTypeName = error.endpointType === 'ollama' ? 'Ollama' : error.endpointType === 'lm-studio' ? 'LM Studio' : error.endpointType === 'gemini' ? 'Gemini' : 'OpenAI' + errorMsg.content = appendErrorDetails(t('errors.model_request_error', { endpointType: endpointTypeName }), error.message) + return false + } + else if (error instanceof ModelRequestTimeoutError) { + const { t } = await useGlobalI18n() + const errorMsg = await agentMessageManager.convertToAssistantMessage() + errorMsg.isError = true + errorMsg.content = appendErrorDetails(t('errors.model_request_timeout'), error.message) return false } else if (error instanceof LMStudioLoadModelError) { diff --git a/entrypoints/sidepanel/utils/chat/chat.ts b/entrypoints/sidepanel/utils/chat/chat.ts index 599eea30..0c48fed0 100644 --- a/entrypoints/sidepanel/utils/chat/chat.ts +++ b/entrypoints/sidepanel/utils/chat/chat.ts @@ -160,8 +160,12 @@ export class ReactiveHistoryManager extends EventEmitter { async appendAssistantMessage(content: string = '') { const userConfig = await getUserConfig() - const model = this.temporaryModelOverride?.model ?? userConfig.llm.model.get() const endpointType = this.temporaryModelOverride?.endpointType ?? userConfig.llm.endpointType.get() + const model = this.temporaryModelOverride?.model ?? (endpointType === 'gemini' + ? userConfig.llm.backends.gemini.model.get() || userConfig.llm.model.get() + : endpointType === 'openai' + ? userConfig.llm.backends.openai.model.get() || userConfig.llm.model.get() + : userConfig.llm.model.get()) this.history.value.push({ id: this.generateId(), @@ -179,8 +183,12 @@ export class ReactiveHistoryManager extends EventEmitter { async appendAgentMessage(content: string = '') { const userConfig = await getUserConfig() - const model = this.temporaryModelOverride?.model ?? userConfig.llm.model.get() const endpointType = this.temporaryModelOverride?.endpointType ?? userConfig.llm.endpointType.get() + const model = this.temporaryModelOverride?.model ?? (endpointType === 'gemini' + ? userConfig.llm.backends.gemini.model.get() || userConfig.llm.model.get() + : endpointType === 'openai' + ? userConfig.llm.backends.openai.model.get() || userConfig.llm.model.get() + : userConfig.llm.model.get()) this.history.value.push({ id: this.generateId(), diff --git a/entrypoints/sidepanel/utils/chat/side-effects.ts b/entrypoints/sidepanel/utils/chat/side-effects.ts index e03b0980..f514b5ac 100644 --- a/entrypoints/sidepanel/utils/chat/side-effects.ts +++ b/entrypoints/sidepanel/utils/chat/side-effects.ts @@ -1,13 +1,19 @@ import { effectScope, watch } from 'vue' import { ActionMessageV1 } from '@/types/chat' +import { getHostChatMap, getPageKeyFromUrl } from '@/utils/host-chat-map' import { useGlobalI18n } from '@/utils/i18n' +import logger from '@/utils/logger' import { lazyInitialize } from '@/utils/memo' +import { s2bRpc } from '@/utils/rpc' +import { getTabStore } from '@/utils/tab-store' import { getUserConfig } from '@/utils/user-config' import { Chat } from './chat' import { welcomeMessage } from './texts' +const log = logger.child('chat-side-effects') + async function appendOrUpdateQuickActionsIfNeeded(chat: Chat) { const { t } = await useGlobalI18n() const userConfig = await getUserConfig() @@ -74,3 +80,72 @@ async function _initChatSideEffects() { } export const initChatSideEffects = lazyInitialize(_initChatSideEffects) + +/** + * Switch the active chat to the one associated with the given page key. + * Creates a new chat if no mapping exists or the previously mapped chat was deleted. + * Updates the page-chat-map after any switch/creation. + * Skips silently if the chat is currently answering. + */ +async function switchChatForPage(chat: Chat, pageKey: string | null): Promise { + if (!pageKey) return + // Don't interrupt an in-progress generation + if (chat.isAnswering()) return + const userConfig = await getUserConfig() + const map = await getHostChatMap() + const existingChatId = map.get(pageKey) + + if (existingChatId) { + if (userConfig.chat.history.currentChatId.get() === existingChatId) return + // Verify the chat still exists in storage + const chatHistory = await s2bRpc.getChatHistory(existingChatId) + if (chatHistory) { + log.debug('switchChatForPage: switching to existing chat', { pageKey, existingChatId }) + await chat.switchToChat(existingChatId) + return + } + // Chat was deleted; remove stale mapping + map.delete(pageKey) + } + + // No valid chat for this page — create a fresh one + log.debug('switchChatForPage: creating new chat for page', { pageKey }) + const newChatId = await chat.createNewChat() + map.set(pageKey, newChatId) +} + +async function _initTabChatSync() { + const chat = await Chat.getInstance() + const tabStore = await getTabStore() + const currentTabInfo = tabStore.currentTabInfo + + // Sync with the active tab immediately on startup + await switchChatForPage(chat, getPageKeyFromUrl(currentTabInfo.value.url)) + + // Re-sync when the sidepanel becomes visible again (user reopens the panel) + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible') { + switchChatForPage(chat, getPageKeyFromUrl(currentTabInfo.value.url)) + } + }) + + runInDetachedScope(() => { + // Switch chat when the user activates a different browser tab + watch(() => currentTabInfo.value.tabId, async (newTabId, oldTabId) => { + if (newTabId === oldTabId) return + await switchChatForPage(chat, getPageKeyFromUrl(currentTabInfo.value.url)) + }) + + // Keep the map up-to-date when the user manually switches / creates a chat + getUserConfig().then((userConfig) => { + watch(() => userConfig.chat.history.currentChatId.get(), async (newChatId) => { + const pageKey = getPageKeyFromUrl(currentTabInfo.value.url) + if (!pageKey) return + const map = await getHostChatMap() + map.set(pageKey, newChatId) + }) + }) + }) +} + +export const initTabChatSync = lazyInitialize(_initTabChatSync) diff --git a/entrypoints/sidepanel/utils/chat/tool-calls/index.ts b/entrypoints/sidepanel/utils/chat/tool-calls/index.ts index a1fb422a..43d40570 100644 --- a/entrypoints/sidepanel/utils/chat/tool-calls/index.ts +++ b/entrypoints/sidepanel/utils/chat/tool-calls/index.ts @@ -18,6 +18,29 @@ import { BrowserSession } from './utils/browser-use' import { makeTaskSummary } from './utils/markdown' const logger = Logger.child('tool-calls-execute') +const MAX_PAGE_CONTENT_CHARS = 24_000 + +function truncateContentForToolResult(content: string, maxChars: number = MAX_PAGE_CONTENT_CHARS) { + if (content.length <= maxChars) { + return { + content, + truncated: false, + } + } + const headChars = Math.floor(maxChars * 0.75) + const tailChars = maxChars - headChars + const truncatedContent = [ + content.slice(0, headChars), + `\n\n[Content truncated due to size: original=${content.length} chars, kept=${maxChars} chars]\n\n`, + content.slice(-tailChars), + ].join('') + return { + content: truncatedContent, + truncated: true, + originalLength: content.length, + truncatedLength: maxChars, + } +} export const executeSearchOnline: AgentToolCallExecute<'search_online'> = async ({ params, abortSignal, taskMessageModifier }) => { const { t } = await useGlobalI18n() @@ -132,13 +155,21 @@ export const executeFetchPage: AgentToolCallExecute<'fetch_page'> = async ({ par }] } else { + const normalizedPageContent = truncateContentForToolResult(content.content) taskMsg.summary = makeTaskSummary('page', t('chat.tool_calls.common.reading_success'), content.title, url) return [{ type: 'tool-result', results: { url, status: 'completed', - page_content: `URL: ${content.url}\n\n ${content.content}`, + page_content: `URL: ${content.url}\n\n${normalizedPageContent.content}`, + ...(normalizedPageContent.truncated + ? { + page_content_truncated: 'true', + page_content_original_length: normalizedPageContent.originalLength?.toString() ?? '', + page_content_truncated_length: normalizedPageContent.truncatedLength?.toString() ?? '', + } + : {}), }, }] } @@ -210,12 +241,20 @@ export const executeViewTab: AgentToolCallExecute<'view_tab'> = async ({ params, }] } taskMsg.summary = makeTaskSummary('tab', t('chat.tool_calls.common.reading_success'), tab.value.title || tab.value.url) + const normalizedTabContent = truncateContentForToolResult(content.content) return [{ type: 'tool-result', results: { tab_id: attachmentId, status: 'completed', - tab_content: `Title: ${content.title}\nURL: ${content.url}\n\n${content.content}`, + tab_content: `Title: ${content.title}\nURL: ${content.url}\n\n${normalizedTabContent.content}`, + ...(normalizedTabContent.truncated + ? { + tab_content_truncated: 'true', + tab_content_original_length: normalizedTabContent.originalLength?.toString() ?? '', + tab_content_truncated_length: normalizedTabContent.truncatedLength?.toString() ?? '', + } + : {}), }, }] } @@ -466,7 +505,7 @@ export const executePageClick: AgentToolCallExecute<'click'> = async ({ params, current_tab_info: { title: result.title, url: result.url, - content: result.content, + content: truncateContentForToolResult(result.content).content, }, }, }, diff --git a/entrypoints/sidepanel/utils/llm.ts b/entrypoints/sidepanel/utils/llm.ts index ce039d2d..d3b621a8 100644 --- a/entrypoints/sidepanel/utils/llm.ts +++ b/entrypoints/sidepanel/utils/llm.ts @@ -23,11 +23,21 @@ interface ExtraOptions { timeout?: number } +const resolveModelForEndpoint = (userConfig: Awaited>, endpointType: LLMEndpointType): string | undefined => { + if (endpointType === 'gemini') { + return userConfig.llm.backends.gemini.model.get() || userConfig.llm.model.get() + } + if (endpointType === 'openai') { + return userConfig.llm.backends.openai.model.get() || userConfig.llm.model.get() + } + return userConfig.llm.model.get() +} + export async function* streamTextInBackground(options: Parameters[0] & ExtraOptions & { temporaryModelOverride?: { model: string, endpointType: string } | null }) { const { abortSignal, timeout = DEFAULT_PENDING_TIMEOUT, temporaryModelOverride, ...restOptions } = options const userConfig = await getUserConfig() - const modelId = temporaryModelOverride?.model ?? userConfig.llm.model.get() const endpointType = (temporaryModelOverride?.endpointType as LLMEndpointType | undefined) ?? userConfig.llm.endpointType.get() + const modelId = temporaryModelOverride?.model ?? resolveModelForEndpoint(userConfig, endpointType) const reasoningPreference = userConfig.llm.reasoning.get() const computedReasoning = restOptions.autoThinking ? restOptions.reasoning @@ -52,7 +62,8 @@ export async function* streamTextInBackground(options: Parameters[0] & ExtraOptions) { const { abortSignal, timeout = DEFAULT_PENDING_TIMEOUT, ...restOptions } = options const userConfig = await getUserConfig() - const modelId = userConfig.llm.model.get() + const endpointType = userConfig.llm.endpointType.get() + const modelId = resolveModelForEndpoint(userConfig, endpointType) const reasoningPreference = userConfig.llm.reasoning.get() const computedReasoning = restOptions.autoThinking ? restOptions.reasoning @@ -77,7 +88,8 @@ export async function generateObjectInBackground(options: const { promise: abortPromise, reject } = Promise.withResolvers>>>() const { abortSignal, timeout = DEFAULT_PENDING_TIMEOUT, ...restOptions } = options const userConfig = await getUserConfig() - const modelId = userConfig.llm.model.get() + const endpointType = userConfig.llm.endpointType.get() + const modelId = resolveModelForEndpoint(userConfig, endpointType) const reasoningPreference = userConfig.llm.reasoning.get() const computedReasoning = restOptions.autoThinking ? restOptions.reasoning diff --git a/types/scroll-targets.ts b/types/scroll-targets.ts index 1d052ffb..e55e69fb 100644 --- a/types/scroll-targets.ts +++ b/types/scroll-targets.ts @@ -1 +1 @@ -export type SettingsScrollTarget = 'quick-actions-block' | 'model-download-section' | 'ollama-server-address-section' | 'lm-studio-server-address-section' +export type SettingsScrollTarget = 'quick-actions-block' | 'model-download-section' | 'ollama-server-address-section' | 'lm-studio-server-address-section' | 'gemini-api-config-section' | 'openai-api-config-section' diff --git a/utils/host-chat-map.ts b/utils/host-chat-map.ts new file mode 100644 index 00000000..d756255b --- /dev/null +++ b/utils/host-chat-map.ts @@ -0,0 +1,53 @@ +import { storage } from 'wxt/utils/storage' + +import { debounce } from './debounce' +import { LRUCache } from './lru-cache' +import { lazyInitialize } from './memo' + +const STORAGE_KEY = 'local:host-chat-map' +const MAX_SIZE = 500 + +/** + * Returns `origin` (protocol + hostname + port, e.g. `https://github.com`) as + * the cache key. Returns `null` for non-http(s) URLs. + */ +export function getPageKeyFromUrl(url: string | undefined): string | null { + if (!url) return null + try { + const urlObj = new URL(url) + if (urlObj.protocol !== 'http:' && urlObj.protocol !== 'https:') return null + return urlObj.origin + } + catch { + return null + } +} + +async function _getHostChatMap() { + const cache = new LRUCache(MAX_SIZE) + + const stored = await storage.getItem<[string, string][]>(STORAGE_KEY) + if (stored) { + cache.loadEntries(stored) + } + + const scheduleSave = debounce(async () => { + await storage.setItem(STORAGE_KEY, cache.entries()) + }, 500) + + return { + get(key: string): string | undefined { + return cache.get(key) + }, + set(key: string, chatId: string): void { + cache.set(key, chatId) + scheduleSave() + }, + delete(key: string): void { + cache.delete(key) + scheduleSave() + }, + } +} + +export const getHostChatMap = lazyInitialize(_getHostChatMap) diff --git a/utils/llm/gemini.ts b/utils/llm/gemini.ts new file mode 100644 index 00000000..727d1f38 --- /dev/null +++ b/utils/llm/gemini.ts @@ -0,0 +1,106 @@ +import logger from '../logger' +import { getUserConfig } from '../user-config' + +const log = logger.child('llm:gemini') + +export interface GeminiModelInfo { + id: string + name: string +} + +const DEFAULT_GEMINI_MODELS: GeminiModelInfo[] = [ + { + id: 'gemini-flash-latest', + name: 'Gemini Flash Latest', + }, + { + id: 'gemini-2.5-flash', + name: 'Gemini 2.5 Flash', + }, + { + id: 'gemini-2.5-pro', + name: 'Gemini 2.5 Pro', + }, +] + +const modelNameMap = new Map(DEFAULT_GEMINI_MODELS.map((model) => [model.id, model.name])) + +export let GEMINI_MODELS: GeminiModelInfo[] = [...DEFAULT_GEMINI_MODELS] + +type GeminiModelListResponse = { + data?: Array<{ id?: string }> +} + +function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/` +} + +function formatModelName(modelId: string): string { + return modelNameMap.get(modelId) ?? modelId +} + +function sortModels(models: GeminiModelInfo[]): GeminiModelInfo[] { + const priority = new Map(DEFAULT_GEMINI_MODELS.map((model, index) => [model.id, index])) + return [...models].sort((a, b) => { + const aPriority = priority.get(a.id) + const bPriority = priority.get(b.id) + if (aPriority !== undefined || bPriority !== undefined) { + return (aPriority ?? Number.MAX_SAFE_INTEGER) - (bPriority ?? Number.MAX_SAFE_INTEGER) + } + return a.id.localeCompare(b.id) + }) +} + +function normalizeModelList(modelIds: string[]): GeminiModelInfo[] { + const uniqueModelIds = [...new Set(modelIds.map((id) => id.trim()).filter(Boolean))] + const models = uniqueModelIds.map((id) => ({ + id, + name: formatModelName(id), + })) + return sortModels(models) +} + +export async function getGeminiModelList() { + const userConfig = await getUserConfig() + const baseUrl = userConfig.llm.backends.gemini.baseUrl.get() + const apiKey = userConfig.llm.backends.gemini.apiKey.get() || userConfig.llm.apiKey.get() + try { + const modelsEndpoint = new URL('models', normalizeBaseUrl(baseUrl)).href + const headers = new Headers({ + Accept: 'application/json', + }) + if (apiKey) { + headers.set('Authorization', `Bearer ${apiKey}`) + } + const response = await fetch(modelsEndpoint, { + method: 'GET', + headers, + }) + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error(`Gemini model list request failed with status ${response.status}${errorText ? `: ${errorText}` : ''}`) + } + const payload = await response.json() as GeminiModelListResponse + const models = normalizeModelList((payload.data ?? []).map((item) => item.id ?? '')) + if (models.length === 0) { + throw new Error('No models found in Gemini model list response') + } + GEMINI_MODELS = models + return { models } + } + catch (error) { + log.error('Error fetching Gemini model list:', error) + if (GEMINI_MODELS.length === 0) { + GEMINI_MODELS = [...DEFAULT_GEMINI_MODELS] + } + return { + models: GEMINI_MODELS, + error: 'Failed to fetch Gemini model list', + } + } +} + +export function isGeminiModel(modelId: string | undefined | null): boolean { + if (!modelId) return false + return GEMINI_MODELS.some((model) => model.id === modelId) +} diff --git a/utils/llm/model-logos.ts b/utils/llm/model-logos.ts index 6ce5bb37..3bc6c767 100644 --- a/utils/llm/model-logos.ts +++ b/utils/llm/model-logos.ts @@ -7,6 +7,9 @@ import LogoDeepseekUrl from '@/assets/icons/model-logo-deepseek.svg?url' import LogoFallback from '@/assets/icons/model-logo-fallback.svg?component' import LogoFallbackSvg from '@/assets/icons/model-logo-fallback.svg?raw' import LogoFallbackUrl from '@/assets/icons/model-logo-fallback.svg?url' +import LogoGemini from '@/assets/icons/model-logo-gemini.svg?component' +import LogoGeminiSvg from '@/assets/icons/model-logo-gemini.svg?raw' +import LogoGeminiUrl from '@/assets/icons/model-logo-gemini.svg?url' import LogoGemma from '@/assets/icons/model-logo-gemma.svg?component' import LogoGemmaSvg from '@/assets/icons/model-logo-gemma.svg?raw' import LogoGemmaUrl from '@/assets/icons/model-logo-gemma.svg?url' @@ -19,6 +22,9 @@ import LogoLlavaUrl from '@/assets/icons/model-logo-llava.svg?url' import LogoMistral from '@/assets/icons/model-logo-mistral.svg?component' import LogoMistralSvg from '@/assets/icons/model-logo-mistral.svg?raw' import LogoMistralUrl from '@/assets/icons/model-logo-mistral.svg?url' +import LogoOpenai from '@/assets/icons/model-logo-openai.svg?component' +import LogoOpenaiSvg from '@/assets/icons/model-logo-openai.svg?raw' +import LogoOpenaiUrl from '@/assets/icons/model-logo-openai.svg?url' import LogoPhi from '@/assets/icons/model-logo-phi.svg?component' import LogoPhiSvg from '@/assets/icons/model-logo-phi.svg?raw' import LogoPhiUrl from '@/assets/icons/model-logo-phi.svg?url' @@ -39,6 +45,12 @@ const matcher = [ svg: LogoDeepseekSvg, url: LogoDeepseekUrl, }, + { + match: /gemini/i, + component: LogoGemini, + svg: LogoGeminiSvg, + url: LogoGeminiUrl, + }, { match: /gemma/i, component: LogoGemma, @@ -69,6 +81,12 @@ const matcher = [ svg: LogoLlavaSvg, url: LogoLlavaUrl, }, + { + match: /openai|chatgpt|gpt-|^o[1-9](?:$|[-_])/i, + component: LogoOpenai, + svg: LogoOpenaiSvg, + url: LogoOpenaiUrl, + }, { match: /phi/i, component: LogoPhi, diff --git a/utils/llm/models.ts b/utils/llm/models.ts index 1ea21bd7..41e2841c 100644 --- a/utils/llm/models.ts +++ b/utils/llm/models.ts @@ -1,3 +1,4 @@ +import { createOpenAICompatible } from '@ai-sdk/openai-compatible' import { LanguageModelV1, wrapLanguageModel } from 'ai' import type { ReasoningOption } from '@/types/reasoning' @@ -6,9 +7,11 @@ import { getUserConfig } from '@/utils/user-config' import { ModelNotFoundError } from '../error' import { makeCustomFetch } from '../fetch' import logger from '../logger' +import { GEMINI_MODELS, isGeminiModel } from './gemini' import { loadModel as loadLMStudioModel } from './lm-studio' import { middlewares } from './middlewares' import { checkModelSupportThinking } from './ollama' +import { isOpenAIModel, OPENAI_MODELS } from './openai' import { LMStudioChatLanguageModel } from './providers/lm-studio/chat-language-model' import { createOllama } from './providers/ollama' import { WebLLMChatLanguageModel } from './providers/web-llm/openai-compatible-chat-language-model' @@ -20,12 +23,29 @@ export async function getModelUserConfig(overrides?: { model?: string, endpointT logger.debug('Detected override model', { overrides }) const userConfig = await getUserConfig() const endpointType = overrides?.endpointType ?? userConfig.llm.endpointType.get() - const model = overrides?.model ?? userConfig.llm.model.get() + const model = overrides?.model ?? ( + endpointType === 'gemini' + ? userConfig.llm.backends.gemini.model.get() || userConfig.llm.model.get() + : endpointType === 'openai' + ? userConfig.llm.backends.openai.model.get() || userConfig.llm.model.get() + : userConfig.llm.model.get() + ) - const baseUrl = userConfig.llm.backends[endpointType === 'lm-studio' ? 'lmStudio' : 'ollama'].baseUrl.get() - const apiKey = userConfig.llm.apiKey.get() - const numCtx = userConfig.llm.backends[endpointType === 'lm-studio' ? 'lmStudio' : 'ollama'].numCtx.get() - const enableNumCtx = userConfig.llm.backends[endpointType === 'lm-studio' ? 'lmStudio' : 'ollama'].enableNumCtx.get() + const backendKey = endpointType === 'lm-studio' + ? 'lmStudio' + : endpointType === 'gemini' + ? 'gemini' + : endpointType === 'openai' + ? 'openai' + : 'ollama' + const baseUrl = userConfig.llm.backends[backendKey].baseUrl.get() + const apiKey = endpointType === 'gemini' + ? userConfig.llm.backends.gemini.apiKey.get() || userConfig.llm.apiKey.get() + : endpointType === 'openai' + ? userConfig.llm.backends.openai.apiKey.get() || userConfig.llm.apiKey.get() + : userConfig.llm.apiKey.get() + const numCtx = userConfig.llm.backends[backendKey].numCtx.get() + const enableNumCtx = userConfig.llm.backends[backendKey].enableNumCtx.get() const reasoningPreference = userConfig.llm.reasoning.get() const reasoning = getReasoningOptionForModel(reasoningPreference, model) if (!model) { @@ -117,6 +137,24 @@ export async function getModel(options: { { supportsStructuredOutputs: true, provider: 'web-llm', defaultObjectGenerationMode: 'json' }, ) } + else if (endpointType === 'gemini') { + const normalizedBaseUrl = options.baseUrl.endsWith('/') ? options.baseUrl.slice(0, -1) : options.baseUrl + const gemini = createOpenAICompatible({ + name: 'gemini', + baseURL: normalizedBaseUrl, + apiKey: options.apiKey, + }) + model = gemini.chatModel(options.model) + } + else if (endpointType === 'openai') { + const normalizedBaseUrl = options.baseUrl.endsWith('/') ? options.baseUrl.slice(0, -1) : options.baseUrl + const openai = createOpenAICompatible({ + name: 'openai', + baseURL: normalizedBaseUrl, + apiKey: options.apiKey, + }) + model = openai.chatModel(options.model) + } else { throw new Error('Unsupported endpoint type ' + endpointType) } @@ -126,7 +164,7 @@ export async function getModel(options: { }) } -export type LLMEndpointType = 'ollama' | 'lm-studio' | 'web-llm' +export type LLMEndpointType = 'ollama' | 'lm-studio' | 'web-llm' | 'gemini' | 'openai' export function parseErrorMessageFromChunk(error: unknown): string | null { if (error && typeof error === 'object' && 'message' in error && typeof (error as { message: unknown }).message === 'string') { @@ -140,3 +178,15 @@ export function isModelSupportPDFToImages(_model: string): boolean { // but it's too slow to process large number of image so we disable this feature temporarily by returning false here return false } + +export function getGeminiModels() { + return GEMINI_MODELS +} + +export { isGeminiModel } + +export function getOpenAIModels() { + return OPENAI_MODELS +} + +export { isOpenAIModel } diff --git a/utils/llm/openai.ts b/utils/llm/openai.ts new file mode 100644 index 00000000..d3353a18 --- /dev/null +++ b/utils/llm/openai.ts @@ -0,0 +1,107 @@ +import logger from '../logger' +import { getUserConfig } from '../user-config' + +const log = logger.child('llm:openai') + +export interface OpenAIModelInfo { + id: string + name: string +} + +const NON_CHAT_MODEL_PREFIXES = [ + 'text-embedding-', + 'text-search-', + 'text-similarity-', + 'text-moderation-', + 'omni-moderation-', + 'code-search-', + 'code-cushman-', + 'code-davinci-', + 'whisper-', + 'tts-', + 'dall-e-', + 'gpt-image-', +] + +const NON_CHAT_MODEL_EXACT_IDS = new Set([ + 'babbage-002', + 'davinci-002', +]) + +const NON_CHAT_MODEL_KEYWORDS = [ + 'transcribe', +] + +export let OPENAI_MODELS: OpenAIModelInfo[] = [] + +type OpenAIModelListResponse = { + data?: Array<{ id?: string }> +} + +function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/` +} + +function sortModels(models: OpenAIModelInfo[]): OpenAIModelInfo[] { + return [...models].sort((a, b) => a.id.localeCompare(b.id)) +} + +function isNonChatModelId(modelId: string): boolean { + if (NON_CHAT_MODEL_EXACT_IDS.has(modelId)) return true + if (NON_CHAT_MODEL_PREFIXES.some((prefix) => modelId.startsWith(prefix))) return true + if (NON_CHAT_MODEL_KEYWORDS.some((keyword) => modelId.includes(keyword))) return true + return false +} + +function normalizeModelList(modelIds: string[]): OpenAIModelInfo[] { + const uniqueModelIds = [...new Set(modelIds.map((id) => id.trim()).filter(Boolean))] + const filteredModelIds = uniqueModelIds.filter((id) => !isNonChatModelId(id)) + const finalModelIds = filteredModelIds.length > 0 ? filteredModelIds : uniqueModelIds + const models = finalModelIds.map((id) => ({ + id, + name: id, + })) + return sortModels(models) +} + +export async function getOpenAIModelList() { + const userConfig = await getUserConfig() + const baseUrl = userConfig.llm.backends.openai.baseUrl.get() + const apiKey = userConfig.llm.backends.openai.apiKey.get() || userConfig.llm.apiKey.get() + try { + const modelsEndpoint = new URL('models', normalizeBaseUrl(baseUrl)).href + const headers = new Headers({ + Accept: 'application/json', + }) + if (apiKey) { + headers.set('Authorization', `Bearer ${apiKey}`) + } + const response = await fetch(modelsEndpoint, { + method: 'GET', + headers, + }) + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error(`OpenAI model list request failed with status ${response.status}${errorText ? `: ${errorText}` : ''}`) + } + const payload = await response.json() as OpenAIModelListResponse + const models = normalizeModelList((payload.data ?? []).map((item) => item.id ?? '')) + if (models.length === 0) { + throw new Error('No models found in OpenAI model list response') + } + OPENAI_MODELS = models + return { models } + } + catch (error) { + log.error('Error fetching OpenAI model list:', error) + return { + models: OPENAI_MODELS, + error: 'Failed to fetch OpenAI model list', + } + } +} + +export function isOpenAIModel(modelId: string | undefined | null): boolean { + if (!modelId) return false + return OPENAI_MODELS.some((model) => model.id === modelId) +} diff --git a/utils/lru-cache.ts b/utils/lru-cache.ts new file mode 100644 index 00000000..967b0afa --- /dev/null +++ b/utils/lru-cache.ts @@ -0,0 +1,51 @@ +export class LRUCache { + private cache: Map + readonly maxSize: number + + constructor(maxSize: number) { + this.maxSize = maxSize + this.cache = new Map() + } + + get(key: K): V | undefined { + if (!this.cache.has(key)) return undefined + const value = this.cache.get(key)! + // Move to end (most recently used) + this.cache.delete(key) + this.cache.set(key, value) + return value + } + + set(key: K, value: V): void { + if (this.cache.has(key)) { + this.cache.delete(key) + } + else if (this.cache.size >= this.maxSize) { + // Delete the first (least recently used) item + const firstKey = this.cache.keys().next().value as K + this.cache.delete(firstKey) + } + this.cache.set(key, value) + } + + delete(key: K): void { + this.cache.delete(key) + } + + has(key: K): boolean { + return this.cache.has(key) + } + + entries(): [K, V][] { + return Array.from(this.cache.entries()) + } + + loadEntries(entries: [K, V][]): void { + // Keep only the most recent maxSize entries + this.cache = new Map(entries.slice(-this.maxSize)) + } + + get size(): number { + return this.cache.size + } +} diff --git a/utils/pinia-store/store.ts b/utils/pinia-store/store.ts index e7865a93..6a476957 100644 --- a/utils/pinia-store/store.ts +++ b/utils/pinia-store/store.ts @@ -3,6 +3,8 @@ import { computed, ref } from 'vue' import { LMStudioModelInfo } from '@/types/lm-studio-models' import { OllamaModelInfo } from '@/types/ollama-models' +import { GEMINI_MODELS, GeminiModelInfo } from '@/utils/llm/gemini' +import { OPENAI_MODELS, OpenAIModelInfo } from '@/utils/llm/openai' import { logger } from '@/utils/logger' import { c2bRpc, s2bRpc, settings2bRpc } from '@/utils/rpc' @@ -19,6 +21,42 @@ const rpc = forRuntimes({ }) export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => { + const remoteCustomModelList = ref>([]) + + const updateRemoteCustomModelList = async () => { + const userConfig = await getUserConfig() + const pairs: Array<{ backend: 'gemini' | 'openai', model: string }> = [] + + const llmEndpointType = userConfig.llm.endpointType.get() + const llmModel = userConfig.llm.model.get() + if ((llmEndpointType === 'gemini' || llmEndpointType === 'openai') && llmModel) { + pairs.push({ backend: llmEndpointType, model: llmModel }) + } + + const translationEndpointType = userConfig.translation.endpointType.get() + const translationModel = userConfig.translation.model.get() + if ((translationEndpointType === 'gemini' || translationEndpointType === 'openai') && translationModel) { + pairs.push({ backend: translationEndpointType, model: translationModel }) + } + + const unique = new Map() + for (const pair of pairs) { + const isPreset = pair.backend === 'gemini' + ? geminiModelList.value.some((model) => model.id === pair.model) + : openaiModelList.value.some((model) => model.id === pair.model) + if (isPreset) continue + const key = `${pair.backend}#${pair.model}` + unique.set(key, { + backend: pair.backend, + model: pair.model, + name: `${pair.model} (Custom)`, + }) + } + + remoteCustomModelList.value = [...unique.values()] + return remoteCustomModelList.value + } + // Ollama model list and connection status const ollamaModelList = ref([]) const ollamaModelListUpdating = ref(false) @@ -115,6 +153,46 @@ export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => return success } + // Gemini model list + const geminiModelList = ref([...GEMINI_MODELS]) + const geminiModelListUpdating = ref(false) + const updateGeminiModelList = async (): Promise => { + try { + geminiModelListUpdating.value = true + const response = await rpc.getGeminiModelList() + log.debug('Gemini model list fetched:', response) + geminiModelList.value = response.models + return geminiModelList.value + } + catch (error) { + log.error('Failed to fetch Gemini model list:', error) + return geminiModelList.value + } + finally { + geminiModelListUpdating.value = false + } + } + + // OpenAI model list + const openaiModelList = ref([...OPENAI_MODELS]) + const openaiModelListUpdating = ref(false) + const updateOpenAIModelList = async (): Promise => { + try { + openaiModelListUpdating.value = true + const response = await rpc.getOpenAIModelList() + log.debug('OpenAI model list fetched:', response) + openaiModelList.value = response.models + return openaiModelList.value + } + catch (error) { + log.error('Failed to fetch OpenAI model list:', error) + return openaiModelList.value + } + finally { + openaiModelListUpdating.value = false + } + } + const checkCurrentModelSupportVision = async () => { const userConfig = await getUserConfig() const endpointType = userConfig.llm.endpointType.get() @@ -134,6 +212,20 @@ export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => return !!modelInfo?.vision } else { + if (endpointType === 'gemini') { + let models = geminiModelList.value + if (models.length === 0) { + models = await updateGeminiModelList() + } + return models.some((model) => model.id === currentModel) + } + if (endpointType === 'openai') { + let models = openaiModelList.value + if (models.length === 0) { + models = await updateOpenAIModelList() + } + return models.some((model) => model.id === currentModel) + } return false } } @@ -162,11 +254,25 @@ export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => model: m.modelKey, name: m.displayName ?? m.modelKey, })), + ...geminiModelList.value.map((m) => ({ + backend: 'gemini' as const, + model: m.id, + name: m.name, + })), + ...openaiModelList.value.map((m) => ({ + backend: 'openai' as const, + model: m.id, + name: m.name, + })), + ...remoteCustomModelList.value, ] }) const modelListUpdating = computed(() => { - return ollamaModelListUpdating.value || lmStudioModelListUpdating.value + return ollamaModelListUpdating.value + || lmStudioModelListUpdating.value + || geminiModelListUpdating.value + || openaiModelListUpdating.value }) // this function has side effects: it may change the common model in user config @@ -203,15 +309,58 @@ export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => } else { status = 'backend-unavailable' } } + else if (endpointType === 'gemini') { + const availableGeminiModels = await updateGeminiModelList() + const currentModel = commonModelConfig.get() + if (currentModel) { + status = 'ok' + } + else if (availableGeminiModels.length > 0) { + commonModelConfig.set(availableGeminiModels[0].id) + status = 'ok' + } + else { + status = 'no-model' + } + } + else if (endpointType === 'openai') { + const availableOpenAIModels = await updateOpenAIModelList() + const currentModel = commonModelConfig.get() + if (currentModel) { + status = 'ok' + } + else if (availableOpenAIModels.length > 0) { + commonModelConfig.set(availableOpenAIModels[0].id) + status = 'ok' + } + else { + status = 'no-model' + } + } + await updateRemoteCustomModelList() return { modelList, commonModel: commonModelConfig.get(), status, endpointType } } const updateModelList = async () => { logger.debug('Updating model list...') - // Always update both Ollama and LMStudio backends so users can see - // all available models when switching between backends in ModelSelector - // WebLLM doesn't need updating as it uses static SUPPORTED_MODELS - await Promise.allSettled([updateOllamaModelList(), updateLMStudioModelList()]) + const userConfig = await getUserConfig() + const llmEndpointType = userConfig.llm.endpointType.get() + const translationEndpointType = userConfig.translation.endpointType.get() + const updates: Promise[] = [] + if (llmEndpointType === 'ollama' || translationEndpointType === 'ollama') { + updates.push(updateOllamaModelList()) + } + if (llmEndpointType === 'lm-studio' || translationEndpointType === 'lm-studio') { + updates.push(updateLMStudioModelList()) + } + if (llmEndpointType === 'gemini' || translationEndpointType === 'gemini') { + updates.push(updateGeminiModelList()) + } + if (llmEndpointType === 'openai' || translationEndpointType === 'openai') { + updates.push(updateOpenAIModelList()) + } + await Promise.allSettled(updates) + await updateRemoteCustomModelList() return modelList.value } @@ -235,6 +384,14 @@ export const useLLMBackendStatusStore = defineStore('llm-backend-status', () => deleteOllamaModel, clearLMStudioModelList, updateLMStudioConnectionStatus, + // Gemini + geminiModelList, + geminiModelListUpdating, + updateGeminiModelList, + // OpenAI + openaiModelList, + openaiModelListUpdating, + updateOpenAIModelList, // Common checkCurrentModelSupportVision, checkModelSupportThinking, diff --git a/utils/rpc/background-fns.ts b/utils/rpc/background-fns.ts index b024aa8d..400d4eb9 100644 --- a/utils/rpc/background-fns.ts +++ b/utils/rpc/background-fns.ts @@ -17,9 +17,11 @@ import { MODELS_NOT_SUPPORTED_FOR_STRUCTURED_OUTPUT } from '../constants' import { ContextMenuManager } from '../context-menu' import { AiSDKError, AppError, CreateTabStreamCaptureError, FetchError, fromError, GenerateObjectSchemaError, ModelRequestError, UnknownError } from '../error' import { parsePartialJson } from '../json/parser/parse-partial-json' +import * as geminiUtils from '../llm/gemini' import * as lmStudioUtils from '../llm/lm-studio' import { getModel, getModelUserConfig, LLMEndpointType, ModelLoadingProgressEvent } from '../llm/models' import * as ollamaUtils from '../llm/ollama' +import * as openaiUtils from '../llm/openai' import { SchemaName, Schemas, selectSchema } from '../llm/output-schema' import { PromptBasedTool } from '../llm/tools/prompt-based/helpers' import { getWebLLMEngine, WebLLMSupportedModel } from '../llm/web-llm' @@ -112,6 +114,14 @@ const normalizeError = (_error: unknown, endpointType?: LLMEndpointType) => { return error } +const resolveTemperatureForEndpoint = (endpointType: LLMEndpointType, temperature?: number) => { + // Some OpenAI-compatible models reject temperature 0 and only accept default temperature 1. + if (endpointType === 'openai') { + return temperature === undefined || temperature === 0 ? 1 : temperature + } + return temperature +} + const streamText = async (options: Pick & ExtraGenerateOptionsWithTools) => { const abortController = new AbortController() const portName = `streamText-${Date.now().toString(32)}` @@ -127,6 +137,7 @@ const streamText = async (options: Pick & ExtraGenerateOptionsWithTools) => { try { + const userConfig = await getModelUserConfig({ model: options.modelId, endpointType: options.endpointType }) + const temperature = resolveTemperatureForEndpoint(userConfig.endpointType) const response = originalGenerateText({ - model: await getModel({ ...(await getModelUserConfig({ model: options.modelId, endpointType: options.endpointType })), ...generateExtraModelOptions(options) }), + model: await getModel({ ...userConfig, ...generateExtraModelOptions(options) }), messages: options.messages, prompt: options.prompt, system: options.system, tools: PromptBasedTool.createFakeAnyTools(), + temperature, maxTokens: options.maxTokens, experimental_activeTools: [], }) @@ -195,13 +210,15 @@ const generateText = async (options: Pick m.id === configuredModel) : modelList.models.length > 0, + } + } + else if (userConfig.llm.endpointType.get() === 'openai') { + const modelList = await c2bRpc.getOpenAIModelList() + const configuredModel = model ?? userConfig.llm.model.get() + return { + backend: true, + model: configuredModel ? modelList.models.some((m) => m.id === configuredModel) : modelList.models.length > 0, + } + } else { throw new UnsupportedEndpointType(userConfig.llm.endpointType.get()) } diff --git a/utils/translation-cache/key-strategy.ts b/utils/translation-cache/key-strategy.ts index 03e4e0a6..f389a505 100644 --- a/utils/translation-cache/key-strategy.ts +++ b/utils/translation-cache/key-strategy.ts @@ -94,7 +94,7 @@ function extractModelName(modelId: string): string { // Remove common prefixes first const modelName = modelId .toLowerCase() - .replace(/^(ollama|webllm|openai|anthropic|chrome-ai)[/:]?/, '') + .replace(/^(ollama|webllm|openai|anthropic|chrome-ai|gemini)[/:]?/, '') // Extract base model name by removing version suffixes and parameter specifications // Split on both '-' and ':' to handle patterns like "deepseek-r1:32b" diff --git a/utils/user-config/index.ts b/utils/user-config/index.ts index 90d37d54..3b2f57fa 100644 --- a/utils/user-config/index.ts +++ b/utils/user-config/index.ts @@ -119,6 +119,20 @@ export async function _getUserConfig() { enableNumCtx: await new Config('llm.backends.lmStudio.enableNumCtx').default(enableNumCtx).build(), baseUrl: await new Config('llm.backends.lmStudio.baseUrl').default('http://localhost:1234/api').build(), }, + gemini: { + apiKey: await new Config('llm.backends.gemini.apiKey').default('').build(), + model: await new Config('llm.backends.gemini.model').default('gemini-flash-latest').build(), + numCtx: await new Config('llm.backends.gemini.numCtx').default(1024 * 8).build(), + enableNumCtx: await new Config('llm.backends.gemini.enableNumCtx').default(false).build(), + baseUrl: await new Config('llm.backends.gemini.baseUrl').default('https://generativelanguage.googleapis.com/v1beta/openai').build(), + }, + openai: { + apiKey: await new Config('llm.backends.openai.apiKey').default('').build(), + model: await new Config('llm.backends.openai.model').default('gpt-5.4').build(), + numCtx: await new Config('llm.backends.openai.numCtx').default(1024 * 8).build(), + enableNumCtx: await new Config('llm.backends.openai.enableNumCtx').default(false).build(), + baseUrl: await new Config('llm.backends.openai.baseUrl').default('https://api.openai.com/v1').build(), + }, }, }, browserAI: { @@ -210,6 +224,12 @@ export async function _getUserConfig() { lmStudioConfig: { open: await new Config('settings.blocks.lmStudioConfig.open').default(true).build(), }, + geminiConfig: { + open: await new Config('settings.blocks.geminiConfig.open').default(true).build(), + }, + openaiConfig: { + open: await new Config('settings.blocks.openaiConfig.open').default(true).build(), + }, }, }, emailTools: { diff --git a/wxt.config.ts b/wxt.config.ts index 55776739..589df20f 100644 --- a/wxt.config.ts +++ b/wxt.config.ts @@ -56,6 +56,7 @@ export default defineConfig({ imports: false, modules: ['@wxt-dev/module-vue'], webExt: { + disabled: true, chromiumArgs: ['--user-data-dir=./.wxt/chrome-data'], }, zip: {