From f012df4c48334b84981da92f3236f143e41a6fbb Mon Sep 17 00:00:00 2001 From: Abdul Rahman Date: Tue, 16 Jun 2026 00:33:33 +0530 Subject: [PATCH 1/2] Refactor agent chat streaming and chat execution services for improved file handling - Removed dependency on `FileUrlService` in `AgentChatStreamingService` and simplified message processing by directly mapping database parts to UI message parts. - Introduced `inlineFilePartsAsBase64` utility to handle inlining of file parts as base64 in `ChatExecutionService`, enhancing file content retrieval and integration. - Updated `ChatExecutionService` to utilize the new utility for processing messages with inlined file content, improving overall message handling efficiency. --- .../services/agent-chat-streaming.service.ts | 34 +++---------- .../services/chat-execution.service.ts | 19 ++++++- .../utils/inline-file-parts-as-base64.util.ts | 50 +++++++++++++++++++ 3 files changed, 73 insertions(+), 30 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts index 8c8f5483484c7..f748930c2f9d3 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/agent-chat-streaming.service.ts @@ -4,13 +4,11 @@ import { generateId } from 'ai'; import { type ExtendedFileUIPart, type ExtendedUIMessagePart, - isExtendedFileUIPart, } from 'twenty-shared/ai'; import { FileFolder } from 'twenty-shared/types'; import { In, Like } from 'typeorm'; import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity'; -import { FileUrlService } from 'src/engine/core-modules/file/file-url/file-url.service'; import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; @@ -57,7 +55,6 @@ export class AgentChatStreamingService { private readonly messageQueueService: MessageQueueService, private readonly agentChatService: AgentChatService, private readonly eventPublisherService: AgentChatEventPublisherService, - private readonly fileUrlService: FileUrlService, ) {} async streamAgentChat({ @@ -271,31 +268,12 @@ export class AgentChatStreamingService { (message) => message.status !== AgentMessageStatus.QUEUED, ); - return Promise.all( - filteredMessages.map(async (message) => ({ - id: message.id, - role: message.role as 'user' | 'assistant' | 'system', - parts: await Promise.all( - mapDBPartsToUIMessageParts(message.parts ?? []).map(async (part) => { - if (isExtendedFileUIPart(part as Record)) { - const filePart = part as ExtendedFileUIPart; - - return { - ...filePart, - url: await this.fileUrlService.signFileByIdUrl({ - fileId: filePart.fileId, - workspaceId, - fileFolder: FileFolder.AgentChat, - }), - } as ExtendedFileUIPart; - } - - return part; - }), - ), - createdAt: message.createdAt, - })), - ); + return filteredMessages.map((message) => ({ + id: message.id, + role: message.role as 'user' | 'assistant' | 'system', + parts: mapDBPartsToUIMessageParts(message.parts), + createdAt: message.createdAt, + })); } private async buildFilePartsFromAttachments( diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts index ec794b2cf2305..c9504f92f6eee 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts @@ -12,7 +12,7 @@ import { type UIMessage, type UITools, } from 'ai'; -import { AppPath } from 'twenty-shared/types'; +import { AppPath, FileFolder } from 'twenty-shared/types'; import { getAppPath, isDefined } from 'twenty-shared/utils'; import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service'; @@ -25,6 +25,7 @@ import { type CodeExecutionStreamEmitter } from 'src/engine/core-modules/tool-pr import { CodeInterpreterService } from 'src/engine/core-modules/code-interpreter/code-interpreter.service'; import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; +import { FileService } from 'src/engine/core-modules/file/services/file.service'; import { ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service'; import { createExecuteToolTool, @@ -58,6 +59,7 @@ import { getCallLevelCacheProviderOptions, injectCacheBreakpoint, } from 'src/engine/metadata-modules/ai/ai-chat/utils/inject-cache-breakpoint.util'; +import { inlineFilePartsAsBase64 } from 'src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util'; import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-telemetry.const'; import { AiModelRegistryService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service'; import { NativeToolBinderService } from 'src/engine/metadata-modules/ai/ai-models/services/native-tool-binder.service'; @@ -100,6 +102,7 @@ export class ChatExecutionService { private readonly nativeToolBinder: NativeToolBinderService, private readonly messagePruningService: MessagePruningService, private readonly metricsService: MetricsService, + private readonly fileService: FileService, ) {} async streamChat({ @@ -263,7 +266,19 @@ export class ChatExecutionService { providerOptions: getCacheProviderOptions(registeredModel.sdkPackage), }; - const rawModelMessages = await convertToModelMessages(processedMessages); + const messagesWithInlinedFiles = await inlineFilePartsAsBase64( + processedMessages, + (fileId) => + this.fileService.getFileContentById({ + fileId, + workspaceId: workspace.id, + fileFolder: FileFolder.AgentChat, + }), + ); + + const rawModelMessages = await convertToModelMessages( + messagesWithInlinedFiles, + ); const pruningResult = this.messagePruningService.pruneIfOverContextWindowLimit( diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts new file mode 100644 index 0000000000000..da5988813ec34 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts @@ -0,0 +1,50 @@ +import { type UIMessage } from 'ai'; +import { isExtendedFileUIPart } from 'twenty-shared/ai'; +import { isDefined } from 'twenty-shared/utils'; + +type FileContent = { + buffer: Buffer; + mimeType: string; +}; + +type LoadFileContent = (fileId: string) => Promise; + +export const inlineFilePartsAsBase64 = async ( + messages: UIMessage[], + loadFileContent: LoadFileContent, +): Promise => { + return Promise.all( + messages.map(async (message) => { + const inlinedParts = await Promise.all( + message.parts.map(async (part) => { + if (!isExtendedFileUIPart(part)) { + return part; + } + + if (part.url.startsWith('data:')) { + return part; + } + + const content = await loadFileContent(part.fileId); + + if (!isDefined(content)) { + return null; + } + + return { + ...part, + mediaType: content.mimeType, + url: `data:${content.mimeType};base64,${content.buffer.toString( + 'base64', + )}`, + }; + }), + ); + + return { + ...message, + parts: inlinedParts.filter(isDefined), + }; + }), + ); +}; From 0af7e375e0c0a0e47f8749f7c44dd6eefd664308 Mon Sep 17 00:00:00 2001 From: Abdul Rahman Date: Tue, 16 Jun 2026 07:02:08 +0530 Subject: [PATCH 2/2] Enhance file handling in ChatExecutionService and inlineFilePartsAsBase64 utility - Updated `ChatExecutionService` to log warnings when AI chat attachments cannot be loaded, improving error visibility. - Modified `inlineFilePartsAsBase64` to return a placeholder message for unavailable attachments, ensuring better user feedback in chat messages. - These changes enhance the robustness of file content retrieval and improve overall user experience in the chat interface. --- .../ai/ai-chat/services/chat-execution.service.ts | 15 ++++++++++++--- .../utils/inline-file-parts-as-base64.util.ts | 9 +++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts index c9504f92f6eee..db84aeac6aa19 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts @@ -268,12 +268,21 @@ export class ChatExecutionService { const messagesWithInlinedFiles = await inlineFilePartsAsBase64( processedMessages, - (fileId) => - this.fileService.getFileContentById({ + async (fileId) => { + const content = await this.fileService.getFileContentById({ fileId, workspaceId: workspace.id, fileFolder: FileFolder.AgentChat, - }), + }); + + if (!isDefined(content)) { + this.logger.warn( + `Could not load AI chat attachment ${fileId} for workspace ${workspace.id}; it will be marked as unavailable in the prompt.`, + ); + } + + return content; + }, ); const rawModelMessages = await convertToModelMessages( diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts index da5988813ec34..9e84c9bd52a1a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/inline-file-parts-as-base64.util.ts @@ -28,7 +28,12 @@ export const inlineFilePartsAsBase64 = async ( const content = await loadFileContent(part.fileId); if (!isDefined(content)) { - return null; + return { + type: 'text' as const, + text: `[Attachment${ + part.filename ? ` "${part.filename}"` : '' + } could not be loaded and is unavailable.]`, + }; } return { @@ -43,7 +48,7 @@ export const inlineFilePartsAsBase64 = async ( return { ...message, - parts: inlinedParts.filter(isDefined), + parts: inlinedParts, }; }), );