diff --git a/agent/src/agent.ts b/agent/src/agent.ts index 9f1082f4..ccd3858a 100644 --- a/agent/src/agent.ts +++ b/agent/src/agent.ts @@ -37,14 +37,15 @@ export const createAgent = ({ }) const generateUserPrompt = (input: AgentInput) => { + const images = input.attachments.filter(attachment => attachment.type === 'image') + const files = input.attachments.filter(attachment => attachment.type === 'file') const text = user(input.query, { contactId: identity.contactId, contactName: identity.contactName, platform: currentPlatform, date: new Date(), + attachments: files, }) - const images = input.attachments.filter(attachment => attachment.type === 'image') - // const files = input.attachments.filter(attachment => attachment.type === 'file') const userMessage: UserModelMessage = { role: 'user', content: [ diff --git a/agent/src/models.ts b/agent/src/models.ts index 666e89ac..e4df5b6d 100644 --- a/agent/src/models.ts +++ b/agent/src/models.ts @@ -40,4 +40,18 @@ export const ScheduleModel = z.object({ pattern: z.string().min(1, 'Schedule pattern is required'), maxCalls: z.number().nullable().optional(), command: z.string().min(1, 'Schedule command is required'), -}) \ No newline at end of file +}) + +export const ImageAttachmentModel = z.object({ + type: z.literal('image'), + base64: z.string().min(1, 'Image base64 is required'), + metadata: z.record(z.string(), z.any()).optional(), +}) + +export const FileAttachmentModel = z.object({ + type: z.literal('file'), + path: z.string().min(1, 'File path is required'), + metadata: z.record(z.string(), z.any()).optional(), +}) + +export const AttachmentModel = z.union([ImageAttachmentModel, FileAttachmentModel]) \ No newline at end of file diff --git a/agent/src/modules/chat.ts b/agent/src/modules/chat.ts index bba620d2..5a107c5b 100644 --- a/agent/src/modules/chat.ts +++ b/agent/src/modules/chat.ts @@ -4,7 +4,7 @@ import { createAgent } from '../agent' import { createAuthFetcher } from '../index' import { ModelConfig } from '../types' import { bearerMiddleware } from '../middlewares/bearer' -import { AllowedActionModel, IdentityContextModel, ModelConfigModel } from '../models' +import { AllowedActionModel, AttachmentModel, IdentityContextModel, ModelConfigModel } from '../models' import { allActions } from '../types' const AgentModel = z.object({ @@ -17,6 +17,7 @@ const AgentModel = z.object({ skills: z.array(z.string()), query: z.string(), identity: IdentityContextModel, + attachments: z.array(AttachmentModel), }) export const chatModule = new Elysia({ prefix: '/chat' }) @@ -35,7 +36,7 @@ export const chatModule = new Elysia({ prefix: '/chat' }) query: body.query, messages: body.messages, skills: body.skills, - attachments: [], + attachments: body.attachments, }) }, { body: AgentModel, @@ -54,7 +55,7 @@ export const chatModule = new Elysia({ prefix: '/chat' }) query: body.query, messages: body.messages, skills: body.skills, - attachments: [], + attachments: body.attachments, })) { yield sse(JSON.stringify(action)) } diff --git a/agent/src/prompts/user.ts b/agent/src/prompts/user.ts index b9089df6..6741d2e8 100644 --- a/agent/src/prompts/user.ts +++ b/agent/src/prompts/user.ts @@ -1,19 +1,23 @@ +import { ContainerFileAttachment } from '../types' + export interface UserParams { contactId: string contactName: string platform: string date: Date + attachments: ContainerFileAttachment[] } export const user = ( query: string, - { contactId, contactName, platform, date }: UserParams + { contactId, contactName, platform, date, attachments }: UserParams ) => { const headers = { 'contact-id': contactId, 'contact-name': contactName, 'platform': platform, 'time': date.toISOString(), + 'attachments': attachments.map(attachment => attachment.path), } return ` --- diff --git a/agent/src/types/attachment.ts b/agent/src/types/attachment.ts index b2fa779a..ea6df047 100644 --- a/agent/src/types/attachment.ts +++ b/agent/src/types/attachment.ts @@ -1,6 +1,6 @@ export interface BaseAgentAttachment { type: string - metadata: Record + metadata?: Record } export interface ImageAttachment extends BaseAgentAttachment {