refactor: agent gateway

agent/README.md

@@ -1,15 +1 @@
-# Elysia with Bun runtime
-
-## Getting Started
-To get started with this template, simply paste this command into your terminal:
-```bash
-bun create elysia ./elysia-example
-```
-
-## Development
-To start the development server run:
-```bash
-bun run dev
-```
-
-Open http://localhost:3000/ with your browser to see the result.
+# @memoh/agent-gateway

agent/src/agent.ts

@@ -1,362 +1,242 @@
-import { generateText, ModelMessage, stepCountIs, streamText, TextStreamPart, ToolChoice, ToolSet } from 'ai'
-import { createChatGateway } from './gateway'
-import { AgentSkill, BaseModelConfig, Schedule } from './types'
-import { system, schedule } from './prompts'
+import { generateText, ImagePart, LanguageModelUsage, ModelMessage, stepCountIs, streamText, UserModelMessage } from 'ai'
+import { AgentInput, AgentParams, allActions, Schedule } from './types'
+import { system, schedule, user, subagentSystem } from './prompts'
 import { AuthFetcher } from './index'
-import { getScheduleTools } from './tools/schedule'
-import { getWebTools } from './tools/web'
-import { subagentSystem } from './prompts/subagent'
-import { getSubagentTools } from './tools/subagent'
-import { getSkillTools } from './tools/skill'
-import { getMemoryTools } from './tools/memory'
-import { getMessageTools } from './tools/message'
-import { getContactTools } from './tools/contact'
+import { createModel } from './model'
+import { AgentAction } from './types/action'
+import { getTools } from './tools'
 
-export enum AgentAction {
-  WebSearch = 'web_search',
-  Message = 'message',
-  Contact = 'contact',
-  Subagent = 'subagent',
-  Schedule = 'schedule',
-  Skill = 'skill',
-  Memory = 'memory',
-}
 
-export interface AgentParams extends BaseModelConfig {
-  locale?: Intl.LocalesArgument
-  language?: string
-  maxSteps?: number
-  maxContextLoadTime?: number
-  platforms?: string[]
-  currentPlatform?: string
-  braveApiKey?: string
-  braveBaseUrl?: string
-  skills?: AgentSkill[]
-  useSkills?: string[]
-  allowed?: AgentAction[]
-  toolContext?: ToolContext
-  toolChoice?: unknown
-}
-
-export interface AgentInput {
-  messages: ModelMessage[]
-  query: string
-}
-
-export interface AgentResult {
-  messages: ModelMessage[]
-  skills: string[]
-}
-
-export interface ToolContext {
-  botId?: string
-  sessionId?: string
-  currentPlatform?: string
-  replyTarget?: string
-  sessionToken?: string
-  contactId?: string
-  contactName?: string
-  contactAlias?: string
-  userId?: string
-}
-
-const withToolLogging = (tools: ToolSet): ToolSet => {
-  const wrapped: ToolSet = {}
-  for (const [name, entry] of Object.entries(tools)) {
-    const tool = entry as {
-      execute?: (input: unknown) => Promise<unknown>
-    }
-    if (!tool?.execute) {
-      wrapped[name] = entry
-      continue
-    }
-    const wrappedTool = {
-      ...(entry as Record<string, unknown>),
-      execute: async (input: unknown) => {
-        console.log('[Tool] call', { name, input })
-        try {
-          const result = await tool.execute?.(input)
-          console.log('[Tool] result', { name })
-          return result
-        } catch (error) {
-          console.error('[Tool] error', { name, error })
-          throw error
-        }
-      },
-    }
-    wrapped[name] = wrappedTool as unknown as ToolSet[string]
-  }
-  return wrapped
-}
-
-export const createAgent = (
-  params: AgentParams,
-  fetcher: AuthFetcher = fetch,
-) => {
-  const gateway = createChatGateway(params.clientType)
-  const messages: ModelMessage[] = []
-  const enabledSkills: AgentSkill[] = params.skills ?? []
-  enabledSkills.push(
-    ...params.useSkills?.map((name) => params.skills?.find((s) => s.name === name)
-  ).filter((s) => s !== undefined) ?? [])
-
-  const allowedActions = params.allowed
-    ?? Object.values(AgentAction)
-
-  const maxSteps = params.maxSteps ?? 50
-
-  const getTools = () => {
-    const tools: ToolSet = {}
-
-    if (allowedActions.includes(AgentAction.Skill)) {
-      const skillTools = getSkillTools({
-        skills: params.skills ?? [],
-        useSkill: (skill) => {
-          if (enabledSkills.some((s) => s.name === skill.name)) {
-            return
-          }
-          enabledSkills.push(skill)
-        }
-      })
-      Object.assign(tools, skillTools)
-    }
-
-    if (allowedActions.includes(AgentAction.Schedule)) {
-      const scheduleTools = getScheduleTools({ fetch: fetcher })
-      Object.assign(tools, scheduleTools)
-    }
-
-    if (params.braveApiKey && allowedActions.includes(AgentAction.WebSearch)) {
-      const webTools = getWebTools({
-        braveApiKey: params.braveApiKey,
-        braveBaseUrl: params.braveBaseUrl,
-      })
-      Object.assign(tools, webTools)
-    }
-
-    if (allowedActions.includes(AgentAction.Subagent)) {
-      const subagentTools = getSubagentTools({
-        fetch: fetcher,
-        apiKey: params.apiKey,
-        baseUrl: params.baseUrl,
-        model: params.model,
-        clientType: params.clientType,
-        braveApiKey: params.braveApiKey,
-        braveBaseUrl: params.braveBaseUrl,
-      })
-      Object.assign(tools, subagentTools)
-    }
-
-    if (allowedActions.includes(AgentAction.Memory)) {
-      const memoryTools = getMemoryTools({ fetch: fetcher })
-      Object.assign(tools, memoryTools)
-    }
-
-    if (allowedActions.includes(AgentAction.Message)) {
-      const messageTools = getMessageTools({
-        fetch: fetcher,
-        toolContext: params.toolContext,
-      })
-      Object.assign(tools, messageTools)
-    }
-
-    if (allowedActions.includes(AgentAction.Contact)) {
-      const contactTools = getContactTools({
-        fetch: fetcher,
-        toolContext: params.toolContext,
-      })
-      Object.assign(tools, contactTools)
-    }
-    
-    return withToolLogging(tools)
-  }
-
-  const generateSystem = () => {
+export const createAgent = ({
+  model: modelConfig,
+  activeContextTime = 24 * 60,
+  brave,
+  language = 'Same as the user input',
+  allowedActions = allActions,
+  identity,
+  platforms = [],
+  currentPlatform = 'Unknown Platform',
+}: AgentParams, fetch: AuthFetcher) => {
+  const model = createModel(modelConfig)
+  
+  const generateSystemPrompt = () => {
     return system({
       date: new Date(),
-      locale: params.locale,
-      language: params.language,
-      maxContextLoadTime: params.maxContextLoadTime ?? 1550,
-      platforms: params.platforms ?? [],
-      currentPlatform: params.currentPlatform,
-      skills: params.skills ?? [],
-      enabledSkills,
-      toolContext: params.toolContext,
+      language,
+      maxContextLoadTime: activeContextTime,
+      platforms,
+      skills: [],
+      enabledSkills: [],
     })
   }
 
-  const shouldForceAutoToolChoice = (error: unknown) => {
-    const message = error instanceof Error
-      ? error.message
-      : String(error ?? '')
-    if (
-      message.includes('Tool choice must be auto')
-      || message.includes('tool_choice')
-      || message.includes('No endpoints found that support the provided')
-    ) {
-      return true
+  const tools = getTools(allowedActions, {
+    fetch,
+    model: modelConfig,
+    brave,
+    identity,
+  })
+
+  const generateUserPrompt = (input: AgentInput) => {
+    const text = user(input.query, {
+      contactId: identity.contactId,
+      contactName: identity.contactName,
+      platform: currentPlatform,
+      date: new Date(),
+    })
+    const images = input.attachments.filter(attachment => attachment.type === 'image')
+    // const files = input.attachments.filter(attachment => attachment.type === 'file')
+    const userMessage: UserModelMessage = {
+      role: 'user',
+      content: [
+        { type: 'text', text },
+        ...images.map(image => ({ type: 'image', image: image.base64 }) as ImagePart),
+      ]
     }
-    if (error instanceof Error && error.cause) {
-      const causeMessage = error.cause instanceof Error
-        ? error.cause.message
-        : String(error.cause)
-      return causeMessage.includes('Tool choice must be auto')
-    }
-    return false
+    return userMessage
   }
 
-  const buildCallSettings = (toolChoice?: unknown) => {
-    const tools = getTools()
-    console.log('[Agent] tools available:', Object.keys(tools))
-    return {
-      model: gateway({
-        apiKey: params.apiKey,
-        baseURL: params.baseUrl,
-      })(params.model),
-      system: generateSystem(),
-      stopWhen: stepCountIs(maxSteps),
+  const ask = async (input: AgentInput) => {
+    const userPrompt = generateUserPrompt(input)
+    const messages = [...input.messages, userPrompt]
+    const { response, reasoning, text, usage } = await generateText({
+      model,
       messages,
+      system: generateSystemPrompt(),
+      stopWhen: stepCountIs(Infinity),
       prepareStep: () => {
         return {
-          system: generateSystem(),
+          system: generateSystemPrompt(),
         }
       },
       tools,
-      toolChoice: toolChoice as ToolChoice<ToolSet> | undefined,
-    }
-  }
-
-  const ask = async (input: AgentInput): Promise<AgentResult> => {
-    messages.push(...input.messages)
-    const user: ModelMessage = {
-      role: 'user',
-      content: input.query,
-    }
-    messages.push(user)
-    let response
-    try {
-      const result = await generateText(buildCallSettings(params.toolChoice))
-      response = result.response
-    } catch (error) {
-      if (params.toolChoice && shouldForceAutoToolChoice(error)) {
-        console.warn('[Chat] toolChoice rejected, fallback to auto')
-        const result = await generateText(buildCallSettings('auto'))
-        response = result.response
-      } else {
-        throw error
-      }
-    }
-    return {
-      messages: [user, ...response.messages],
-      skills: enabledSkills.map((s) => s.name),
-    }
-  }
-
-  const askAsSubagent = async (
-    input: AgentInput,
-    options: {
-      name: string
-      description?: string
-    }
-  ): Promise<AgentResult> => {
-    messages.push(...input.messages)
-    const user: ModelMessage = {
-      role: 'user',
-      content: input.query,
-    }
-    messages.push(user)
-    const { response } = await generateText({
-      model: gateway({
-        apiKey: params.apiKey,
-        baseURL: params.baseUrl,
-      })(params.model),
-      system: subagentSystem({ date: new Date(), name: options.name, description: options.description }),
-      stopWhen: stepCountIs(maxSteps),
-      messages,
-      prepareStep: () => {
-        return {
-          system: subagentSystem({ date: new Date(), name: options.name, description: options.description }),
-        }
-      },
-      tools: getTools(),
     })
     return {
-      messages: [user, ...response.messages],
-      skills: enabledSkills.map((s) => s.name),
+      messages: [userPrompt, ...response.messages],
+      reasoning: reasoning.map(part => part.text),
+      usage,
+      text,
     }
   }
 
-  async function* stream(input: AgentInput): AsyncGenerator<TextStreamPart<ToolSet>, AgentResult> {
-    messages.push(...input.messages)
-    const user: ModelMessage = {
+  const askAsSubagent = async (params: {
+    input: string
+    name: string
+    description: string
+    messages: ModelMessage[]
+  }) => {
+    const userPrompt: UserModelMessage = {
       role: 'user',
-      content: input.query,
+      content: [
+        { type: 'text', text: params.input },
+      ]
     }
-    messages.push(user)
-    let response
-    let fullStream
-    try {
-      const result = streamText(buildCallSettings(params.toolChoice))
-      response = result.response
-      fullStream = result.fullStream
-    } catch (error) {
-      if (params.toolChoice && shouldForceAutoToolChoice(error)) {
-        console.warn('[Chat] toolChoice rejected, fallback to auto')
-        const result = streamText(buildCallSettings('auto'))
-        response = result.response
-        fullStream = result.fullStream
-      } else {
-        throw error
-      }
-    }
-    for await (const event of fullStream) {
-      yield event
-    }
-    return {
-      messages: [user, ...(await response).messages],
-      skills: enabledSkills.map((s) => s.name),
-    }
-  }
-
-  const triggerSchedule = async (
-    input: AgentInput,
-    scheduleData: Schedule
-  ): Promise<AgentResult> => {
-    messages.push(...input.messages)
-    const user: ModelMessage = {
-      role: 'user',
-      content: schedule({
-        schedule: scheduleData,
-        locale: params.locale,
+    const generateSubagentSystemPrompt = () => {
+      return subagentSystem({
         date: new Date(),
-      }),
+        name: params.name,
+        description: params.description,
+      })
     }
-    messages.push(user)
-    const { response } = await generateText({
-      model: gateway({
-        apiKey: params.apiKey,
-        baseURL: params.baseUrl,
-      })(params.model),
-      system: generateSystem(),
-      stopWhen: stepCountIs(maxSteps),
+    const messages = [...params.messages, userPrompt]
+    const { response, reasoning, text, usage } = await generateText({
+      model,
       messages,
+      system: generateSubagentSystemPrompt(),
+      stopWhen: stepCountIs(Infinity),
       prepareStep: () => {
         return {
-          system: generateSystem(),
+          system: generateSubagentSystemPrompt(),
         }
       },
-      tools: getTools(),
+      tools,
     })
     return {
-      messages: [user, ...response.messages],
-      skills: enabledSkills.map((s) => s.name),
+      messages: [userPrompt, ...response.messages],
+      reasoning: reasoning.map(part => part.text),
+      usage,
+      text,
+    }
+  }
+
+  const triggerSchedule = async (params: {
+    schedule: Schedule
+    messages: ModelMessage[]
+  }) => {
+    const scheduleMessage: UserModelMessage = {
+      role: 'user',
+      content: [
+        { type: 'text', text: schedule({ schedule: params.schedule, date: new Date() }) },
+      ]
+    }
+    const messages = [...params.messages, scheduleMessage]
+    const { response, reasoning, text, usage } = await generateText({
+      model,
+      messages,
+      system: generateSystemPrompt(),
+      stopWhen: stepCountIs(Infinity),
+    })
+    return {
+      messages: [scheduleMessage, ...response.messages],
+      reasoning: reasoning.map(part => part.text),
+      usage,
+      text,
+    }
+  }
+
+  async function* stream(input: AgentInput): AsyncGenerator<AgentAction> {
+    const userPrompt = generateUserPrompt(input)
+    const messages = [...input.messages, userPrompt]
+    const result: {
+      messages: ModelMessage[]
+      reasoning: string[]
+      usage: LanguageModelUsage | null
+    } = {
+      messages: [],
+      reasoning: [],
+      usage: null
+    }
+    const { fullStream } = streamText({
+      model,
+      messages,
+      system: generateSystemPrompt(),
+      stopWhen: stepCountIs(Infinity),
+      prepareStep: () => {
+        return {
+          system: generateSystemPrompt(),
+        }
+      },
+      tools,
+      onFinish: ({ usage, reasoning, response }) => {
+        result.usage = usage as never
+        result.reasoning = reasoning.map(part => part.text)
+        result.messages = response.messages
+      }
+    })
+    yield {
+      type: 'agent_start',
+      input,
+    }
+    for await (const chunk of fullStream) {
+      switch (chunk.type) {
+        case 'reasoning-start': yield {
+          type: 'reasoning_start',
+          metadata: chunk
+        }; break
+        case 'reasoning-delta': yield {
+          type: 'reasoning_delta',
+          delta: chunk.text
+        }; break
+        case 'reasoning-end': yield {
+          type: 'reasoning_end',
+          metadata: chunk
+        }; break
+        case 'text-start': yield {
+          type: 'text_start',
+        }; break
+        case 'text-delta': yield {
+          type: 'text_delta',
+          delta: chunk.text
+        }; break
+        case 'text-end': yield {
+          type: 'text_end',
+          metadata: chunk
+        }; break
+        case 'tool-call': yield {
+          type: 'tool_call_start',
+          toolName: chunk.toolName,
+          toolCallId: chunk.toolCallId,
+          input: chunk.input,
+          metadata: chunk
+        }; break
+        case 'tool-result': yield {
+          type: 'tool_call_end',
+          toolName: chunk.toolName,
+          toolCallId: chunk.toolCallId,
+          input: chunk.input,
+          result: chunk.output,
+          metadata: chunk
+        }; break
+        case 'file': yield {
+          type: 'image_delta',
+          image: chunk.file.base64,
+          metadata: chunk
+        }
+      }
+    }
+    yield {
+      type: 'agent_end',
+      messages: [userPrompt, ...result.messages],
+      skills: [],
+      reasoning: result.reasoning,
+      usage: result.usage!,
     }
   }
 
   return {
-    ask,
     stream,
-    triggerSchedule,
+    ask,
     askAsSubagent,
+    triggerSchedule,
   }
 }

agent/src/gateway.ts

@@ -1,23 +0,0 @@
-import { createGateway as createAiGateway } from 'ai'
-import { createOpenAI } from '@ai-sdk/openai'
-import { createAnthropic } from '@ai-sdk/anthropic'
-import { createGoogleGenerativeAI } from '@ai-sdk/google'
-import { ClientType } from './types'
-
-export const createChatGateway = (clientType: ClientType) => {
-  if (clientType === ClientType.OPENAI) {
-    return (options: Parameters<typeof createOpenAI>[0]) => {
-      const openai = createOpenAI(options)
-      const baseURL = (options?.baseURL ?? '').toLowerCase()
-      if (baseURL.includes('openrouter.ai') || baseURL.includes('dashscope.aliyuncs.com')) {
-        return openai.chat
-      }
-      return openai
-    }
-  }
-  const clients = {
-    [ClientType.ANTHROPIC]: createAnthropic,
-    [ClientType.GOOGLE]: createGoogleGenerativeAI,
-  }
-  return (clients[clientType] ?? createAiGateway)
-}

agent/src/model.ts

@@ -0,0 +1,21 @@
+import { createGateway as createAiGateway } from 'ai'
+import { createOpenAI } from '@ai-sdk/openai'
+import { createAnthropic } from '@ai-sdk/anthropic'
+import { createGoogleGenerativeAI } from '@ai-sdk/google'
+import { ClientType, ModelConfig } from './types'
+
+export const createModel = (model: ModelConfig) => {
+  const apiKey = model.apiKey.toLowerCase().trim()
+  const baseURL = model.baseUrl.toLowerCase().trim()
+  const modelId = model.modelId.toLowerCase().trim()
+  const clients = {
+    [ClientType.OpenAI]: createOpenAI,
+    [ClientType.OpenAICompatible]: createOpenAI,
+    [ClientType.Anthropic]: createAnthropic,
+    [ClientType.Google]: createGoogleGenerativeAI,
+  }
+  return (clients[model.clientType] ?? createAiGateway)({
+    apiKey,
+    baseURL,
+  })(modelId)
+}

agent/src/models.ts

@@ -0,0 +1,43 @@
+import z from 'zod'
+import { allActions } from './types'
+
+export const AgentSkillModel = z.object({
+  name: z.string().min(1, 'Skill name is required'),
+  description: z.string().min(1, 'Skill description is required'),
+  content: z.string().min(1, 'Skill content is required'),
+  metadata: z.record(z.string(), z.any()).optional(),
+})
+
+export const ClientTypeModel = z.enum(['openai', 'openai-compatible', 'anthropic', 'google'])
+
+export const ModelConfigModel = z.object({
+  modelId: z.string().min(1, 'Model ID is required'),
+  clientType: ClientTypeModel,
+  input: z.array(z.enum(['text', 'image'])),
+  apiKey: z.string().min(1, 'API key is required'),
+  baseUrl: z.string(),
+})
+
+export const AllowedActionModel = z.enum(allActions)
+
+export const IdentityContextModel = z.object({
+  botId: z.string().min(1, 'Bot ID is required'),
+  sessionId: z.string().min(1, 'Session ID is required'),
+  containerId: z.string().min(1, 'Container ID is required'),
+  contactId: z.string().min(1, 'Contact ID is required'),
+  contactName: z.string().min(1, 'Contact name is required'),
+  contactAlias: z.string().optional(),
+  userId: z.string().optional(),
+  currentPlatform: z.string().optional(),
+  replyTarget: z.string().optional(),
+  sessionToken: z.string().optional(),
+})
+
+export const ScheduleModel = z.object({
+  id: z.string().min(1, 'Schedule ID is required'),
+  name: z.string().min(1, 'Schedule name is required'),
+  description: z.string().min(1, 'Schedule description is required'),
+  pattern: z.string().min(1, 'Schedule pattern is required'),
+  maxCalls: z.number().nullable().optional(),
+  command: z.string().min(1, 'Schedule command is required'),
+})

agent/src/modules/chat.ts

@@ -2,219 +2,63 @@ import { Elysia, sse } from 'elysia'
 import z from 'zod'
 import { createAgent } from '../agent'
 import { createAuthFetcher } from '../index'
-import { ClientType } from '../types'
-import { ModelMessage } from 'ai'
+import { ModelConfig } from '../types'
 import { bearerMiddleware } from '../middlewares/bearer'
-import { loadConfig } from '../config'
-
-const Skill = z.object({
-  name: z.string().min(1, 'Skill name is required'),
-  description: z.string().min(1, 'Skill description is required'),
-  content: z.string().min(1, 'Skill content is required'),
-})
-
-const ChatBody = z.object({
-  apiKey: z.string().min(1, 'API key is required'),
-  baseUrl: z.string().min(1, 'Base URL is required'),
-  model: z.string().min(1, 'Model is required'),
-  clientType: z.enum([
-    'openai',
-    'anthropic',
-    'google',
-  ]),
-  locale: z.string().optional(),
-  language: z.string().optional(),
-  maxSteps: z.number().optional(),
-  maxContextLoadTime: z.number().min(1, 'Max context load time is required'),
-  platforms: z.array(z.string()).optional(),
-  currentPlatform: z.string().optional(),
-  skills: z.array(Skill).optional(),
-  useSkills: z.array(z.string()).optional(),
+import { AllowedActionModel, IdentityContextModel, ModelConfigModel } from '../models'
+import { allActions } from '../types'
 
+const AgentModel = z.object({
+  model: ModelConfigModel,
+  activeContextTime: z.number(),
+  platforms: z.array(z.string()),
+  currentPlatform: z.string(),
+  allowedActions: z.array(AllowedActionModel).optional().default(allActions),
   messages: z.array(z.any()),
-  query: z.string().min(1, 'Query is required'),
-  toolContext: z.object({
-    botId: z.string().optional(),
-    sessionId: z.string().optional(),
-    currentPlatform: z.string().optional(),
-    replyTarget: z.string().optional(),
-    sessionToken: z.string().optional(),
-    contactId: z.string().optional(),
-    contactName: z.string().optional(),
-    contactAlias: z.string().optional(),
-    userId: z.string().optional(),
-  }).optional(),
-  toolChoice: z.union([
-    z.literal('auto'),
-    z.literal('none'),
-    z.literal('required'),
-    z.object({
-      type: z.literal('tool'),
-      toolName: z.string(),
-    }),
-  ]).nullable().optional(),
+  skills: z.array(z.string()),
+  query: z.string(),
+  identity: IdentityContextModel,
 })
 
-const ScheduleBody = z.object({
-  schedule: z.object({
-    id: z.string().min(1, 'Schedule ID is required'),
-    name: z.string().min(1, 'Schedule name is required'),
-    description: z.string().min(1, 'Schedule description is required'),
-    pattern: z.string().min(1, 'Schedule pattern is required'),
-    maxCalls: z.number().nullable().optional(),
-    command: z.string().min(1, 'Schedule command is required'),
-  }),
-}).and(ChatBody)
-
-const config = loadConfig('../config.toml')
-
 export const chatModule = new Elysia({ prefix: '/chat' })
   .use(bearerMiddleware)
   .post('/', async ({ body, bearer }) => {
+    const authFetcher = createAuthFetcher(bearer)
     const { ask } = createAgent({
-      apiKey: body.apiKey,
-      baseUrl: body.baseUrl,
-      model: body.model,
-      clientType: body.clientType as ClientType,
-      locale: body.locale,
-      language: body.language,
-      maxSteps: body.maxSteps,
-      maxContextLoadTime: body.maxContextLoadTime,
+      model: body.model as ModelConfig,
+      activeContextTime: body.activeContextTime,
       platforms: body.platforms,
       currentPlatform: body.currentPlatform,
-      braveApiKey: config.brave?.api_key,
-      braveBaseUrl: config.brave?.base_url,
+      allowedActions: body.allowedActions,
+      identity: body.identity,
+    }, authFetcher)
+    return ask({
+      query: body.query,
+      messages: body.messages,
       skills: body.skills,
-      useSkills: body.useSkills,
-      toolContext: body.toolContext,
-      toolChoice: body.toolChoice,
-    }, createAuthFetcher(bearer))
-    try {
-      const result = await ask({
-        messages: body.messages as unknown as ModelMessage[],
-        query: body.query,
-      })
-      console.log('[Chat] response', {
-        type: 'chat',
-        messages: result.messages?.length ?? 0,
-        toolChoice: body.toolChoice ?? null,
-      })
-      // Debug: log message structure
-      if (result.messages?.length > 0) {
-        console.log('[Chat] message sample', JSON.stringify(result.messages[result.messages.length - 1], null, 2))
-      }
-      return result
-    } catch (error) {
-      console.error('[Chat] error', {
-        type: 'chat',
-        clientType: body.clientType,
-        model: body.model,
-        baseUrl: body.baseUrl,
-        error,
-      })
-      throw error
-    }
+      attachments: [],
+    })
   }, {
-    body: ChatBody,
+    body: AgentModel,
   })
   .post('/stream', async function* ({ body, bearer }) {
-    console.log('[Chat] request', {
-      type: 'stream',
-      clientType: body.clientType,
-      model: body.model,
-      baseUrl: body.baseUrl,
-      bearer,
-      toolChoice: body.toolChoice ?? null,
-    })
+    const authFetcher = createAuthFetcher(bearer)
     const { stream } = createAgent({
-      apiKey: body.apiKey,
-      baseUrl: body.baseUrl,
-      model: body.model,
-      clientType: body.clientType as ClientType,
-      locale: body.locale,
-      language: body.language,
-      maxSteps: body.maxSteps,
-      maxContextLoadTime: body.maxContextLoadTime,
+      model: body.model as ModelConfig,
+      activeContextTime: body.activeContextTime,
       platforms: body.platforms,
       currentPlatform: body.currentPlatform,
-      braveApiKey: config.brave?.api_key,
-      braveBaseUrl: config.brave?.base_url,
+      allowedActions: body.allowedActions,
+      identity: body.identity,
+    }, authFetcher)
+    for await (const action of stream({
+      query: body.query,
+      messages: body.messages,
       skills: body.skills,
-      useSkills: body.useSkills,
-      toolContext: body.toolContext,
-      toolChoice: body.toolChoice,
-    }, createAuthFetcher(bearer))
-    try {
-      const streanGenerator = stream({
-        messages: body.messages as unknown as ModelMessage[],
-        query: body.query,
-      })
-      while (true) {
-        const chunk = await streanGenerator.next()
-        if (chunk.done) {
-          console.log('[Chat] response', { type: 'stream', messages: chunk.value?.messages?.length ?? 0 })
-          yield sse({
-            type: 'done',
-            data: chunk.value,
-          })
-          break
-        }
-        yield sse({
-          type: 'delta',
-          data: chunk.value
-        })
-      }
-    } catch (error) {
-      console.error('[Chat] error', {
-        type: 'stream',
-        clientType: body.clientType,
-        model: body.model,
-        baseUrl: body.baseUrl,
-        error,
-      })
-      throw error
+      attachments: [],
+    })) {
+      yield sse(JSON.stringify(action))
     }
   }, {
-    body: ChatBody,
+    body: AgentModel,
   })
-  .post('/schedule', async ({ body, bearer }) => {
-    console.log('[Chat] schedule request', {
-      type: 'schedule',
-      bearer,
-      body,
-    })
-    const { triggerSchedule } = createAgent({
-      apiKey: body.apiKey,
-      baseUrl: body.baseUrl,
-      model: body.model,
-      clientType: body.clientType as ClientType,
-      locale: body.locale,
-      language: body.language,
-      maxSteps: body.maxSteps,
-      maxContextLoadTime: body.maxContextLoadTime,
-      platforms: body.platforms,
-      currentPlatform: body.currentPlatform,
-      braveApiKey: config.brave?.api_key,
-      braveBaseUrl: config.brave?.base_url,
-      skills: body.skills,
-      useSkills: body.useSkills,
-      toolContext: body.toolContext,
-      toolChoice: body.toolChoice,
-    }, createAuthFetcher(bearer))
-    try {
-      return await triggerSchedule({
-        messages: body.messages as unknown as ModelMessage[],
-        query: body.query,
-      }, body.schedule)
-    } catch (error) {
-      console.error('[Chat] schedule error', {
-        type: 'schedule',
-        bearer,
-        body,
-        error,
-      })
-      throw error
-    }
-  }, {
-    body: ScheduleBody,
-  })
+  

agent/src/prompts/index.ts

@@ -1,4 +1,5 @@
 export * from './system'
 export * from './schedule'
-export * from './shared'
+export * from './user'
+export * from './subagent'
 export * from './utils'

agent/src/prompts/schedule.ts

@@ -1,26 +1,24 @@
 import { Schedule } from '../types'
-import { time } from './shared'
 
 export interface ScheduleParams {
   schedule: Schedule
-  locale?: Intl.LocalesArgument
   date: Date
 }
 
 export const schedule = (params: ScheduleParams) => {
+  const headers = {
+    'schedule-name': params.schedule.name,
+    'schedule-description': params.schedule.description,
+    'schedule-id': params.schedule.id,
+    'max-calls': params.schedule.maxCalls ?? 'Unlimited',
+    'cron-pattern': params.schedule.pattern,
+  }
   return `
+** This is a scheduled task automatically send to you by the system **
 ---
-notice: **This is a scheduled task automatically send to you by the system, not the user input**
-${time({ date: params.date, locale: params.locale })}
-schedule-name: ${params.schedule.name}
-schedule-description: ${params.schedule.description}
-schedule-id: ${params.schedule.id}
-max-calls: ${params.schedule.maxCalls ?? 'Unlimited'}
-cron-pattern: ${params.schedule.pattern}
+${Bun.YAML.stringify(headers)}
 ---
 
-**COMMAND**
-
 ${params.schedule.command}
   `.trim()
 }

agent/src/prompts/shared.ts

@@ -1,9 +0,0 @@
-export const time = (params: {
-  date: Date
-  locale?: Intl.LocalesArgument
-}) => {
-  return `
-date: ${params.date.toLocaleDateString(params.locale)}
-time: ${params.date.toLocaleTimeString(params.locale)}
-  `.trim()
-}

agent/src/prompts/subagent.ts

@@ -1,5 +1,3 @@
-import { time } from './shared'
-
 export interface SubagentParams {
   date: Date
   name: string
@@ -7,11 +5,14 @@ export interface SubagentParams {
 }
 
 export const subagentSystem = ({ date, name, description }: SubagentParams) => {
+  const headers = {
+    'name': name,
+    'description': description,
+    'time-now': date.toISOString(),
+  }
   return `
 ---
-${time({ date })}
-name: ${name}
-description: ${description}
+${Bun.YAML.stringify(headers)}
 ---
 
 You are a subagent, which is a specialized assistant for a specific task.

agent/src/prompts/system.ts

@@ -1,57 +1,42 @@
-import { time } from './shared'
 import { quote } from './utils'
 import { AgentSkill } from '../types'
 
 export interface SystemParams {
   date: Date
-  locale?: Intl.LocalesArgument
-  language?: string
+  language: string
   maxContextLoadTime: number
   platforms: string[]
-  currentPlatform?: string
   skills: AgentSkill[]
   enabledSkills: AgentSkill[]
-  toolContext?: ToolContext
-}
-
-export interface ToolContext {
-  botId?: string
-  sessionId?: string
-  currentPlatform?: string
-  replyTarget?: string
-  sessionToken?: string
-  contactId?: string
-  contactName?: string
-  contactAlias?: string
-  userId?: string
 }
 
 export const skillPrompt = (skill: AgentSkill) => {
   return `
-### ${skill.name}
+**${quote(skill.name)}**
 > ${skill.description}
 
 ${skill.content}
   `.trim()
 }
 
-export const system = ({ date, locale, language, maxContextLoadTime, platforms, currentPlatform, skills, enabledSkills, toolContext }: SystemParams) => {
-  const toolContextBlock = [
-    toolContext?.botId ? `bot-id: ${toolContext.botId}` : '',
-    toolContext?.sessionId ? `session-id: ${toolContext.sessionId}` : '',
-    toolContext?.replyTarget ? `reply-target: ${toolContext.replyTarget}` : '',
-    toolContext?.contactId ? `contact-id: ${toolContext.contactId}` : '',
-    toolContext?.contactName ? `contact-name: ${toolContext.contactName}` : '',
-    toolContext?.contactAlias ? `contact-alias: ${toolContext.contactAlias}` : '',
-  ].filter(Boolean).join('\n')
+export const system = ({ 
+  date,
+  language,
+  maxContextLoadTime,
+  platforms,
+  skills,
+  enabledSkills,
+}: SystemParams) => {
+  const headers = {
+    'language': language,
+    'available-platforms': platforms.join(','),
+    'max-context-load-time': maxContextLoadTime.toString(),
+    'time-now': date.toISOString(),
+  }
+
   return `
 ---
-${time({ date, locale })}
-language: ${language ?? 'Same as user input'}
-available-platforms:
-${platforms.map(platform => `  - ${platform}`).join('\n')}
-current-platform: ${currentPlatform ?? 'Unknown Platform'}
-${toolContextBlock ? toolContextBlock : ''}
+${Bun.YAML.stringify(headers)}
 ---
 You are a personal housekeeper assistant, which able to manage the master's daily affairs.
 

agent/src/prompts/user.ts

@@ -0,0 +1,24 @@
+export interface UserParams {
+  contactId: string
+  contactName: string
+  platform: string
+  date: Date
+}
+
+export const user = (
+  query: string,
+  { contactId, contactName, platform, date }: UserParams
+) => {
+  const headers = {
+    'contact-id': contactId,
+    'contact-name': contactName,
+    'platform': platform,
+    'time': date.toISOString(),
+  }
+  return `
+---
+${Bun.YAML.stringify(headers)}
+---
+${query}
+  `.trim()
+}

agent/src/tools/index.ts

@@ -1,3 +1,38 @@
-export { getWebTools } from './web'
-export { getScheduleTools } from './schedule'
+import { AuthFetcher } from '..'
+import { AgentAction, BraveConfig, IdentityContext, ModelConfig } from '../types'
+import { ToolSet } from 'ai'
+import { getWebTools } from './web'
+import { getScheduleTools } from './schedule'
+import { getMemoryTools } from './memory'
+import { getSubagentTools } from './subagent'
 
+export interface ToolsParams {
+  fetch: AuthFetcher
+  model: ModelConfig
+  brave?: BraveConfig
+  identity: IdentityContext
+}
+
+export const getTools = (
+  actions: AgentAction[],
+  { fetch, model, brave, identity }: ToolsParams
+) => {
+  const tools: ToolSet = {}
+  if (actions.includes(AgentAction.Web) && brave) {
+    const webTools = getWebTools({ brave })
+    Object.assign(tools, webTools)
+  }
+  if (actions.includes(AgentAction.Schedule)) {
+    const scheduleTools = getScheduleTools({ fetch })
+    Object.assign(tools, scheduleTools)
+  }
+  if (actions.includes(AgentAction.Memory)) {
+    const memoryTools = getMemoryTools({ fetch })
+    Object.assign(tools, memoryTools)
+  }
+  if (actions.includes(AgentAction.Subagent)) {
+    const subagentTools = getSubagentTools({ fetch, model, brave, identity })
+    Object.assign(tools, subagentTools)
+  }
+  return tools
+}

agent/src/tools/subagent.ts

@@ -1,16 +1,18 @@
 import { tool } from 'ai'
 import { z } from 'zod'
-import { AgentAction, createAgent } from '../agent'
-import { BaseModelConfig } from '../types'
+import { createAgent } from '../agent'
+import { ModelConfig, BraveConfig } from '../types'
 import { AuthFetcher } from '..'
+import { AgentAction, IdentityContext } from '../types/agent'
 
-export interface SubagentToolParams extends BaseModelConfig {
+export interface SubagentToolParams {
   fetch: AuthFetcher
-  braveApiKey?: string
-  braveBaseUrl?: string
+  model: ModelConfig
+  brave?: BraveConfig
+  identity: IdentityContext
 }
 
-export const getSubagentTools = ({ fetch, apiKey, baseUrl, model, clientType, braveApiKey, braveBaseUrl }: SubagentToolParams) => {
+export const getSubagentTools = ({ fetch, model, brave, identity }: SubagentToolParams) => {
   const listSubagents = tool({
     description: 'List subagents for current user',
     inputSchema: z.object({}),
@@ -65,20 +67,19 @@ export const getSubagentTools = ({ fetch, apiKey, baseUrl, model, clientType, br
       const contextPayload = await contextResponse.json()
       const contextMessages = Array.isArray(contextPayload?.messages) ? contextPayload.messages : []
       const { askAsSubagent } = createAgent({
-        apiKey,
-        baseUrl,
         model,
-        clientType,
-        braveApiKey,
-        braveBaseUrl,
-        allowed: [
-          AgentAction.WebSearch,
-        ]
-      })
+        brave,
+        allowedActions: [
+          AgentAction.Web,
+        ],
+        identity,
+      }, fetch)
       const result = await askAsSubagent({
         messages: contextMessages,
-        query,
-      }, { name, description: target.description })
+        input: query,
+        name: target.name,
+        description: target.description,
+      })
       const updatedMessages = [...contextMessages, ...result.messages]
       await fetch(`/subagents/${target.id}/context`, {
         method: 'PUT',

agent/src/tools/web.ts

@@ -3,12 +3,12 @@ import { z } from 'zod'
 import { Readability } from '@mozilla/readability'
 import { JSDOM } from 'jsdom'
 import TurndownService from 'turndown'
+import { BraveConfig } from '../types'
 
 const turndownService = new TurndownService()
 
 interface WebToolParams {
-  braveApiKey: string
-  braveBaseUrl?: string
+  brave: BraveConfig
 }
 
 interface BraveSearchResult {
@@ -25,7 +25,8 @@ interface BraveSearchResponse {
   }
 }
 
-export const getWebTools = ({ braveApiKey, braveBaseUrl = 'https://api.search.brave.com/res/v1/' }: WebToolParams) => {
+export const getWebTools = ({ brave }: WebToolParams) => {
+  const { apiKey, baseUrl = 'https://api.search.brave.com/res/v1/' } = brave
   const webSearch = tool({
     description: 'Search the web for information using Brave Search API. Use this when you need current information, facts, news, or any web content.',
     inputSchema: z.object({
@@ -34,7 +35,7 @@ export const getWebTools = ({ braveApiKey, braveBaseUrl = 'https://api.search.br
     }),
     execute: async ({ query, count = 10 }) => {
       try {
-        const url = new URL('web/search', braveBaseUrl)
+        const url = new URL('web/search', baseUrl)
         url.searchParams.append('q', query)
         url.searchParams.append('count', Math.min(count, 20).toString())
         
@@ -43,7 +44,7 @@ export const getWebTools = ({ braveApiKey, braveBaseUrl = 'https://api.search.br
           headers: {
             'Accept': 'application/json',
             'Accept-Encoding': 'gzip',
-            'X-Subscription-Token': braveApiKey,
+            'X-Subscription-Token': apiKey,
           },
         })
 

agent/src/types.ts

@@ -1,27 +0,0 @@
-export enum ClientType {
-  OPENAI = 'openai',
-  ANTHROPIC = 'anthropic',
-  GOOGLE = 'google',
-}
-
-export interface BaseModelConfig {
-  apiKey: string
-  baseUrl: string
-  model: string
-  clientType: ClientType
-}
-
-export interface Schedule {
-  id: string
-  name: string
-  description: string
-  pattern: string
-  maxCalls?: number | null
-  command: string
-}
-
-export interface AgentSkill {
-  name: string
-  description: string
-  content: string
-}

agent/src/types/action.ts

@@ -0,0 +1,86 @@
+import { LanguageModelUsage, ModelMessage } from 'ai'
+import { AgentInput } from './agent'
+import { AgentAttachment } from './attachment'
+
+export interface BaseAction {
+  type: string
+  metadata?: Record<string, unknown>
+}
+
+export interface AgentStartAction extends BaseAction {
+  type: 'agent_start'
+  input: AgentInput
+}
+
+export interface ReasoningStartAction extends BaseAction {
+  type: 'reasoning_start'
+}
+
+export interface ReasoningDeltaAction extends BaseAction {
+  type: 'reasoning_delta'
+  delta: string
+}
+
+export interface ReasoningEndAction extends BaseAction {
+  type: 'reasoning_end'
+}
+
+export interface TextStartAction extends BaseAction {
+  type: 'text_start'
+}
+
+export interface TextDeltaAction extends BaseAction {
+  type: 'text_delta'
+  delta: string
+}
+
+export interface AttachmentDeltaAction extends BaseAction {
+  type: 'attachment_delta'
+  attachments: AgentAttachment[]
+}
+
+export interface ImageDeltaAction extends BaseAction {
+  type: 'image_delta'
+  image: string
+}
+
+export interface TextEndAction extends BaseAction {
+  type: 'text_end'
+}
+
+export interface ToolCallStartAction extends BaseAction {
+  type: 'tool_call_start'
+  toolName: string
+  toolCallId: string
+  input: unknown
+}
+
+export interface ToolCallEndAction extends BaseAction {
+  type: 'tool_call_end'
+  toolName: string
+  toolCallId: string
+  input: unknown
+  result: unknown
+}
+
+export interface AgentEndAction extends BaseAction {
+  type: 'agent_end'
+  messages: ModelMessage[]
+  skills: string[]
+  reasoning: string[]
+  usage: LanguageModelUsage
+}
+
+export type AgentAction = 
+  | AgentStartAction
+  | ReasoningStartAction
+  | ReasoningDeltaAction
+  | ReasoningEndAction
+  | TextStartAction
+  | TextDeltaAction
+  | AttachmentDeltaAction
+  | ImageDeltaAction
+  | TextEndAction
+  | ToolCallStartAction
+  | ToolCallEndAction
+  | AgentEndAction

agent/src/types/agent.ts

@@ -0,0 +1,61 @@
+import { ModelMessage } from 'ai'
+import { ModelConfig } from './model'
+import { AgentAttachment } from './attachment'
+
+export interface IdentityContext {
+  botId: string
+  sessionId: string
+  containerId: string
+
+  contactId: string
+  contactName: string
+  contactAlias?: string
+  userId?: string
+
+  currentPlatform?: string
+  replyTarget?: string
+  sessionToken?: string
+}
+
+export enum AgentAction {
+  Web = 'web',
+  Message = 'message',
+  Contact = 'contact',
+  Subagent = 'subagent',
+  Schedule = 'schedule',
+  Skill = 'skill',
+  Memory = 'memory',
+}
+
+export const allActions = Object.values(AgentAction)
+
+export interface BraveConfig {
+  apiKey: string
+  baseUrl: string
+}
+
+export interface AgentParams {
+  model: ModelConfig
+  language?: string
+  activeContextTime?: number
+  allowedActions?: AgentAction[]
+  brave?: BraveConfig
+  identity: IdentityContext
+  platforms?: string[]
+  currentPlatform?: string
+}
+
+export interface AgentInput {
+  messages: ModelMessage[]
+  attachments: AgentAttachment[]
+  skills: string[]
+  query: string
+}
+
+export interface AgentSkill {
+  name: string
+  description: string
+  content: string
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  metadata: Record<string, any>
+}

agent/src/types/attachment.ts

@@ -0,0 +1,16 @@
+export interface BaseAgentAttachment {
+  type: string
+  metadata: Record<string, unknown>
+}
+
+export interface ImageAttachment extends BaseAgentAttachment {
+  type: 'image'
+  base64: string
+}
+
+export interface ContainerFileAttachment extends BaseAgentAttachment {
+  type: 'file'
+  path: string
+}
+
+export type AgentAttachment = ImageAttachment | ContainerFileAttachment

agent/src/types/index.ts

@@ -0,0 +1,4 @@
+export * from './agent'
+export * from './model'
+export * from './schedule'
+export * from './attachment'

agent/src/types/model.ts

@@ -0,0 +1,19 @@
+export enum ClientType {
+  OpenAI = 'openai',
+  OpenAICompatible = 'openai-compatible',
+  Anthropic = 'anthropic',
+  Google = 'google',
+}
+
+export enum ModelInput {
+  Text = 'text',
+  Image = 'image',
+}
+
+export interface ModelConfig {
+  apiKey: string
+  baseUrl: string
+  modelId: string
+  clientType: ClientType
+  input: ModelInput[]
+}

agent/src/types/schedule.ts

@@ -0,0 +1,8 @@
+export interface Schedule {
+  id: string
+  name: string
+  description: string
+  pattern: string
+  maxCalls?: number | null
+  command: string
+}