refactor: use mem0 as long-memory maneger

packages/agent/.gitignore

@@ -1 +1,2 @@
-screenshot-*.png
+screenshot-*.png
+memory.db

packages/agent/client/README.md

@@ -7,11 +7,27 @@ A command-line interface for the personal housekeeper assistant agent.
 1. Create a `.env` file in the project root (not in this directory) with the following variables:
 
 ```env
+# Main chat model
 MODEL=gpt-4o
 BASE_URL=https://api.openai.com/v1
 API_KEY=your-api-key-here
-EMBEDDING_MODEL=text-embedding-3-small
 MODEL_CLIENT_TYPE=openai
+
+# Embedding model for memory search (using mem0ai)
+EMBEDDING_MODEL=text-embedding-3-small
+EMBEDDING_BASE_URL=https://api.openai.com/v1
+EMBEDDING_API_KEY=your-api-key-here
+EMBEDDING_CLIENT_TYPE=openai
+EMBEDDING_DIMENSIONS=1536
+
+# Summary model for memory generation (optional, defaults to main model)
+SUMMARY_MODEL=gpt-4o-mini
+SUMMARY_BASE_URL=https://api.openai.com/v1
+SUMMARY_API_KEY=your-api-key-here
+SUMMARY_CLIENT_TYPE=openai
+
+# Database
+DATABASE_URL=postgresql://user:password@localhost:5432/byte
 ```
 
 2. Make sure the database is set up and running (required for memory storage).
@@ -33,9 +49,9 @@ bun run index.ts
 ## Features
 
 - **Interactive Chat**: Type your messages and get responses from the AI agent
-- **Long-term Memory**: Conversations are automatically saved and can be recalled
+- **Long-term Memory**: Conversations are automatically saved with LLM-generated summaries
 - **Context Loading**: Automatically loads recent conversations (last 60 minutes)
-- **Memory Search**: The agent can search through past conversations using natural language
+- **Memory Search**: The agent can search through past conversations using natural language and embeddings
 - **Tool Calling**: Supports automatic tool execution with multi-step reasoning
 - **Multi-Provider Support**: Works with OpenAI, Anthropic, and Google AI (via Vercel AI SDK)
 
@@ -46,9 +62,46 @@ bun run index.ts
 
 ## Environment Variables
 
-- `MODEL`: The LLM model ID (e.g., `gpt-4o`, `claude-3-5-sonnet-20241022`, `gemini-pro`)
-- `BASE_URL`: The API base URL
-- `API_KEY`: Your API key
-- `EMBEDDING_MODEL`: The embedding model for memory search (e.g., `text-embedding-3-small`)
-- `MODEL_CLIENT_TYPE`: The model provider type (default: `openai`, options: `openai`, `anthropic`, `google`)
+### Required
+
+- `MODEL`: The main LLM model ID (e.g., `gpt-4o`, `claude-3-5-sonnet-20241022`)
+- `BASE_URL`: The API base URL for the main model
+- `API_KEY`: Your API key for the main model
+- `EMBEDDING_MODEL`: The embedding model for memory search (e.g., `text-embedding-3-small`)
+- `DATABASE_URL`: PostgreSQL connection string with pgvector extension enabled
+
+### Optional
+
+- `MODEL_CLIENT_TYPE`: The model provider type (default: `openai`, options: `openai`, `anthropic`, `google`)
+- `EMBEDDING_BASE_URL`: Base URL for embedding API (default: same as `BASE_URL`)
+- `EMBEDDING_API_KEY`: API key for embedding (default: same as `API_KEY`)
+- `EMBEDDING_CLIENT_TYPE`: Provider type for embedding (default: `openai`)
+- `EMBEDDING_DIMENSIONS`: The dimensions of the embedding model (default: `1536`)
+- `SUMMARY_MODEL`: The model used to summarize conversations for memory (default: same as `MODEL`)
+- `SUMMARY_BASE_URL`: Base URL for summary model (default: same as `BASE_URL`)
+- `SUMMARY_API_KEY`: API key for summary model (default: same as `API_KEY`)
+- `SUMMARY_CLIENT_TYPE`: Provider type for summary model (default: same as `MODEL_CLIENT_TYPE`)
+
+## Memory System
+
+The agent uses [mem0ai](https://github.com/mem0ai/mem0) for sophisticated memory management:
+
+1. **Conversation Storage**: After each conversation, mem0ai uses an LLM to extract and store key information
+2. **Embedding Generation**: The extracted information is converted to embedding vectors using your configured embedding model
+3. **Vector Storage**: Embeddings are stored in PostgreSQL with pgvector extension
+4. **Semantic Search**: The agent can search past memories using natural language queries via vector similarity
+5. **Context Loading**: Recent conversations are automatically loaded from history into context
+
+### Model Recommendations
+
+- **Main Model**: Use a powerful model like `gpt-4o` or `claude-3-5-sonnet-20241022` for best conversation quality
+- **Summary Model**: Use a cheaper/faster model like `gpt-4o-mini` for memory extraction to save costs
+- **Embedding Model**: Use `text-embedding-3-small` (1536 dims) or `text-embedding-3-large` (3072 dims) for OpenAI
+
+### Database Requirements
+
+The memory system requires PostgreSQL with the `pgvector` extension installed. Make sure:
+1. PostgreSQL is installed and running
+2. pgvector extension is enabled: `CREATE EXTENSION IF NOT EXISTS vector;`
+3. mem0ai will automatically create the required tables on first run
 

packages/agent/client/index.ts

@@ -1,36 +1,61 @@
 import { createInterface } from 'node:readline'
 import { stdin as input, stdout as output } from 'node:process'
 import { createAgent } from '../src/agent'
-import { createMemorySearch, createAddMemory, filterByTimestamp, MemoryUnit } from '@memohome/memory'
-import { ModelClientType } from '@memohome/shared'
+import { createMemory, filterByTimestamp, MemoryUnit } from '@memohome/memory'
+import { ModelClientType, ChatModel, EmbeddingModel } from '@memohome/shared'
 
 // Load environment variables
 const MODEL = process.env.MODEL
 const BASE_URL = process.env.BASE_URL
 const API_KEY = process.env.API_KEY
-const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL
 const MODEL_CLIENT_TYPE = process.env.MODEL_CLIENT_TYPE || 'openai'
 
+const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL
+const EMBEDDING_BASE_URL = process.env.EMBEDDING_BASE_URL || BASE_URL
+const EMBEDDING_API_KEY = process.env.EMBEDDING_API_KEY || API_KEY
+const EMBEDDING_CLIENT_TYPE = process.env.EMBEDDING_CLIENT_TYPE || 'openai'
+const EMBEDDING_DIMENSIONS = parseInt(process.env.EMBEDDING_DIMENSIONS || '1536', 10)
+
+const SUMMARY_MODEL = process.env.SUMMARY_MODEL || MODEL
+const SUMMARY_BASE_URL = process.env.SUMMARY_BASE_URL || BASE_URL
+const SUMMARY_API_KEY = process.env.SUMMARY_API_KEY || API_KEY
+const SUMMARY_CLIENT_TYPE = process.env.SUMMARY_CLIENT_TYPE || MODEL_CLIENT_TYPE
+
 if (!MODEL || !BASE_URL || !API_KEY || !EMBEDDING_MODEL) {
   console.error('Error: Missing required environment variables')
   console.error('Required: MODEL, BASE_URL, API_KEY, EMBEDDING_MODEL')
   console.error('Optional: MODEL_CLIENT_TYPE (default: openai)')
+  console.error('Optional: SUMMARY_MODEL (default: same as MODEL)')
+  console.error('Optional: SUMMARY_BASE_URL (default: same as BASE_URL)')
+  console.error('Optional: SUMMARY_API_KEY (default: same as API_KEY)')
+  console.error('Optional: SUMMARY_CLIENT_TYPE (default: same as MODEL_CLIENT_TYPE)')
   process.exit(1)
 }
 
 const USER_ID = 'cli-user'
 
-// Create memory functions
-const searchMemory = createMemorySearch({
-  model: EMBEDDING_MODEL,
-  apiKey: API_KEY,
-  baseURL: BASE_URL,
-})
+// Create model configurations
+const embeddingModel: EmbeddingModel = {
+  modelId: EMBEDDING_MODEL!,
+  baseUrl: EMBEDDING_BASE_URL!,
+  apiKey: EMBEDDING_API_KEY!,
+  clientType: EMBEDDING_CLIENT_TYPE as ModelClientType,
+  dimensions: EMBEDDING_DIMENSIONS,
+  name: `Embedding: ${EMBEDDING_MODEL}`,
+}
 
-const addMemory = createAddMemory({
-  model: EMBEDDING_MODEL,
-  apiKey: API_KEY,
-  baseURL: BASE_URL,
+const summaryModel: ChatModel = {
+  modelId: SUMMARY_MODEL!,
+  baseUrl: SUMMARY_BASE_URL!,
+  apiKey: SUMMARY_API_KEY!,
+  clientType: SUMMARY_CLIENT_TYPE as ModelClientType,
+  name: `Summary: ${SUMMARY_MODEL}`,
+}
+
+// Create memory instance
+const memoryInstance = createMemory({
+  summaryModel,
+  embeddingModel,
 })
 
 // Create agent
@@ -48,17 +73,19 @@ const agent = createAgent({
     return await filterByTimestamp(from, to, USER_ID)
   },
   onSearchMemory: async (query: string) => {
-    return await searchMemory({ user: USER_ID, query, maxResults: 5 })
+    const results = await memoryInstance.searchMemory(query, USER_ID)
+    // Transform search results to MemoryUnit format
+    // Note: mem0ai returns semantic search results, not full conversation history
+    return results
   },
   onFinish: async (messages) => {
-    // Save conversation to memory - type conversion handled internally
+    // Save conversation to memory
     const memoryUnit: MemoryUnit = {
       messages: messages as unknown as MemoryUnit['messages'],
       timestamp: new Date(),
       user: USER_ID,
-      raw: '', // will be generated by addMemory
     }
-    await addMemory({ memory: memoryUnit })
+    await memoryInstance.addMemory(memoryUnit)
   },
 })
 
@@ -67,7 +94,7 @@ async function main() {
   console.log('Type your message and press Enter. Type "exit" to quit.\n')
 
   // Load context
-  await agent.loadContext()
+  // await agent.loadContext()
 
   const rl = createInterface({ input, output })
 

packages/agent/package.json

@@ -14,11 +14,19 @@
     "@ai-sdk/anthropic": "^3.0.9",
     "@ai-sdk/google": "^3.0.6",
     "@ai-sdk/openai": "^3.0.7",
+    "@memohome/ai-gateway": "workspace:*",
     "@memohome/memory": "workspace:*",
     "@memohome/shared": "workspace:*",
     "ai": "^6.0.25",
     "dotenv": "^17.2.3",
+    "sqlite3": "^5.1.7",
     "xsai": "^0.4.1",
     "zod": "^4.3.5"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "sqlite3",
+      "mem0ai"
+    ]
   }
 }

packages/agent/src/agent.ts

@@ -2,22 +2,17 @@ import { streamText, ModelMessage, stepCountIs } from 'ai'
 import { AgentParams } from './types'
 import { system } from './prompts'
 import { getMemoryTools } from './tools'
-import { MemoryUnit } from '@memohome/memory'
-import { createGateway } from './gateway'
+import { createChatGateway } from '@memohome/ai-gateway'
 
 export const createAgent = (params: AgentParams) => {
   const messages: ModelMessage[] = []
-  const memory: MemoryUnit[] = []
 
-  const gateway = createGateway(params.model)
+  const gateway = createChatGateway(params.model)
 
   const getTools = async () => {
     return {
       ...getMemoryTools({
-        searchMemory: params.onSearchMemory ?? (() => Promise.resolve([])),
-        onLoadMemory: async (memory) => {
-          memory.push(...memory)
-        },
+        searchMemory: params.onSearchMemory ?? (() => Promise.resolve([]))
       }),
     }
   }
@@ -36,12 +31,11 @@ export const createAgent = (params: AgentParams) => {
       language: params.language ?? 'Same as user input',
       locale: params.locale,
       maxContextLoadTime: params.maxContextLoadTime,
-      memory,
     })
   }
 
   async function* ask(input: string) {
-    await loadContext()
+    // await loadContext()
     const user = {
       role: 'user',
       content: input,

packages/agent/src/gateway.ts

@@ -1,17 +0,0 @@
-import { createGateway as createAiGateway } from 'ai'
-import { createOpenAI } from '@ai-sdk/openai'
-import { createAnthropic } from '@ai-sdk/anthropic'
-import { createGoogleGenerativeAI } from '@ai-sdk/google'
-import { BaseModel, ModelClientType } from '@memohome/shared'
-
-export const createGateway = (model: BaseModel) => {
-  const clients = {
-    [ModelClientType.OPENAI]: createOpenAI,
-    [ModelClientType.ANTHROPIC]: createAnthropic,
-    [ModelClientType.GOOGLE]: createGoogleGenerativeAI,
-  }
-  return (clients[model.clientType] ?? createAiGateway)({
-    apiKey: model.apiKey,
-    baseURL: model.baseUrl,
-  })(model.modelId)
-}

packages/agent/src/prompts/system.ts

@@ -1,15 +1,13 @@
-import { MemoryUnit } from '@memohome/memory'
-import { block, quote } from './utils'
+import { quote } from './utils'
 
 export interface SystemParams {
   date: Date
   locale?: Intl.LocalesArgument
   language: string
   maxContextLoadTime: number
-  memory: MemoryUnit[]
 }
 
-export const system = ({ date, locale, language, maxContextLoadTime, memory }: SystemParams) => {
+export const system = ({ date, locale, language, maxContextLoadTime }: SystemParams) => {
   return `
   ---
   date: ${date.toLocaleDateString(locale)}
@@ -26,9 +24,5 @@ export const system = ({ date, locale, language, maxContextLoadTime, memory }: S
   **Memory**
   - Your context has been loaded from the last ${maxContextLoadTime} minutes.
   - You can use ${quote('search-memory')} to search for past memories with natural language.
-  - The search result is performed as chat history, load into your system prompt as a context.
-
-  **Past Memory Loaded**
-  ${block(memory.map(m => m.raw).join('\n\n'), 'memory')}
   `.trim()
 }

packages/agent/src/tools/memory.ts

@@ -1,13 +1,11 @@
-import { MemoryUnit } from '@memohome/memory'
 import { tool } from 'ai'
 import { z } from 'zod'
 
 export interface GetMemoryToolParams {
-  searchMemory: (query: string) => Promise<MemoryUnit[]>
-  onLoadMemory: (memory: MemoryUnit[]) => Promise<void>
+  searchMemory: (query: string) => Promise<object[]>
 }
 
-export const getMemoryTools = ({ searchMemory, onLoadMemory }: GetMemoryToolParams) => {
+export const getMemoryTools = ({ searchMemory }: GetMemoryToolParams) => {
   const searchMemoryTool = tool({
     description: 'Search chat history in the memory',
     inputSchema: z.object({
@@ -15,10 +13,10 @@ export const getMemoryTools = ({ searchMemory, onLoadMemory }: GetMemoryToolPara
     }),
     execute: async ({ query }) => {
       const memory = await searchMemory(query)
-      onLoadMemory(memory)
+      console.log(memory)
       return {
         success: true,
-        message: `${memory.length} memories has load into your context`,
+        memories: memory,
       }
     },
   })

packages/agent/src/types.ts

@@ -1,9 +1,9 @@
 import type { MemoryUnit } from '@memohome/memory'
-import { BaseModel } from '@memohome/shared'
+import { ChatModel } from '@memohome/shared'
 import { ModelMessage } from 'ai'
 
 export interface AgentParams {
-  model: BaseModel
+  model: ChatModel
 
   /**
    * Unit: minutes
@@ -20,7 +20,7 @@ export interface AgentParams {
 
   onReadMemory?: (from: Date, to: Date) => Promise<MemoryUnit[]>
 
-  onSearchMemory?: (query: string) => Promise<MemoryUnit[]>
+  onSearchMemory?: (query: string) => Promise<object[]>
 
   onFinish?: (messages: ModelMessage[]) => Promise<void>